Commit 7fd3f5b5 by Rafael Monnerat

Update Release Candidate

2 parents 45299ea7 b71ce50e
Showing 71 changed files with 233 additions and 276 deletions
......@@ -15,3 +15,4 @@ slapos.cookbook.egg-info
*.egg/
TEST_KNOWN_HOSTS
node_modules
update-release.sh
[buildout]
extends =
../../component/golang/buildout.cfg
gowork.cfg
parts =
gowork
caddy
[gowork]
# Caddy 1.x+ uses go modules, for which gowork does not work yet
golang = ${golang1.12:location}
install =
github.com/mholt/caddy
[gowork.goinstall]
command = :
depends =
${caddy:recipe}
[caddy]
recipe = slapos.recipe.cmmi
path = ${go_github.com_mholt_caddy:location}
go = ${gowork:golang}/bin/go
configure-command = :
make-targets =
make-binary = cd ${:path}/caddy && ${:go} install -v
environment =
PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory}
# revision and repository can be used to control which caddy version is used
revision = db2741c6e0a1c06340391c5b9fa282b876a33361
repository = github.com/mholt/caddy/caddy
recipe = plone.recipe.command
update-command = ${:command}
stop-on-error = True
# GO111MODULE=on enables go modules support
# the chmod is needed as modules are fetched with u-w
command =
. ${gowork:env.sh} &&
cd ${gowork:directory} &&
export GO111MODULE=on &&
go get ${:repository}@${:revision} &&
chmod -R u+w .
output = ${gowork:bin}/caddy
location = ${:output}
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_mholt_caddy:recipe}
[go_github.com_mholt_caddy]
<= go-git-package
go.importpath = github.com/mholt/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v0.11.1-5-gdd393ce3a67e6a773be87185528a00f2e0a9eb26
[buildout]
parts =
lz4
[lz4]
recipe = slapos.recipe.cmmi
url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz
md5sum = d5ce78f7b1b76002bbfffa6f78a5fc4e
configure-command = true
......@@ -9,11 +9,13 @@ extends =
../jemalloc/buildout.cfg
../libaio/buildout.cfg
../libxml2/buildout.cfg
../lz4/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pkgconfig/buildout.cfg
../readline/buildout.cfg
../snappy/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
../unixodbc/buildout.cfg
......@@ -26,8 +28,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.2.22
md5sum = f390235995b72b4c50948a43eb7e41fe
version = 10.2.23
md5sum = 941c9ac6ee7709fd88a4098ecfc0a4b0
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
......@@ -48,10 +50,12 @@ configure-options =
-DWITH_EMBEDDED_SERVER=0
-DWITH_JEMALLOC=yes
-DWITH_INNODB_BZIP2=ON
-DWITH_INNODB_LZ4=OFF
-DWITH_INNODB_LZ4=ON
-DWITH_INNODB_LZMA=ON
-DWITH_INNODB_LZO=OFF
-DWITH_INNODB_SNAPPY=OFF
-DWITH_INNODB_SNAPPY=ON
-DWITH_ROCKSDB_LZ4=ON
-DWITH_ROCKSDB_snappy=ON
-DWITH_ROCKSDB_ZSTD=ON
-DWITH_SAFEMALLOC=OFF
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
......@@ -63,13 +67,13 @@ configure-options =
-DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${zstd:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${lz4:location}/include -I${snappy:location}/include -I${zstd:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${lz4:location}/lib:${snappy:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${zstd:location}/include
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
......
[buildout]
extends =
../cmake/buildout.cfg
parts =
snappy
[snappy]
recipe = slapos.recipe.cmmi
url = https://github.com/google/snappy/archive/1.1.7.tar.gz
md5sum = ee9086291c9ae8deb4dac5e0b85bf54a
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=${:location}
-DBUILD_SHARED_LIBS=ON
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
......@@ -90,6 +90,8 @@ About SSL and SlapOS Master Zero Knowledge
SSL keys and certificates are directly send to the frontend cluster in order to follow zero knowledge principle of SlapOS Master.
*Note*: Until master partition or slave specific certificate is uploaded each slave is served with fallback certificate. This fallback certificate is self signed, does not match served hostname and results with lack of response on HTTPs.
Master partition
----------------
......@@ -218,14 +220,10 @@ caddy_custom_https
~~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the https port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above.
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
caddy_custom_http
~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the http port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
url
~~~
Necessary to activate cache. ``url`` of backend to use.
......@@ -343,7 +341,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be::
"caddy_custom_https":'
https://www.example.com:%(https_port)s, https://example.com:%(https_port)s {
bind %(local_ipv4)s
tls %%(certificate)s %%(certificate)s
tls %(certificate)s %(certificate)s
log / %(access_log)s {combined}
errors %(error_log)s
......
Generally things to be done with ``caddy-frontend``:
* tests: add assertion with results of promises in etc/promise for each partition
* README: cleanup the documentation, explain various specifics
* check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
* (new) ``type:websocket`` slave
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 111ff0794c90657b658e3d50525e7fed
md5sum = fd2ff61d9270109115ced8f56fb0be17
[template-common]
filename = instance-common.cfg.in
......@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = abbbc8f24cdef389b9b2859b0ef8dd0e
md5sum = ab5312fb5454d5358b22b000cf6ed124
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 81ad603fe0a1e29948bd81b457e8d7a4
md5sum = 37edefdb9963daa67b01e5d55d97c17d
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = dfbe4378610aa42f2cbc2a55d386324e
md5sum = f9efdfe7a7e3a78f0b15f414b5469316
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -42,23 +42,19 @@ md5sum = 38e9994be01ea1b8a379f8ff7aa05438
[template-caddy-frontend-configuration]
filename = templates/Caddyfile.in
md5sum = df8c08c9aecb48fdbcdfca40f9cf74a4
md5sum = dfec964a9f194293567b09d0f10e4b3d
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
md5sum = 0979a03476e86bf038516c9565dadc17
[caddy-custom-http-validator]
filename = templates/caddy-custom-http-validator.in
md5sum = a264208e960cdcd25ef27ed8cf730240
[template-not-found-html]
filename = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = 4308b63820d3682511ce54040d1ae60e
md5sum = b882c408202cd2dd13f619210321a528
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
......@@ -66,7 +62,7 @@ md5sum = 907372828d1ceb05c41240078196f439
[template-log-access]
filename = templates/template-log-access.conf.in
md5sum = 704f37bfdd52fe628ae81d41abba8d7a
md5sum = f8068179333ce19e95df561c70073857
[template-empty]
filename = templates/empty.in
......@@ -90,7 +86,7 @@ md5sum = cd6bb9bd0734f17469b0ca88f8b1a531
[template-nginx-configuration]
filename = templates/nginx.cfg.in
md5sum = 30f30ef3539fe6b7ab99162ae8e71a87
md5sum = d4c6c585c8a7da12c16b4b8e5a1cd90a
[template-nginx-eventsource-slave-virtualhost]
filename = templates/nginx-eventsource-slave.conf.in
......@@ -98,11 +94,11 @@ md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-nginx-notebook-slave-virtualhost]
filename = templates/nginx-notebook-slave.conf.in
md5sum = ac17212a53be2c08ab84682ec665148d
md5sum = 982489258b9c2cafc9b52a94e7a8660f
[template-apache-lazy-script-call]
[template-caddy-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in
md5sum = ebe5d3d19923eb812a40019cb11276d8
md5sum = b9f73f6323f9fceea054c46c854d2862
[template-graceful-script]
filename = templates/graceful-script.sh.in
......@@ -122,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 5597b2184b445af69ad6d517d0729ad6
md5sum = cc6f32656e76f4b79b5e47567b930f74
......@@ -35,7 +35,7 @@ parts +=
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = 67bd60ea1bfb4fc6aafdfe4fa204f725731f20cf
revision = 73a14b0e88afe7512f2fefe6ee9e0000fa523d5d
[kedifa-develop]
recipe = zc.recipe.egg:develop
......@@ -111,7 +111,7 @@ openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
trafficserver = ${trafficserver7:location}
sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-getter = ${:bin_directory}/kedifa-getter
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
monitor_template = ${monitor-template:output}
......@@ -152,7 +152,6 @@ context =
key template_kedifa template-kedifa:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
key caddy_custom_http_validator caddy-custom-http-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
key caucase_jinja2_library caucase-jinja2-library:target
......@@ -169,13 +168,6 @@ filename = caddy-backend-url-validator.in
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[caddy-custom-http-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/${:filename}
filename = caddy-custom-http-validator.in
output = ${buildout:directory}/caddy-custom-http-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
......
......@@ -75,7 +75,7 @@ context =
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% set unauthorized_message = 'slave not authorized' %}
{% for slave in slave_instance_list %}
{% for slave in sorted(slave_instance_list) %}
{% set slave_error_list = [] %}
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
......@@ -114,8 +114,6 @@ context =
{% if not unauthorized_message in slave_error_list %}
{% do slave_error_list.append(unauthorized_message) %}
{% endif %}
{% elif subprocess_module.call([caddy_custom_http_validator, '' ~ slave[key]]) == 1 %}
{% do slave_error_list.append('slave %s configuration invalid' % (key,)) %}
{% endif %}
{% endif %}
{% endfor %} {# for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] #}
......
......@@ -159,7 +159,7 @@ command-line = {{ parameter_dict['caddy'] }}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
-root ${directory:csr_id}
wrapper-path = ${directory:service}/expose-csr_id
......
......@@ -139,13 +139,6 @@
"title": "Verify Backend Certificates",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file",
"textarea": true,
"title": "SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_proxy_ca_crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
......
......@@ -97,27 +97,6 @@
"title": "Prefer gzip Encoding for Backend",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file",
"textarea": true,
"title": "SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_crt": {
"default": "",
"description": "Content of the SSL Certificate file",
"textarea": true,
"title": "SSL Certificate",
"type": "string"
},
"ssl_key": {
"default": "",
"description": "Content of the SSL Key file",
"textarea": true,
"title": "SSL Key",
"type": "string"
}
},
"title": "Input Parameters",
"type": "object"
......
......@@ -59,7 +59,6 @@ extra-context =
import validators validators
key cluster_identification instance-parameter:root-instance-title
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw caddy_custom_http_validator {{ caddy_custom_http_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
......
......@@ -2,6 +2,8 @@
extends = common.cfg
[versions]
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
# Versions pinned for kedifa need urllib3 >= 1.18
urllib3 = 1.24
requests = 2.20.0
......
......@@ -4,45 +4,38 @@ import {{frontend_configuration.get('log-access-configuration')}}
import {{ slave_configuration_directory }}/*.conf
import {{ slave_with_cache_configuration_directory }}/*.conf
{%- set ssl = {} -%}
{%- if os_module.path.exists(master_certificate) -%}
{%- do ssl.__setitem__('certificate', master_certificate) -%}
{%- do ssl.__setitem__('key', master_certificate) -%}
{#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
{%- elif os_module.path.getsize(apache_certificate) > 0 and os_module.path.getsize(apache_key) > 0 -%}
{%- do ssl.__setitem__('certificate', apache_certificate) -%}
{%- do ssl.__setitem__('key', apache_key) -%}
{%- endif -%}
{#- BBB: SlapOS Master non-zero knowledge END #}
# Catch-all and 404 for not configured instances
{% if 'key' in ssl %}
:{{ https_port }} {
tls {{ ssl['certificate'] }} {{ ssl['key'] }}
tls {{ master_certificate }} {{ master_certificate }}
bind {{ local_ipv4 }}
# Compress the output
gzip
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
{% endif %}
:{{ http_port }} {
bind {{ local_ipv4 }}
# Compress the output
gzip
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
# Access to server-status Caddy-style
https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-key'] }}
tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
# Compress the output
gzip
bind {{ local_ipv4 }}
......@@ -52,8 +45,11 @@ https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv
}
expvar
pprof
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
{% if software_type == slap_software_type %}
{% set kedifa_updater_mapping = [] %}
{% set cached_server_dict = {} %}
{% set part_list = [] %}
{% set cache_port = caddy_configuration.get('cache-port') %}
......@@ -20,7 +21,6 @@ recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
import os_module os
raw common_profile {{ common_profile }}
${:extra-context}
......@@ -32,12 +32,7 @@ notifempty = true
create = true
{% if master_key_download_url %}
{% do part_list.append('master-key-download') %}
[master-key-download]
recipe = plone.recipe.command
destination = {{ master_certificate }}
command = {{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ master_key_download_url }}
update-command = ${:command}
{% do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
{% endif %}
{% if slave_kedifa_information %}
......@@ -174,34 +169,11 @@ bytes = 8
{# ################################################## #}
{# Set Slave Certificates if needed #}
{% set cert_dirname = slave_reference.replace('-','.') %}
{% set autocert_dir = '/'.join([autocert, cert_dirname]) %}
[{{ slave_reference }}-path]
recipe = slapos.cookbook:mkdirectory
cert = {{ autocert_dir }}
{# Set certificate key for custom configuration #}
{% set certificate = '%s/certificate.pem' % (autocert_dir, ) %}
{# Set certificate key for custom configuration #}
{% set cert_name = slave_reference.replace('-','.') + '.pem' %}
{% set certificate = '%s/%s' % (autocert, cert_name) %}
{% do slave_parameter_dict.__setitem__('certificate', certificate )%}
[{{ slave_reference }}-key-download]
recipe = plone.recipe.command
destination = {{ '${' + slave_reference + '-path:cert}/downloaded.pem' }}
used = {{ '${' + slave_reference + '-path:cert}/certificate.pem' }}
source-master = ${master-key-download:destination}
command =
{{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ key_download_url }}
if [ -f ${:destination} ] ; then
# if the slave specific certificate is available, use it
ln -sf ${:destination} ${:used}
elif [ -f ${:source-master} ] ; then
# if the master provided certificate is available, use it
ln -sf ${:source-master} ${:used}
else
rm -f ${:used}
fi
update-command = ${:command}
# BBB: SlapOS Master non-zero knowledge BEGIN
{# Set ssl certificates for each slave #}
{% for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
{% if cert_name in slave_instance %}
......@@ -217,6 +189,7 @@ template = {{ empty_template }}
rendered = {{ cert_file }}
extra-context =
key content {{ cert_title + '-config:value' }}
# BBB: SlapOS Master non-zero knowledge BEGIN
# Store certificate in config
[{{ cert_title + '-config' }}]
value = {{ dumps(slave_instance.get(cert_name)) }}
......@@ -224,42 +197,29 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
{% endfor %}
{#- Set Up Certs #}
{% do slave_instance.__setitem__('apache_certificate', apache_certificate) %}
{% do slave_instance.__setitem__('apache_key', apache_key) %}
{% if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
{% set cert_title = '%s-crt' % (slave_reference) %}
{% set key_title = '%s-key' % (slave_reference) %}
{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
{% set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) %}
{% set cert_file = '/'.join([bbb_ssl_directory, cert_title.replace('-','.')]) %}
{% do kedifa_updater_mapping.append((key_download_url, certificate, cert_file)) %}
{% do part_list.append(cert_title) %}
{% do part_list.append(key_title) %}
{% do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %}
{% do slave_parameter_dict.__setitem__("ssl_key", key_file) %}
{% do slave_instance.__setitem__('path_to_ssl_crt', cert_file) %}
{% do slave_instance.__setitem__('path_to_ssl_key', key_file) %}
[{{key_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ key_file }}
key-content = {{ dumps(slave_instance.get('ssl_key')) }}
extra-context =
key content :key-content
[{{cert_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '')) }}
cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '') + '\n' + slave_instance.get('ssl_key')) }}
extra-context =
key content :cert-content
{% endif %}
{% else %}
{% do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %}
{% endif %}
# BBB: SlapOS Master non-zero knowledge END
{# ########################################## #}
{# Set Slave Configuration #}
[{{ slave_configuration_section_name }}]
certificate = {{ '${' + slave_reference + '-key-download:used}' }}
certificate = {{ certificate }}
https_port = {{ dumps('' ~ https_port) }}
http_port = {{ dumps('' ~ http_port) }}
local_ipv4 = {{ dumps('' ~ local_ipv4) }}
......@@ -466,7 +426,6 @@ global_ipv6 = {{ dumps(global_ipv6) }}
https_port = {{ dumps(https_port) }}
http_port = {{ dumps(http_port) }}
ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }}
ip_access_key = {{ frontend_configuration.get('ip-access-key') }}
access_log = {{ dumps(access_log) }}
error_log = {{ dumps(error_log) }}
not_found_file = {{ dumps(not_found_file) }}
......@@ -493,12 +452,44 @@ monitor-base-url = {{ monitor_base_url }}
csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
csr_id-certificate = ${get-csr_id-certificate:certificate}
[kedifa-updater]
recipe = slapos.cookbook:wrapper
command-line = {{ kedifa_updater }}
--server-ca-certificate {{ kedifa_caucase_ca_certificate }}
--identity {{ kedifa_login_certificate }}
--master-certificate {{ master_certificate }}
--on-update "{{ frontend_graceful_reload }} ; {{ nginx_graceful_reload }}"
${kedifa-updater-mapping:file}
{{ kedifa_updater_state_file }}
wrapper-path = {{ service_directory }}/kedifa-updater
hash-files = ${buildout:directory}/software_release/buildout.cfg
[kedifa-updater-mapping]
recipe = slapos.recipe.template:jinja2
file = {{ kedifa_updater_mapping_file }}
template = inline:
{% for mapping in kedifa_updater_mapping %}
{{ mapping[0] }} {{ mapping[1] }} {{ mapping[2] }}
{% endfor %}
rendered = ${:file}
[caddy-log-access-header]
# Caddy refuse to start if an `import`ed file is empty, so we prepend a header
# so that the file is never empty.
< = jinja2-template-base
template = inline: # This file contain directives to serve directories with log files
rendered = {{frontend_configuration.get('log-access-configuration')}}
[buildout]
extends =
{{ common_profile }}
{{ logrotate_base_instance }}
parts +=
kedifa-updater
caddy-log-access-header
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
......@@ -570,7 +561,7 @@ command-line = {{ caddy_executable }}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
-root {{ directory_csr_id }}
wrapper-path = {{ service_directory }}/expose-csr_id
......
......@@ -9,6 +9,7 @@ if [ ! -f $PIDFILE ]; then
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
rm -f $PIDFILE
else
ps --pid `cat $PIDFILE` &>/dev/null
if [ $? -eq 0 ]; then
......@@ -17,6 +18,7 @@ else
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
rm -f $PIDFILE
fi
fi
#!${dash:location}/bin/dash
config="$1"
echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1
......@@ -26,25 +26,11 @@
{%- set default_path = slave_parameter.get('default-path', '').strip('/') | urlencode %}
# SSL enabled hosts
{% set ssl = {} %}
{% if os_module.path.exists(slave_parameter['certificate']) %}
{% do ssl.__setitem__('certificate', slave_parameter['certificate']) %}
{% do ssl.__setitem__('key', slave_parameter['certificate']) %}
{#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
{% elif 'path_to_ssl_crt' in slave_parameter and 'path_to_ssl_key' in slave_parameter %}
{% do ssl.__setitem__('certificate', slave_parameter['path_to_ssl_crt']) %}
{% do ssl.__setitem__('key', slave_parameter['path_to_ssl_key']) %}
{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 and os_module.path.getsize(slave_parameter['apache_key']) > 0 %}
{% do ssl.__setitem__('certificate', slave_parameter['apache_certificate']) %}
{% do ssl.__setitem__('key', slave_parameter['apache_key']) %}
{% endif %}
{#- BBB: SlapOS Master non-zero knowledge END -#}
{% if 'key' in ssl %}
{{ https_host_list|join(', ') }} {
bind {{ slave_parameter['local_ipv4'] }}
# Compress the output
gzip
tls {{ ssl['certificate'] }} {{ ssl['key'] }} {
tls {{ slave_parameter['certificate'] }} {{ slave_parameter['certificate'] }} {
{%- if enable_h2 %}
# Allow HTTP2
alpn h2 http/1.1
......@@ -53,8 +39,13 @@
alpn http/1.1
{%- endif %} {#- if enable_h2 #}
} {# tls #}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
errors {{ slave_parameter.get('error_log') }}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log') }} {
rotate_size 0
}
{%- if not (slave_type == 'zope' and backend_url) %}
{% if prefer_gzip %}
......@@ -189,7 +180,6 @@
{%- endif %} {#- if backend_url #}
{%- endif %} {#- if slave_type == 'zope' and backend_url #}
} {# https_host_list|join(', ') #}
{% endif %}
# SSL-disabled hosts
{{ http_host_list|join(', ') }} {
......@@ -197,8 +187,12 @@
# Compress the output
gzip
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
errors {{ slave_parameter.get('error_log') }}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log') }} {
rotate_size 0
}
{%- if not (slave_type == 'zope' and backend_url) %}
{%- if prefer_gzip %}
......
......@@ -5,28 +5,18 @@
{%- set https_upstream = https_url.split("/")[2] %}
# SSL-enabled
{% set ssl = {} %}
{% if os_module.path.exists(slave_parameter['certificate']) %}
{% do ssl.__setitem__('certificate', slave_parameter['certificate']) %}
{% do ssl.__setitem__('key', slave_parameter['certificate']) %}
{#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
{% elif 'path_to_ssl_crt' in slave_parameter and 'path_to_ssl_key' in slave_parameter %}
{% do ssl.__setitem__('certificate', slave_parameter['path_to_ssl_crt']) %}
{% do ssl.__setitem__('key', slave_parameter['path_to_ssl_key']) %}
{% elif os_module.path.getsize(slave_parameter['apache_certificate']) > 0 and os_module.path.getsize(slave_parameter['apache_key']) > 0 %}
{% do ssl.__setitem__('certificate', slave_parameter['apache_certificate']) %}
{% do ssl.__setitem__('key', slave_parameter['apache_key']) %}
{% endif %}
{#- BBB: SlapOS Master non-zero knowledge END -#}
{% if 'key' in ssl %}
https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_https_port'] }} {
bind {{ slave_parameter['local_ipv4'] }}
# Compress the output
gzip
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
errors {{ slave_parameter.get('error_log') }}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log') }} {
rotate_size 0
}
tls {{ ssl['certificate'] }} {{ ssl['key'] }} {
tls {{ slave_parameter['certificate'] }} {{ slave_parameter['certificate'] }} {
alpn http/1.1
}
......@@ -50,15 +40,18 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
insecure_skip_verify
}
}
{% endif %}
# SSL-disabled
http://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_http_port'] }} {
bind {{ slave_parameter['local_ipv4'] }}
# Compress the output
gzip
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
errors {{ slave_parameter.get('error_log') }}
log / {{ slave_parameter.get('access_log') }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ slave_parameter.get('error_log') }} {
rotate_size 0
}
proxy / {{ upstream }} {
try_duration {{ slave_parameter['proxy_try_duration'] }}s
......
......@@ -58,38 +58,31 @@
import {{ slave_configuration_directory }}/*.conf
# Catch-all and 404 for not configured instances
{%- set ssl = {} -%}
{%- if os_module.path.exists(master_certificate) -%}
{%- do ssl.__setitem__('certificate', master_certificate) -%}
{%- do ssl.__setitem__('key', master_certificate) -%}
{#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
{%- elif os_module.path.getsize(apache_certificate) > 0 and os_module.path.getsize(apache_key) > 0 -%}
{%- do ssl.__setitem__('certificate', apache_certificate) -%}
{%- do ssl.__setitem__('key', apache_key) -%}
{%- endif -%}
{#- BBB: SlapOS Master non-zero knowledge END -#}
# Catch-all and 404 for not configured instances
{% if 'key' in ssl %}
:{{ port }} {
tls {{ ssl['certificate'] }} {{ ssl['key'] }}
tls {{ master_certificate }} {{ master_certificate }}
bind {{ local_ip }}
# Serve an error 204 (No Content) for favicon.ico
status 204 /favicon.ico
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
{% endif %}
:{{ plain_port }} {
bind {{ local_ip }}
# Serve an error 204 (No Content) for favicon.ico
status 204 /favicon.ico
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
......@@ -4,13 +4,16 @@ https://[{{ parameter_dict['global_ipv6'] }}]:{{ parameter_dict['https_port'] }}
bind {{ parameter_dict['local_ipv4'] }}
root {{ directory }}/
browse
tls {{ parameter_dict['ip_access_certificate'] }} {{ parameter_dict['ip_access_key'] }}
tls {{ parameter_dict['ip_access_certificate'] }} {{ parameter_dict['ip_access_certificate'] }}
basicauth "{{ slave }}" {{ slave_password[slave] | trim }} {
"Log Access {{ slave }}"
/
}
log / {{ parameter_dict['access_log'] }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ parameter_dict['access_log'] }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ parameter_dict['error_log'] }} {
rotate_size 0
* {{ parameter_dict['not_found_file'] }}
}
}
......
......@@ -12,14 +12,13 @@ T-2/etc/plugin/caddy_ssl_cached.py: OK
T-2/etc/plugin/check-_test-error-log-last-day.py: OK
T-2/etc/plugin/check-_test-error-log-last-hour.py: OK
T-2/etc/plugin/check-free-disk-space.py: OK
T-2/etc/plugin/frontend-caddy-configuration-promise.py: OK
T-2/etc/plugin/frontend-caddy-configuration-promise.py: ERROR
T-2/etc/plugin/monitor-bootstrap-status.py: OK
T-2/etc/plugin/monitor-httpd-listening-on-tcp.py: OK
T-2/etc/plugin/nginx-configuration-promise.py: OK
T-2/etc/plugin/nginx-