Commit 7fd3f5b5 by Rafael Monnerat

Update Release Candidate

2 parents 45299ea7 b71ce50e
Showing 71 changed files with 366 additions and 445 deletions
......@@ -15,3 +15,4 @@ slapos.cookbook.egg-info
*.egg/
TEST_KNOWN_HOSTS
node_modules
update-release.sh
[buildout]
extends =
../../component/golang/buildout.cfg
gowork.cfg
parts =
gowork
caddy
[gowork]
# Caddy 1.x+ uses go modules, for which gowork does not work yet
golang = ${golang1.12:location}
install =
github.com/mholt/caddy
[gowork.goinstall]
command = :
depends =
${caddy:recipe}
[caddy]
recipe = slapos.recipe.cmmi
path = ${go_github.com_mholt_caddy:location}
go = ${gowork:golang}/bin/go
configure-command = :
make-targets =
make-binary = cd ${:path}/caddy && ${:go} install -v
environment =
PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gowork:directory}
# revision and repository can be used to control which caddy version is used
revision = db2741c6e0a1c06340391c5b9fa282b876a33361
repository = github.com/mholt/caddy/caddy
recipe = plone.recipe.command
update-command = ${:command}
stop-on-error = True
# GO111MODULE=on enables go modules support
# the chmod is needed as modules are fetched with u-w
command =
. ${gowork:env.sh} &&
cd ${gowork:directory} &&
export GO111MODULE=on &&
go get ${:repository}@${:revision} &&
chmod -R u+w .
output = ${gowork:bin}/caddy
location = ${:output}
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_mholt_caddy:recipe}
[go_github.com_mholt_caddy]
<= go-git-package
go.importpath = github.com/mholt/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v0.11.1-5-gdd393ce3a67e6a773be87185528a00f2e0a9eb26
[buildout]
parts =
lz4
[lz4]
recipe = slapos.recipe.cmmi
url = https://github.com/lz4/lz4/archive/v1.8.3.tar.gz
md5sum = d5ce78f7b1b76002bbfffa6f78a5fc4e
configure-command = true
......@@ -9,11 +9,13 @@ extends =
../jemalloc/buildout.cfg
../libaio/buildout.cfg
../libxml2/buildout.cfg
../lz4/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pkgconfig/buildout.cfg
../readline/buildout.cfg
../snappy/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
../unixodbc/buildout.cfg
......@@ -26,8 +28,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.2.22
md5sum = f390235995b72b4c50948a43eb7e41fe
version = 10.2.23
md5sum = 941c9ac6ee7709fd88a4098ecfc0a4b0
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.2.16_create_system_tables__no_test.patch#3fd5f9febabdb42d4b6653969a0194f9
......@@ -48,10 +50,12 @@ configure-options =
-DWITH_EMBEDDED_SERVER=0
-DWITH_JEMALLOC=yes
-DWITH_INNODB_BZIP2=ON
-DWITH_INNODB_LZ4=OFF
-DWITH_INNODB_LZ4=ON
-DWITH_INNODB_LZMA=ON
-DWITH_INNODB_LZO=OFF
-DWITH_INNODB_SNAPPY=OFF
-DWITH_INNODB_SNAPPY=ON
-DWITH_ROCKSDB_LZ4=ON
-DWITH_ROCKSDB_snappy=ON
-DWITH_ROCKSDB_ZSTD=ON
-DWITH_SAFEMALLOC=OFF
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
......@@ -63,13 +67,13 @@ configure-options =
-DCMAKE_LIBRARY_PATH=${unixodbc:location}/lib
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${zstd:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include -I${unixodbc:location}/include -I${lz4:location}/include -I${snappy:location}/include -I${zstd:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib:${unixodbc:location}/lib:${lz4:location}/lib:${snappy:location}/lib:${zstd:location}/lib:${gcc:location}/lib:${gcc:location}/lib64
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${zstd:location}/include
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include:${unixodbc:location}/include:${lz4:location}/include:${snappy:location}/include:${zstd:location}/include
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib -L${unixodbc:location}/lib -L${lz4:location}/lib -L${snappy:location}/lib -L${zstd:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
......
[buildout]
extends =
../cmake/buildout.cfg
parts =
snappy
[snappy]
recipe = slapos.recipe.cmmi
url = https://github.com/google/snappy/archive/1.1.7.tar.gz
md5sum = ee9086291c9ae8deb4dac5e0b85bf54a
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=${:location}
-DBUILD_SHARED_LIBS=ON
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
......@@ -90,6 +90,8 @@ About SSL and SlapOS Master Zero Knowledge
SSL keys and certificates are directly send to the frontend cluster in order to follow zero knowledge principle of SlapOS Master.
*Note*: Until master partition or slave specific certificate is uploaded each slave is served with fallback certificate. This fallback certificate is self signed, does not match served hostname and results with lack of response on HTTPs.
Master partition
----------------
......@@ -218,14 +220,10 @@ caddy_custom_https
~~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the https port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above.
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
caddy_custom_http
~~~~~~~~~~~~~~~~~
Raw Caddy configuration in python template format (i.e. write "%%" for one "%") for the slave listening to the http port. Its content will be templatified in order to access functionalities such as cache access, ssl certificates... The list is available above
*Note*: The system will reject slaves which does not pass validation of caddy configuration, despite them being in ``-frontend-authorized-slave-string``, as otherwise this will lead to the whole frontend to fail.
url
~~~
Necessary to activate cache. ``url`` of backend to use.
......@@ -343,7 +341,7 @@ Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be::
"caddy_custom_https":'
https://www.example.com:%(https_port)s, https://example.com:%(https_port)s {
bind %(local_ipv4)s
tls %%(certificate)s %%(certificate)s
tls %(certificate)s %(certificate)s
log / %(access_log)s {combined}
errors %(error_log)s
......
Generally things to be done with ``caddy-frontend``:
* tests: add assertion with results of promises in etc/promise for each partition
* README: cleanup the documentation, explain various specifics
* check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
* (new) ``type:websocket`` slave
......
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 111ff0794c90657b658e3d50525e7fed
md5sum = fd2ff61d9270109115ced8f56fb0be17
[template-common]
filename = instance-common.cfg.in
......@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = abbbc8f24cdef389b9b2859b0ef8dd0e
md5sum = ab5312fb5454d5358b22b000cf6ed124
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 81ad603fe0a1e29948bd81b457e8d7a4
md5sum = 37edefdb9963daa67b01e5d55d97c17d
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = dfbe4378610aa42f2cbc2a55d386324e
md5sum = f9efdfe7a7e3a78f0b15f414b5469316
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -42,23 +42,19 @@ md5sum = 38e9994be01ea1b8a379f8ff7aa05438
[template-caddy-frontend-configuration]
filename = templates/Caddyfile.in
md5sum = df8c08c9aecb48fdbcdfca40f9cf74a4
md5sum = dfec964a9f194293567b09d0f10e4b3d
[caddy-backend-url-validator]
filename = templates/caddy-backend-url-validator.in
md5sum = 0979a03476e86bf038516c9565dadc17
[caddy-custom-http-validator]
filename = templates/caddy-custom-http-validator.in
md5sum = a264208e960cdcd25ef27ed8cf730240
[template-not-found-html]
filename = templates/notfound.html
md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = 4308b63820d3682511ce54040d1ae60e
md5sum = b882c408202cd2dd13f619210321a528
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
......@@ -66,7 +62,7 @@ md5sum = 907372828d1ceb05c41240078196f439
[template-log-access]
filename = templates/template-log-access.conf.in
md5sum = 704f37bfdd52fe628ae81d41abba8d7a
md5sum = f8068179333ce19e95df561c70073857
[template-empty]
filename = templates/empty.in
......@@ -90,7 +86,7 @@ md5sum = cd6bb9bd0734f17469b0ca88f8b1a531
[template-nginx-configuration]
filename = templates/nginx.cfg.in
md5sum = 30f30ef3539fe6b7ab99162ae8e71a87
md5sum = d4c6c585c8a7da12c16b4b8e5a1cd90a
[template-nginx-eventsource-slave-virtualhost]
filename = templates/nginx-eventsource-slave.conf.in
......@@ -98,11 +94,11 @@ md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-nginx-notebook-slave-virtualhost]
filename = templates/nginx-notebook-slave.conf.in
md5sum = ac17212a53be2c08ab84682ec665148d
md5sum = 982489258b9c2cafc9b52a94e7a8660f
[template-apache-lazy-script-call]
[template-caddy-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in
md5sum = ebe5d3d19923eb812a40019cb11276d8
md5sum = b9f73f6323f9fceea054c46c854d2862
[template-graceful-script]
filename = templates/graceful-script.sh.in
......@@ -122,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 5597b2184b445af69ad6d517d0729ad6
md5sum = cc6f32656e76f4b79b5e47567b930f74
......@@ -35,7 +35,7 @@ parts +=
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = 67bd60ea1bfb4fc6aafdfe4fa204f725731f20cf
revision = 73a14b0e88afe7512f2fefe6ee9e0000fa523d5d
[kedifa-develop]
recipe = zc.recipe.egg:develop
......@@ -111,7 +111,7 @@ openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
trafficserver = ${trafficserver7:location}
sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-getter = ${:bin_directory}/kedifa-getter
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
monitor_template = ${monitor-template:output}
......@@ -152,7 +152,6 @@ context =
key template_kedifa template-kedifa:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
key caddy_custom_http_validator caddy-custom-http-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
key caucase_jinja2_library caucase-jinja2-library:target
......@@ -169,13 +168,6 @@ filename = caddy-backend-url-validator.in
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[caddy-custom-http-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/${:filename}
filename = caddy-custom-http-validator.in
output = ${buildout:directory}/caddy-custom-http-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
......
......@@ -100,7 +100,6 @@ single-custom-personal = ${dynamic-custom-personal-template-slave-list:rendered}
template-log-access = {{ parameter_dict['template_log_access'] }}
log-access-configuration = ${directory:etc}/log-access.conf
ip-access-certificate = ${self-signed-ip-access:certificate}
ip-access-key = ${self-signed-ip-access:key}
caddy-directory = {{ parameter_dict['caddy_location'] }}
caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
caddy-https-port = ${configuration:port}
......@@ -111,17 +110,16 @@ recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
key = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.key
certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
stop-on-error = True
command =
[ -f ${:key} ] && [ -f ${:certificate} ] && exit 0
rm -f ${:key} ${:certificate}
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:key} \
-keyout ${:certificate} \
-subj "/CN=Self Signed IP Access" \
-reqexts SAN \
-extensions SAN \
......@@ -129,6 +127,25 @@ command =
<(printf "\n[SAN]\nsubjectAltName=IP:${:ipv6},IP:${:ipv4}")) \
-out ${:certificate}'
[self-signed-fallback-access]
# Self Signed certificate for HTTPS access to the frontend with fallback certificate
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
stop-on-error = True
command =
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:certificate} \
-subj "/CN=Fallback certificate/OU={{ instance_parameter['configuration.frontend-name'] }}" \
-out ${:certificate}'
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
......@@ -138,7 +155,6 @@ slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
context =
import json_module json
import os_module os
raw common_profile {{ parameter_dict['common_profile'] }}
raw logrotate_base_instance {{ parameter_dict['logrotate_base_instance'] }}
key slap_software_type :slap_software_type
......@@ -212,7 +228,9 @@ bin_directory = {{ parameter_dict['bin_directory'] }}
caddy_executable = {{ parameter_dict['caddy'] }}
caucase_url = {{ slapparameter_dict['kedifa-caucase-url'] }}
sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
kedifa-getter = {{ parameter_dict['kedifa-getter'] }}
kedifa-updater = {{ parameter_dict['kedifa-updater'] }}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
kedifa-csr = {{ parameter_dict['kedifa-csr'] }}
service_directory = ${directory:service}
extra-context =
......@@ -222,7 +240,9 @@ extra-context =
key nginx_configuration_directory caddy-directory:nginx-slave-configuration
key caddy_cached_configuration_directory caddy-directory:slave-with-cache-configuration
key slave_with_cache_configuration_directory caddy-directory:slave-with-cache-configuration
key kedifa_getter :kedifa-getter
key kedifa_updater :kedifa-updater
key kedifa_updater_mapping_file :kedifa-updater-mapping-file
key kedifa_updater_state_file :kedifa-updater-state-file
key kedifa_csr :kedifa-csr
key caddy_executable :caddy_executable
key caucase_url :caucase_url
......@@ -259,6 +279,8 @@ extra-context =
key template_notebook_slave_configuration software-release-path:template-nginx-notebook-slave-virtualhost
key software_type :software_type
key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
key frontend_graceful_reload caddy-configuration:frontend-graceful-command
key nginx_graceful_reload nginx-configuration:nginx-graceful-command
section frontend_configuration frontend-configuration
section caddy_configuration caddy-configuration
section nginx_configuration nginx-configuration
......@@ -277,10 +299,10 @@ extra-context =
key service_directory directory:service
key run_directory directory:etc-run
key not_found_file caddy-configuration:not-found-file
# BBB: SlapOS Master non-zero knowledge BEGIN
key custom_ssl_directory caddy-directory:custom-ssl-directory
# BBB: SlapOS Master non-zero knowledge BEGIN
key bbb_ssl_directory directory:bbb-ssl-dir
key apache_certificate apache-certificate:rendered
key apache_key apache-key:rendered
# BBB: SlapOS Master non-zero knowledge END
[dynamic-virtualhost-template-slave]
......@@ -323,7 +345,6 @@ extra-context =
key password monitor-htpasswd:passwd
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
key apache_key apache-key:rendered
# BBB: SlapOS Master non-zero knowledge END
[caddy-wrapper]
......@@ -332,7 +353,8 @@ environment =
CADDYPATH=${directory:frontend_cluster}
command-line = {{ parameter_dict['caddy'] }}
-conf ${dynamic-caddy-frontend-template:rendered}
-log stdout
-log ${caddy-configuration:error-log}
-log-roll-mb 0
{% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
-http2=false
{% else %}
......@@ -343,7 +365,7 @@ command-line = {{ parameter_dict['caddy'] }}
{% endif %}
-grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
wrapper-path = ${directory:bin}/caddy-wrapper
[caddy-frontend]
......@@ -369,9 +391,7 @@ slave-log = ${directory:log}/httpd
nginx-slave-configuration = ${directory:etc}/nginx-slave-conf.d/
autocert = ${directory:srv}/autocert
master-autocert-dir = ${:autocert}/master-autocert
# BBB: SlapOS Master non-zero knowledge BEGIN
custom-ssl-directory = ${:slave-configuration}/ssl
# BBB: SlapOS Master non-zero knowledge END
[caddy-configuration]
frontend-configuration = ${directory:etc}/Caddyfile
......@@ -393,21 +413,23 @@ command-line = ${frontend-caddy-validate:rendered}
wrapper-path = ${directory:bin}/caddy-configtest
# BBB: SlapOS Master non-zero knowledge BEGIN
[apache-key]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
rendered = ${directory:bbb-ssl-dir}/frontend.key
content = ${configuration:apache-key}
extra-context =
key content :content
[get-self-signed-fallback-access]
recipe = collective.recipe.shelloutput
commands =
certificate = cat ${self-signed-fallback-access:certificate}
[apache-certificate]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
recipe = slapos.recipe.template:jinja2
template = inline:
{% raw %}
{{ certificate or fallback_certificate }}
{{ key or '' }}
{% endraw %}
context =
key certificate configuration:apache-certificate
key key configuration:apache-key
key fallback_certificate get-self-signed-fallback-access:certificate
rendered = ${directory:bbb-ssl-dir}/frontend.crt
content = ${configuration:apache-certificate}
extra-context =
key content :content
# BBB: SlapOS Master non-zero knowledge END
[logrotate-entry-caddy]
......@@ -415,6 +437,11 @@ extra-context =
name = caddy
log = ${caddy-configuration:error-log} ${caddy-configuration:access-log}
rotate-num = 30
# Note: Slaves do not define their own reload, as this would be repeated,
# because sharedscripts work per entry, and each slave needs its own
# olddir
# Here we trust that there will be something to be rotated with error
# or access log, and that this will trigger postrotate script.
post = ${frontend-caddy-lazy-graceful:rendered} &
[logrotate-entry-nginx]
......@@ -557,7 +584,7 @@ template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
mode = 0700
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*/*.pem ${caddy-directory:custom-ssl-directory}/*.key ${caddy-directory:custom-ssl-directory}/*.crt ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.key ${directory:bbb-ssl-dir}/*.crt
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:slave-with-cache-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/caddy_graceful_signature
extra-context =
......@@ -571,7 +598,7 @@ extra-context =
template = {{ parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-nginx-safe-graceful
mode = 0700
path_list = ${dynamic-nginx-frontend-template:rendered} ${caddy-directory:nginx-slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*/*.pem ${caddy-directory:custom-ssl-directory}/*.key ${caddy-directory:custom-ssl-directory}/*.crt ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.key ${directory:bbb-ssl-dir}/*.crt
path_list = ${dynamic-nginx-frontend-template:rendered} ${caddy-directory:nginx-slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
signature_file = ${directory:run}/nginx_graceful_signature
extra-context =
......@@ -741,7 +768,8 @@ environment =
CADDYPATH=${directory:nginx_cluster}
command-line = {{ parameter_dict['caddy'] }}
-conf ${dynamic-nginx-frontend-template:rendered}
-log stdout
-log ${nginx-configuration:error_log}
-log-roll-mb 0
{% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
-http2=false
{% else %}
......@@ -749,7 +777,7 @@ command-line = {{ parameter_dict['caddy'] }}
{% endif %}
-grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
wrapper-path = ${directory:bin}/nginx-wrapper
[nginx-frontend]
......@@ -774,7 +802,6 @@ extra-context =
key master_certificate caddy-configuration:master-certificate
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
key apache_key apache-key:rendered
# BBB: SlapOS Master non-zero knowledge END
[nginx-configuration]
......
......@@ -75,7 +75,7 @@ context =
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% set unauthorized_message = 'slave not authorized' %}
{% for slave in slave_instance_list %}
{% for slave in sorted(slave_instance_list) %}
{% set slave_error_list = [] %}
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
......@@ -114,8 +114,6 @@ context =
{% if not unauthorized_message in slave_error_list %}
{% do slave_error_list.append(unauthorized_message) %}
{% endif %}
{% elif subprocess_module.call([caddy_custom_http_validator, '' ~ slave[key]]) == 1 %}
{% do slave_error_list.append('slave %s configuration invalid' % (key,)) %}
{% endif %}
{% endif %}
{% endfor %} {# for key in ['caddy_custom_http', 'caddy_custom_https', 'apache_custom_http', 'apache_custom_https'] #}
......
......@@ -159,7 +159,7 @@ command-line = {{ parameter_dict['caddy'] }}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
-root ${directory:csr_id}
wrapper-path = ${directory:service}/expose-csr_id
......
......@@ -139,13 +139,6 @@
"title": "Verify Backend Certificates",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file",
"textarea": true,
"title": "SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_proxy_ca_crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the backend (to be used with ssl-proxy-verify)",
......
......@@ -97,27 +97,6 @@
"title": "Prefer gzip Encoding for Backend",
"type": "string"
},
"ssl_ca_crt": {
"default": "",
"description": "Content of the CA certificate file",
"textarea": true,
"title": "SSL Certificate Authority's Certificate",
"type": "string"
},
"ssl_crt": {
"default": "",
"description": "Content of the SSL Certificate file",
"textarea": true,
"title": "SSL Certificate",
"type": "string"
},
"ssl_key": {
"default": "",
"description": "Content of the SSL Key file",
"textarea": true,
"title": "SSL Key",
"type": "string"
}
},
"title": "Input Parameters",
"type": "object"
......
......@@ -59,7 +59,6 @@ extra-context =
import validators validators
key cluster_identification instance-parameter:root-instance-title
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw caddy_custom_http_validator {{ caddy_custom_http_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
......
......@@ -2,6 +2,8 @@
extends = common.cfg
[versions]
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
# Versions pinned for kedifa need urllib3 >= 1.18
urllib3 = 1.24
requests = 2.20.0
......
......@@ -4,45 +4,38 @@ import {{frontend_configuration.get('log-access-configuration')}}
import {{ slave_configuration_directory }}/*.conf
import {{ slave_with_cache_configuration_directory }}/*.conf
{%- set ssl = {} -%}
{%- if os_module.path.exists(master_certificate) -%}
{%- do ssl.__setitem__('certificate', master_certificate) -%}
{%- do ssl.__setitem__('key', master_certificate) -%}
{#- BBB: SlapOS Master non-zero knowledge BEGIN -#}
{%- elif os_module.path.getsize(apache_certificate) > 0 and os_module.path.getsize(apache_key) > 0 -%}
{%- do ssl.__setitem__('certificate', apache_certificate) -%}
{%- do ssl.__setitem__('key', apache_key) -%}
{%- endif -%}
{#- BBB: SlapOS Master non-zero knowledge END #}
# Catch-all and 404 for not configured instances
{% if 'key' in ssl %}
:{{ https_port }} {
tls {{ ssl['certificate'] }} {{ ssl['key'] }}
tls {{ master_certificate }} {{ master_certificate }}
bind {{ local_ipv4 }}
# Compress the output
gzip
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
{% endif %}
:{{ http_port }} {
bind {{ local_ipv4 }}
# Compress the output
gzip
status 404 /
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
# Access to server-status Caddy-style
https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {
tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-key'] }}
tls {{ frontend_configuration['ip-access-certificate'] }} {{ frontend_configuration['ip-access-certificate'] }}
# Compress the output
gzip
bind {{ local_ipv4 }}
......@@ -52,8 +45,11 @@ https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv
}
expvar
pprof
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}"
log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
rotate_size 0
}
errors {{ error_log }} {
rotate_size 0
* {{ not_found_file }}
}
}
{% if software_type == slap_software_type %}
{% set kedifa_updater_mapping = [] %}
{% set cached_server_dict = {} %}
{% set part_list = [] %}
{% set cache_port = caddy_configuration.get('cache-port') %}
......@@ -20,7 +21,6 @@ recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
import os_module os
raw common_profile {{ common_profile }}
${:extra-context}
......@@ -32,12 +32,7 @@ notifempty = true
create = true
{% if master_key_download_url %}
{% do part_list.append('master-key-download') %}
[master-key-download]
recipe = plone.recipe.command
destination = {{ master_certificate }}
command = {{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ master_key_download_url }}
update-command = ${:command}
{% do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
{% endif %}
{% if slave_kedifa_information %}
......@@ -174,34 +169,11 @@ bytes = 8
{# ################################################## #}
{# Set Slave Certificates if needed #}
{% set cert_dirname = slave_reference.replace('-','.') %}
{% set autocert_dir = '/'.join([autocert, cert_dirname]) %}
[{{ slave_reference }}-path]
recipe = slapos.cookbook:mkdirectory
cert = {{ autocert_dir }}
{# Set certificate key for custom configuration #}
{% set certificate = '%s/certificate.pem' % (autocert_dir, ) %}
{# Set certificate key for custom configuration #}
{% set cert_name = slave_reference.replace('-','.') + '.pem' %}
{% set certificate = '%s/%s' % (autocert, cert_name) %}
{% do slave_parameter_dict.__setitem__('certificate', certificate )%}
[{{ slave_reference }}-key-download]
recipe = plone.recipe.command
destination = {{ '${' + slave_reference + '-path:cert}/downloaded.pem' }}
used = {{ '${' + slave_reference + '-path:cert}/certificate.pem' }}
source-master = ${master-key-download:destination}
command =
{{ kedifa_getter }} --out ${:destination} --server-ca-certificate {{ kedifa_caucase_ca_certificate }} --identity {{ kedifa_login_certificate }} {{ key_download_url }}
if [ -f ${:destination} ] ; then
# if the slave specific certificate is available, use it
ln -sf ${:destination} ${:used}
elif [ -f ${:source-master} ] ; then
# if the master provided certificate is available, use it
ln -sf ${:source-master} ${:used}
else
rm -f ${:used}
fi
update-command = ${:command}
# BBB: SlapOS Master non-zero knowledge BEGIN
{# Set ssl certificates for each slave #}
{% for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
{% if cert_name in slave_instance %}
......@@ -217,6 +189,7 @@ template = {{ empty_template }}
rendered = {{ cert_file }}
extra-context =
key content {{ cert_title + '-config:value' }}
# BBB: SlapOS Master non-zero knowledge BEGIN
# Store certificate in config
[{{ cert_title + '-config' }}]
value = {{ dumps(slave_instance.get(cert_name)) }}
......@@ -224,42 +197,29 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
{% endfor %}
{#- Set Up Certs #}
{% do slave_instance.__setitem__('apache_certificate', apache_certificate) %}
{% do slave_instance.__setitem__('apache_key', apache_key) %}
{% if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %}
{% set cert_title = '%s-crt' % (slave_reference) %}
{% set key_title = '%s-key' % (slave_reference) %}
{% set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}
{% set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) %}
{% set cert_file = '/'.join([bbb_ssl_directory, cert_title.replace('-','.')]) %}
{% do kedifa_updater_mapping.append((key_download_url, certificate, cert_file)) %}
{% do part_list.append(cert_title) %}
{% do part_list.append(key_title) %}
{% do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %}
{% do slave_parameter_dict.__setitem__("ssl_key", key_file) %}
{% do slave_instance.__setitem__('path_to_ssl_crt', cert_file) %}
{% do slave_instance.__setitem__('path_to_ssl_key', key_file) %}
[{{key_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ key_file }}
key-content = {{ dumps(slave_instance.get('ssl_key')) }}
extra-context =
key content :key-content
[{{cert_title}}]
< = jinja2-template-base
template = {{ empty_template }}
rendered = {{ cert_file }}
cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '')) }}
cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '') + '\n' + slave_instance.get('ssl_key')) }}
extra-context =
key content :cert-content
{% endif %}
{% else %}
{% do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %}
{% endif %}
# BBB: SlapOS Master non-zero knowledge END
{# ########################################## #}
{# Set Slave Configuration #}
[{{ slave_configuration_section_name }}]
certificate = {{ '${' + slave_reference + '-key-download:used}' }}
certificate = {{ certificate }}
https_port = {{ dumps('' ~ https_port) }}
http_port = {{ dumps('' ~ http_port) }}
local_ipv4 = {{ dumps('' ~ local_ipv4) }}
......@@ -466,7 +426,6 @@ global_ipv6 = {{ dumps(global_ipv6) }}
https_port = {{ dumps(https_port) }}
http_port = {{ dumps(http_port) }}
ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }}
ip_access_key = {{ frontend_configuration.get('ip-access-key') }}
access_log = {{ dumps(access_log) }}
error_log = {{ dumps(error_log) }}
not_found_file = {{ dumps(not_found_file) }}
......@@ -493,12 +452,44 @@ monitor-base-url = {{ monitor_base_url }}
csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt
csr_id-certificate = ${get-csr_id-certificate:certificate}
[kedifa-updater]
recipe = slapos.cookbook:wrapper
command-line = {{ kedifa_updater }}
--server-ca-certificate {{ kedifa_caucase_ca_certificate }}
--identity {{ kedifa_login_certificate }}
--master-certificate {{ master_certificate }}
--on-update "{{ frontend_graceful_reload }} ; {{ nginx_graceful_reload }}"
${kedifa-updater-mapping:file}
{{ kedifa_updater_state_file }}
wrapper-path = {{ service_directory }}/kedifa-updater
hash-files = ${buildout:directory}/software_release/buildout.cfg
[kedifa-updater-mapping]
recipe = slapos.recipe.template:jinja2
file = {{ kedifa_updater_mapping_file }}
template = inline:
{% for mapping in kedifa_updater_mapping %}
{{ mapping[0] }} {{ mapping[1] }} {{ mapping[2] }}
{% endfor %}
rendered = ${:file}
[caddy-log-access-header]
# Caddy refuse to start if an `import`ed file is empty, so we prepend a header
# so that the file is never empty.
< = jinja2-template-base
template = inline: # This file contain directives to serve directories with log files
rendered = {{frontend_configuration.get('log-access-configuration')}}
[buildout]
extends =
{{ common_profile }}
{{ logrotate_base_instance }}
parts +=
kedifa-updater
caddy-log-access-header
{% for part in part_list %}
{{ ' %s' % part }}
{% endfor %}
......@@ -570,7 +561,7 @@ command-line = {{ caddy_executable }}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-sni-challenge
-disable-tls-alpn-challenge
-root {{ directory_csr_id }}
wrapper-path = {{ service_directory }}/expose-csr_id
......