Commit d1334b4b authored by Łukasz Nowak's avatar Łukasz Nowak

Feature/caddy frontend improvement 202010

See merge request !836
parents 7f8cd116 ff9a04e2
Pipeline #11964 failed with stage
......@@ -448,6 +448,15 @@ This allows backends to:
Technical notes
===============
Profile development guidelines
------------------------------
Keep the naming in instance profiles:
* ``software_parameter_dict`` for values coming from software
* ``instance_parameter_dict`` for **local** values generated by the instance, except ``configuration``
* ``slapparameter_dict`` for values coming from SlapOS Master
Instantiated cluster structure
------------------------------
......
......@@ -14,29 +14,29 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = bfb647325103640c19c38e7b7f6e2833
md5sum = 28bf0c4c75c028bed79fc38786831b3e
[template-common]
[profile-common]
filename = instance-common.cfg.in
md5sum = 5784bea3bd608913769ff9a8afcccb68
[template-apache-frontend]
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 02aac183352a6fd6ddd336d2e3757405
md5sum = e7d7e1448b6420657e953026573311ca
[template-caddy-replicate]
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 54c0648c8593699dae0c565bc7dd8629
md5sum = 59f3a67999f5fb3e595486e2b801af08
[template-slave-list]
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 2690fb2b5b6cefb7de53f35c214bdd52
md5sum = 64d57678c12f539247fe2532c5b8d6b8
[template-replicate-publish-slave-information]
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
md5sum = 7e3ee70c447f8203273d78f66ab519c3
md5sum = de268251dafa5ad83ebf5b20636365d9
[template-caddy-frontend-configuration]
[profile-caddy-frontend-configuration]
_update_hash_filename_ = templates/Caddyfile.in
md5sum = 2503056e35463e045db3329bb8b6fae8
......@@ -54,7 +54,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = 80081a7ded0029bb24b4fe2d06b6ae95
md5sum = bf40f8d0a049a8dd924ccc731956c87e
[template-log-access]
_update_hash_filename_ = templates/template-log-access.conf.in
......@@ -112,13 +112,13 @@ md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
filename = caddyprofiledummy.py
md5sum = 38792c2dceae38ab411592ec36fff6a8
[template-kedifa]
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = d76fe7bf062410eda7049446ed06a736
md5sum = 3daebc4b37088fa01183a853920d4143
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
md5sum = be899b04e1aa652ed510f20d4ea523dd
md5sum = 3ec9e088817f6a0e3b3b71919590e6b3
[template-slave-introspection-httpd-nginx]
_update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in
......
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/caddy/buildout.cfg
../../component/gzip/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/trafficserver/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/caucase/buildout.cfg
# Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg
parts +=
caucase-eggs
template
template-caddy-frontend
template-caddy-replicate
caddy
logrotate
rdiff-backup
caddyprofiledeps
kedifa-develop
kedifa
[kedifa-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = d6bbd7db215e12871c1536f22a8fbf994227270c
[kedifa-develop]
recipe = zc.recipe.egg:develop
setup = ${kedifa-repository:location}
[kedifa]
recipe = zc.recipe.egg
eggs =
${python-cryptography:egg}
kedifa
[caddyprofiledeps-setup]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/setup.py
[caddyprofiledeps-dummy]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/caddyprofiledummy.py
[caddyprofiledeps-prepare]
recipe = plone.recipe.command
stop-on-error = True
location = ${buildout:parts-directory}/${:_buildout_section_name_}
update-command = ${:command}
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
cp ${caddyprofiledeps-setup:target} ${:location}/ &&
cp ${caddyprofiledeps-dummy:target} ${:location}/
[caddyprofiledeps-develop]
recipe = zc.recipe.egg:develop
setup = ${caddyprofiledeps-prepare:location}
[caddyprofiledeps]
depends = ${caddyprofiledeps-develop:recipe}
recipe = zc.recipe.egg
eggs =
caddyprofiledeps
websockify
collective.recipe.shelloutput
[template-common]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-common.cfg.in
rendered = ${buildout:directory}/instance-common.cfg
mode = 0644
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
[template-frontend-parameter-section]
common_profile = ${template-common:rendered}
logrotate_base_instance = ${template-logrotate-base:rendered}
bin_directory = ${buildout:bin-directory}
sixtunnel = ${6tunnel:location}
nginx = ${nginx-output:nginx}
nginx_mime = ${nginx-output:mime}
caddy = ${caddy:output}
caddy_location = ${caddy:location}
haproxy_executable = ${haproxy:location}/sbin/haproxy
rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
curl = ${curl:location}
dash = ${dash:location}
gzip = ${gzip:location}
logrotate = ${logrotate:location}
openssl = ${openssl:location}/bin/openssl
openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
trafficserver = ${trafficserver:location}
sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
xz_location = ${xz-utils:location}
htpasswd = ${:bin_directory}/htpasswd
monitor_template = ${monitor-template:output}
template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
template_graceful_script = ${template-graceful-script:target}
template_validate_script = ${template-validate-script:target}
template_rotate_script = ${template-rotate-script:target}
template_configuration_state_script = ${template-configuration-state-script:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
template_log_access = ${template-log-access:target}
template_not_found_html = ${template-not-found-html:target}
template_slave_list = ${template-slave-list:target}
template_trafficserver_records_config = ${template-trafficserver-records-config:target}
template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
template_wrapper = ${template-wrapper:output}
template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
key common_profile template-common:rendered
key monitor2_template monitor2-template:rendered
key template_caddy_frontend template-caddy-frontend:target
key template_caddy_replicate template-caddy-replicate:target
key template_kedifa template-kedifa:target
key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
key caddy_backend_url_validator caddy-backend-url-validator:output
section template_frontend_parameter_dict template-frontend-parameter-section
key caucase_jinja2_library caucase-jinja2-library:target
[template-caddy-frontend]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
mode = 0644
[caddy-backend-url-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[template-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
mode = 0644
[template-kedifa]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-kedifa.cfg.in
mode = 0644
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[template-slave-list]
<=download-template
[template-replicate-publish-slave-information]
<=download-template
[template-caddy-frontend-configuration]
<=download-template
[template-not-found-html]
<=download-template
[template-default-slave-virtualhost]
<=download-template
[template-backend-haproxy-configuration]
<=download-template
[template-log-access]
<=download-template
[template-empty]
<=download-template
[template-slave-introspection-httpd-nginx]
<=download-template
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
<=download-template
[template-trafficserver-storage-config]
<=download-template
[template-trafficserver-logging-yaml]
<=download-template
[template-rotate-script]
<=download-template
[template-caddy-lazy-script-call]
<=download-template
[template-graceful-script]
<=download-template
[template-validate-script]
<=download-template
[template-configuration-state-script]
<=download-template
[template-backend-haproxy-rsyslogd-conf]
<=download-template
# Development profile of caddy-frontend.
# Exactly the same as software.cfg, but fetch the slapos.cookbook
# from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
# You'll need to run buildout twice for this profile.
[buildout]
extends =
# Extend in this order, otherwise "parts" will be taken from git profile
common.cfg
parts +=
slapos.toolbox-dev
[slapos.toolbox-dev]
recipe = zc.recipe.egg:develop
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location}
{%- if slap_software_type == software_type -%}
{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
{% import "caucase" as caucase with context %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
[buildout]
extends =
{{ parameter_dict['common_profile'] }}
{{ parameter_dict['monitor_template'] }}
{{ parameter_dict['logrotate_base_instance'] }}
{{ software_parameter_dict['profile_common'] }}
{{ software_parameter_dict['profile_monitor'] }}
{{ software_parameter_dict['profile_logrotate_base'] }}
parts =
directory
......@@ -98,20 +98,14 @@ slave-introspection-var = ${:var}/slave-introspection
[switch-caddy-softwaretype]
recipe = slapos.cookbook:softwaretype
single-default = ${dynamic-custom-personal-template-slave-list:rendered}
single-custom-personal = ${dynamic-custom-personal-template-slave-list:rendered}
single-default = ${dynamic-custom-personal-profile-slave-list:rendered}
single-custom-personal = ${dynamic-custom-personal-profile-slave-list:rendered}
[frontend-configuration]
template-log-access = {{ parameter_dict['template_log_access'] }}
log-access-configuration = ${directory:etc}/log-access.conf
ip-access-certificate = ${self-signed-ip-access:certificate}
caddy-directory = {{ parameter_dict['caddy_location'] }}
caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
caddy-ipv6 = {{ instance_parameter_dict['ipv6-random'] }}
caddy-https-port = ${configuration:port}
nginx = {{ parameter_dict['nginx'] }}
nginx_mime = {{ parameter_dict['nginx_mime'] }}
htpasswd = {{ parameter_dict['htpasswd'] }}
slave-introspection-template = {{ parameter_dict['template_slave_introspection_httpd_nginx'] }}
slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
slave-introspection-https-port = ${configuration:slave-introspection-https-port}
slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
......@@ -122,21 +116,22 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
command =
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
{{ software_parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:certificate} \
-subj "/CN=Self Signed IP Access" \
-reqexts SAN \
-extensions SAN \
-config <(cat {{ parameter_dict['openssl_cnf'] }} \
-config <(cat {{ software_parameter_dict['openssl_cnf'] }} \
<(printf "\n[SAN]\nsubjectAltName=IP:${:ipv6},IP:${:ipv4}")) \
-out ${:certificate}'
......@@ -145,18 +140,19 @@ command =
recipe = plone.recipe.command
update-command = ${:command}
ipv6 = ${slap-network-information:global-ipv6}
ipv4 = {{instance_parameter['ipv4-random']}}
ipv4 = {{instance_parameter_dict['ipv4-random']}}
certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
command =
[ -f ${:certificate} ] && exit 0
rm -f ${:certificate}
/bin/bash -c ' \
{{ parameter_dict['openssl'] }} req \
{{ software_parameter_dict['openssl'] }} req \
-new -newkey rsa:2048 -sha256 \
-nodes -x509 -days 36500 \
-keyout ${:certificate} \
-subj "/CN=Fallback certificate/OU={{ instance_parameter['configuration.frontend-name'] }}" \
-subj "/CN=Fallback certificate/OU={{ instance_parameter_dict['configuration.frontend-name'] }}" \
-out ${:certificate}'
[jinja2-template-base]
......@@ -164,24 +160,23 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
slapparameter_dict = {{ dumps(slapparameter_dict) }}
slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
context =
import json_module json
raw common_profile {{ parameter_dict['common_profile'] }}
raw logrotate_base_instance {{ parameter_dict['logrotate_base_instance'] }}
raw monitor_template {{ parameter_dict['monitor_template'] }}
raw profile_common {{ software_parameter_dict['profile_common'] }}
raw profile_logrotate_base {{ software_parameter_dict['profile_logrotate_base'] }}
raw profile_monitor {{ software_parameter_dict['profile_monitor'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
[software-release-path]
template-empty = {{ parameter_dict['template_empty'] }}
template-default-slave-virtualhost = {{ parameter_dict['template_default_slave_virtualhost'] }}
template-backend-haproxy-configuration = {{ parameter_dict['template_backend_haproxy_configuration'] }}
template-backend-haproxy-rsyslogd-conf = {{ parameter_dict['template_backend_haproxy_rsyslogd_conf'] }}
caddy-location = {{ parameter_dict['caddy_location'] }}
template-empty = {{ software_parameter_dict['template_empty'] }}
template-default-slave-virtualhost = {{ software_parameter_dict['template_default_slave_virtualhost'] }}
template-backend-haproxy-configuration = {{ software_parameter_dict['template_backend_haproxy_configuration'] }}
template-backend-haproxy-rsyslogd-conf = {{ software_parameter_dict['template_backend_haproxy_rsyslogd_conf'] }}
[kedifa-login-config]
d = ${directory:ca-dir}
......@@ -195,11 +190,11 @@ crl = ${:d}/kedifa-login-crl.pem
[kedifa-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
command =
{% if slapparameter_dict['kedifa-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:template-csr}
......@@ -209,11 +204,12 @@ command =
update-command = ${:command}
template-csr = ${kedifa-login-config:template-csr}
key = ${kedifa-login-config:key}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/kedifa-login-certificate-caucase-updater',
url=slapparameter_dict['kedifa-caucase-url'],
data_dir='${directory:srv}/caucase-updater',
......@@ -231,7 +227,6 @@ certificate = ${kedifa-login-config:certificate}
cas-ca-certificate = ${kedifa-login-config:cas-ca-certificate}
csr = ${caucase-updater-csr:csr}
crl = ${kedifa-login-config:crl}
kedifa-updater = {{ parameter_dict['kedifa-updater'] }}
kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }}
......@@ -248,11 +243,11 @@ crl = ${:d}/crl.pem
[backend-client-login-csr]
recipe = plone.recipe.command
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
command =
{% if slapparameter_dict['backend-client-caucase-url'] %}
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:template-csr}
......@@ -262,11 +257,12 @@ command =
update-command = ${:command}
template-csr = ${backend-client-login-config:template-csr}
key = ${backend-client-login-config:key}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
{{ caucase.updater(
prefix='backend-client-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/backend-client-login-certificate-caucase-updater',
url=slapparameter_dict['backend-client-caucase-url'],
data_dir='${directory:srv}/backend-client-caucase-updater',
......@@ -277,79 +273,54 @@ stop-on-error = True
template_csr='${backend-client-login-csr:template-csr}'
)}}
[dynamic-custom-personal-template-slave-list]
[dynamic-custom-personal-profile-slave-list]
< = jinja2-template-base
depends = ${caddyprofiledeps:recipe}
template = {{ parameter_dict['template_slave_list'] }}
template = {{ software_parameter_dict['profile_slave_list'] }}
filename = custom-personal-instance-slave-list.cfg
slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
extra_slave_instance_list = {{ dumps(instance_parameter.get('configuration.extra_slave_instance_list')) }}
master_key_download_url = {{ dumps(slapparameter_dict['master-key-download-url']) }}
local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
local_ipv6 = {{ dumps(instance_parameter['ipv6-random']) }}
software_type = single-custom-personal
bin_directory = {{ parameter_dict['bin_directory'] }}
caddy_executable = {{ parameter_dict['caddy'] }}
sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
organization = {{ slapparameter_dict['cluster-identification'] }}
organizational-unit = {{ instance_parameter['configuration.frontend-name'] }}
organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
extra-context =
key caddy_configuration_directory caddy-directory:slave-configuration
key backend_client_caucase_url :backend-client-caucase-url
import urlparse_module urlparse
import furl_module furl
key caddy_executable :caddy_executable
key http_port configuration:plain_http_port
key https_port configuration:port
key public_ipv4 configuration:public-ipv4
key slave_instance_list :slave_instance_list
key extra_slave_instance_list :extra_slave_instance_list
key master_key_download_url :master_key_download_url
key autocert caddy-directory:autocert
key master_certificate caddy-configuration:master-certificate
key caddy_log_directory caddy-directory:slave-log
key expose_csr_id_organization :organization
key expose_csr_id_organizational_unit :organizational-unit
key local_ipv4 :local_ipv4
key local_ipv6 :local_ipv6
key global_ipv6 slap-network-information:global-ipv6
key empty_template software-release-path:template-empty
key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
key software_type :software_type
key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
key frontend_graceful_reload caddy-configuration:frontend-graceful-command
section frontend_configuration frontend-configuration
section caddy_configuration caddy-configuration
key monitor_base_url monitor-instance-parameter:monitor-base-url
key bin_directory :bin_directory
key enable_http2_by_default configuration:enable-http2-by-default
key global_disable_http2 configuration:global-disable-http2
key ciphers configuration:ciphers
key access_log caddy-configuration:access-log
key error_log caddy-configuration:error-log
key sixtunnel_executable :sixtunnel_executable
key not_found_file caddy-configuration:not-found-file
key custom_ssl_directory caddy-directory:custom-ssl-directory
section kedifa_configuration kedifa-configuration
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
# BBB: SlapOS Master non-zero knowledge END
## backend haproxy
key template_backend_haproxy_configuration software-release-path:template-backend-haproxy-configuration
section backend_haproxy_configuration backend-haproxy-configuration
## full configuration
## Configuration passed by section
section configuration configuration
section backend_haproxy_configuration backend-haproxy-configuration
section instance_parameter_dict instance-parameter-section
section frontend_configuration frontend-configuration
section caddy_configuration caddy-configuration
section kedifa_configuration kedifa-configuration
section software_parameter_dict software-parameter-section
# Deploy Caddy Frontend with Jinja power
[dynamic-caddy-frontend-template]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_frontend_configuration'] }}
template = {{ software_parameter_dict['template_caddy_frontend_configuration'] }}
rendered = ${caddy-configuration:frontend-configuration}
local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
local_ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
extra-context =
key httpd_home software-release-path:caddy-location
key httpd_mod_ssl_cache_directory caddy-directory:mod-ssl
key instance_home buildout:directory
key master_certificate caddy-configuration:master-certificate
key access_log caddy-configuration:access-log
......@@ -373,16 +344,16 @@ template = inline:
#!/bin/sh
export CADDYPATH=${directory:frontend_cluster}
ulimit -n $(ulimit -Hn)
exec {{ parameter_dict['caddy'] }} \
exec {{ software_parameter_dict['caddy'] }} \
-conf ${dynamic-caddy-frontend-template:rendered} \
-log ${caddy-configuration:error-log} \
-log-roll-mb 0 \
{% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
{% if instance_parameter_dict['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
-http2=false \
{% else %}
-http2=true \
{% endif %}
-grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s \
-grace {{ instance_parameter_dict['configuration.mpm-graceful-shutdown-timeout'] }}s \
-disable-http-challenge \
-disable-tls-alpn-challenge \
"$@"
......@@ -400,14 +371,12 @@ hash-files = ${caddy-wrapper:rendered}
recipe = plone.recipe.command
update-command = ${:command}
filename = notfound.html
command = ln -sf {{ parameter_dict['template_not_found_html'] }} ${caddy-directory:document-root}/${:filename}
command = ln -sf {{ software_parameter_dict['template_not_found_html'] }} ${caddy-directory:document-root}/${:filename}
[caddy-directory]
recipe = slapos.cookbook:mkdirectory
document-root = ${directory:srv}/htdocs
slave-configuration = ${directory:etc}/caddy-slave-conf.d/
cache = ${directory:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl
slave-log = ${directory:log}/httpd
autocert = ${directory:srv}/autocert
master-autocert-dir = ${:autocert}/master-autocert
......@@ -469,7 +438,7 @@ delaycompress =
recipe = slapos.cookbook:mkdirectory
configuration = ${directory:etc}/trafficserver
local-state = ${directory:var}/trafficserver
bin_path = {{ parameter_dict['trafficserver'] }}/bin
bin_path = {{ software_parameter_dict['trafficserver'] }}/bin
log = ${directory:log}/trafficserver
cache-path = ${directory:srv}/ats_cache
logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
......@@ -477,7 +446,7 @@ logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
[trafficserver-variable]
wrapper-path = ${directory:service}/trafficserver
reload-path = ${directory:etc-run}/trafficserver-reload
local-ip = {{ instance_parameter['ipv4-random'] }}
local-ip = {{ instance_parameter_dict['ipv4-random'] }}
input-port = 23432
hostname = ${configuration:frontend-name}
plugin-config =
......@@ -485,24 +454,24 @@ ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
cache-path = ${trafficserver-directory:cache-path}
disk-cache-size = ${configuration:disk-cache-size}
ram-cache-size = ${configuration:ram-cache-size}
templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
templates-dir = {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
request-timeout = ${configuration:request-timeout}
[trafficserver-configuration-directory]
recipe = plone.recipe.command
command = cp -rn {{ parameter_dict['trafficserver'] }}/etc/trafficserver/* ${:target}
command = cp -rn {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/* ${:target}
target = ${trafficserver-directory:configuration}
[trafficserver-launcher]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_manager
command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_manager
wrapper-path = ${trafficserver-variable:wrapper-path}
environment = TS_ROOT=${buildout:directory}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[trafficserver-reload]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
wrapper-path = ${trafficserver-variable:reload-path}
environment = TS_ROOT=${buildout:directory}
......@@ -519,19 +488,19 @@ context =
[trafficserver-records-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_records_config'] }}
template = {{ software_parameter_dict['template_trafficserver_records_config'] }}
filename = records.config
extra-context =
import os_module os
[trafficserver-storage-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_storage_config'] }}
template = {{ software_parameter_dict['template_trafficserver_storage_config'] }}
filename = storage.config
[trafficserver-logging-yaml]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_trafficserver_logging_yaml'] }}
template = {{ software_parameter_dict['template_trafficserver_logging_yaml'] }}
filename = logging.yaml
[trafficserver-remap-config]
......@@ -542,7 +511,7 @@ template = inline:
map / http://{{ ipv4 }}:{{ http_port }}
{%- endraw %}
extra-context =
raw ipv4 {{ instance_parameter['ipv4-random'] }}
raw ipv4 {{ instance_parameter_dict['ipv4-random'] }}
key https_port backend-haproxy-configuration:https-port
key http_port backend-haproxy-configuration:http-port
......@@ -550,14 +519,14 @@ filename = remap.config
[trafficserver-plugin-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
filename = plugin.config
context =
key content trafficserver-variable:plugin-config
[trafficserver-ip-allow-config]
< = trafficserver-jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
filename = ip_allow.config
context =
key content trafficserver-variable:ip-allow-config
......@@ -571,7 +540,7 @@ config-port = ${trafficserver-variable:input-port}
[trafficserver-ctl]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl
command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_ctl
wrapper-path = ${directory:bin}/traffic_ctl
environment = TS_ROOT=${buildout:directory}
......@@ -583,10 +552,10 @@ config-wrapper-path = ${trafficserver-ctl:wrapper-path}
[trafficserver-rotate-script]
< = jinja2-template-base
template = {{ parameter_dict['template_rotate_script'] }}
template = {{ software_parameter_dict['template_rotate_script'] }}
rendered = ${directory:bin}/trafficserver-rotate
mode = 0700
xz_binary = {{ parameter_dict['xz_location'] ~ '/bin/xz' }}
xz_binary = {{ software_parameter_dict['xz_location'] ~ '/bin/xz' }}
pattern = *.old
# days to keep log files
keep_days = 365
......@@ -610,12 +579,12 @@ command = ${trafficserver-rotate-script:rendered}
### Caddy Graceful and promises
[frontend-caddy-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_configuration_state_script'] }}
template = {{ software_parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
sha256sum = {{ parameter_dict['sha256sum'] }}
sha256sum = {{ software_parameter_dict['sha256sum'] }}
extra-context =
key path_list :path_list
......@@ -632,7 +601,7 @@ signature_file = ${directory:run}/validate_configuration_state_signature
[frontend-caddy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
template = {{ software_parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
mode = 0700
......@@ -642,7 +611,7 @@ extra-context =
[frontend-caddy-validate]
< = jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
template = {{ software_parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/frontend-caddy-validate
mode = 0700
last_state_file = ${directory:run}/caddy_configuration_last_state
......@@ -654,7 +623,7 @@ extra-context =
[frontend-caddy-lazy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
template = {{ software_parameter_dict['template_caddy_lazy_script_call'] }}
rendered = ${directory:bin}/frontend-caddy-lazy-graceful
mode = 0700
pid-file = ${directory:run}/lazy-graceful.pid
......@@ -667,7 +636,7 @@ extra-context =
# Promises checking configuration:
[promise-helper-last-configuration-state]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
rendered = ${directory:bin}/frontend-read-last-configuration-state
mode = 0700
content =
......@@ -686,42 +655,42 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv4_https.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v4-http]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv4_http.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:plain_http_port}
[promise-caddy-frontend-v6-https]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv6_https.py
config-hostname = {{ instance_parameter['ipv6-random'] }}
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v6-http]
<= monitor-promise-base
module = check_port_listening
name = caddy_frontend_ipv6_http.py
config-hostname = {{ instance_parameter['ipv6-random'] }}
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:plain_http_port}
[promise-backend-haproxy-http]
<= monitor-promise-base
module = check_port_listening
name = backend_haproxy_http.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:http-port}
[promise-backend-haproxy-https]
<= monitor-promise-base
module = check_port_listening
name = backend_haproxy_https.py
config-hostname = {{ instance_parameter['ipv4-random'] }}
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:https-port}
[backend-haproxy-configuration]
......@@ -748,13 +717,13 @@ statistic-frontend-secure_access = ${backend-haproxy-statistic-frontend:connecti
[backend-haproxy]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file}
command-line = {{ software_parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file}
wrapper-path = ${directory:service}/backend-haproxy
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[backend-haproxy-rsyslogd-lazy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
template = {{ software_parameter_dict['template_caddy_lazy_script_call'] }}
rendered = ${directory:bin}/backend-haproxy-rsyslogd-lazy-graceful
mode = 0700
pid-file = ${directory:run}/backend-haproxy-rsyslogd-lazy-graceful.pid
......@@ -779,12 +748,12 @@ delaycompress =
[backend-haproxy-configuration-state]
<= jinja2-template-base
template = {{ parameter_dict['template_configuration_state_script'] }}
template = {{ software_parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${backend-haproxy-configuration:file} ${backend-client-login-config:certificate}
sha256sum = {{ parameter_dict['sha256sum'] }}
sha256sum = {{ software_parameter_dict['sha256sum'] }}
extra-context =
key path_list :path_list
......@@ -801,7 +770,7 @@ signature_file = ${directory:run}/backend_haproxy_validate_configuration_state_s
[backend-haproxy-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
template = {{ software_parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/backend-haproxy-safe-graceful
mode = 0700
......@@ -811,11 +780,11 @@ extra-context =
[backend-haproxy-validate]
<= jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
template = {{ software_parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/backend-haproxy-validate
mode = 0700
last_state_file = ${directory:run}/backend_haproxy_configuration_last_state
validate_command = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file} -c
validate_command = {{ software_parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file} -c
extra-context =
key validate_command :validate_command
key configuration_state_command backend-haproxy-configuration-state-validate:rendered
......@@ -829,7 +798,7 @@ config-verification-script = ${promise-backend-haproxy-configuration-helper:rend
[promise-backend-haproxy-configuration-helper]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
rendered = ${directory:bin}/backend-haproxy-read-last-configuration-state
mode = 0700
content =
......@@ -855,7 +824,7 @@ extra-context =
[backend-haproxy-rsyslogd]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['rsyslogd_executable'] }} -i ${backend-haproxy-rsyslogd-config:pid-file} -n -f ${backend-haproxy-rsyslogd-configuration:rendered}
command-line = {{ software_parameter_dict['rsyslogd_executable'] }} -i ${backend-haproxy-rsyslogd-config:pid-file} -n -f ${backend-haproxy-rsyslogd-configuration:rendered}
wrapper-path = ${directory:service}/backend-haproxy-rsyslogd
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
......@@ -867,8 +836,8 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
# Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
# directly, and in our case it can come from the network, thus resulting
# with need to strip !py!'u'
monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter['configuration.monitor-password'] | string }}
monitor-httpd-port = {{ instance_parameter_dict['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter_dict['configuration.monitor-password'] | string }}
[monitor-conf-parameters]
private-path-list +=
......@@ -877,35 +846,35 @@ private-path-list +=
[monitor-traffic-summary-last-stats-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
template = {{ software_parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/traffic-summary-last-stats_every_1_hour
mode = 0700
command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ parameter_dict['trafficserver'] }}/bin/traffic_logstats -f ${trafficserver-directory:log}/squid.blog)</pre>"
command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ software_parameter_dict['trafficserver'] }}/bin/traffic_logstats -f ${trafficserver-directory:log}/squid.blog)</pre>"
extra-context =
key content monitor-traffic-summary-last-stats-wrapper:command
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
template = {{ software_parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/ats-cache-stats_every_1_hour
mode = 0700
command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ parameter_dict['trafficserver'] }}/bin/traffic_shell ${monitor-ats-cache-stats-config:rendered})</pre>"
command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ software_parameter_dict['trafficserver'] }}/bin/traffic_shell ${monitor-ats-cache-stats-config:rendered})</pre>"
extra-context =
key content monitor-ats-cache-stats-wrapper:command
[monitor-caddy-server-status-wrapper]
< = jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
template = {{ software_parameter_dict['template_wrapper'] }}
rendered = ${directory:bin}/monitor-caddy-server-status-wrapper
mode = 0700
command = {{ parameter_dict['curl'] }}/bin/curl -s http://{{ instance_parameter['ipv4-random'] }}:${configuration:plain_http_port}/server-status -u ${monitor-instance-parameter:username}:${monitor-htpasswd:passwd} 2>&1
command = {{ software_parameter_dict['curl'] }}/bin/curl -s http://{{ instance_parameter_dict['ipv4-random'] }}:${configuration:plain_http_port}/server-status -u ${monitor-instance-parameter:username}:${monitor-htpasswd:passwd} 2>&1
extra-context =
key content monitor-caddy-server-status-wrapper:command
[monitor-ats-cache-stats-config]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
rendered = ${trafficserver-configuration-directory:target}/cache-config.stats
mode = 644
context =
......@@ -930,31 +899,31 @@ extra-context =
[slave-introspection-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Slave Introspection Frontend {{ instance_parameter['configuration.frontend-name'] }}
name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.slave-introspection-https-port'] }}/
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
config-https-only = true
return = domain secure_access
[backend-haproxy-statistic-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Backend Haproxy Statistic Frontend {{ instance_parameter['configuration.frontend-name'] }}
name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.backend-haproxy-statistic-port'] }}/
config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
config-https-only = true
return = domain secure_access
[slave-introspection-configuration-state]
<= jinja2-template-base
template = {{ parameter_dict['template_configuration_state_script'] }}
template = {{ software_parameter_dict['template_configuration_state_script'] }}
rendered = ${directory:bin}/${:_buildout_section_name_}
mode = 0700
path_list = ${frontend-configuration:slave-introspection-configuration} ${frontend-configuration:ip-access-certificate}
sha256sum = {{ parameter_dict['sha256sum'] }}
sha256sum = {{ software_parameter_dict['sha256sum'] }}
extra-context =
key path_list :path_list
......@@ -971,7 +940,7 @@ signature_file = ${directory:run}/slave_introspection_validate_configuration_sta
[slave-introspection-graceful]
< = jinja2-template-base
template = {{ parameter_dict['template_graceful_script'] }}
template = {{ software_parameter_dict['template_graceful_script'] }}
rendered = ${directory:etc-run}/slave-introspection-safe-graceful
mode = 0700
......@@ -981,11 +950,11 @@ extra-context =
[slave-introspection-validate]
<= jinja2-template-base
template = {{ parameter_dict['template_validate_script'] }}
template = {{ software_parameter_dict['template_validate_script'] }}
rendered = ${directory:bin}/slave-introspection-validate
mode = 0700
last_state_file = ${directory:run}/slave_introspection_configuration_last_state
validate_command = {{ parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
validate_command = {{ software_parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
extra-context =
key validate_command :validate_command
key configuration_state_command slave-introspection-configuration-state-validate:rendered
......@@ -999,7 +968,7 @@ config-verification-script = ${promise-slave-introspection-configuration-helper:
[promise-slave-introspection-configuration-helper]
< = jinja2-template-base
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
rendered = ${directory:bin}/slave-introspection-read-last-configuration-state
mode = 0700
content =
......@@ -1012,7 +981,7 @@ context =
<= monitor-promise-base
module = check_port_listening
name = slave_introspection_https.py
config-hostname = {{ instance_parameter['ipv6-random'] }}
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port}
[logrotate-entry-slave-introspection]
......@@ -1031,9 +1000,25 @@ config-command =
${logrotate:wrapper-path} -d
[configuration]
{%- for key, value in instance_parameter.iteritems() -%}
{%- for key, value in instance_parameter_dict.iteritems() -%}
{%- if key.startswith('configuration.') %}
{{ key.replace('configuration.', '') }} = {{ dumps(value) }}
{%- endif -%}
{%- endfor -%}
{%- endif -%} {# if slap_software_type == software_type #}
{%- endfor %}
[instance-parameter-section]
{#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
{%- for key, value in instance_parameter_dict.iteritems() -%}
{%- if not key.startswith('configuration.') and key not in DROP_KEY_LIST %}
{{ key }} = {{ dumps(value) }}
{%- endif -%}
{%- endfor %}
[software-parameter-section]
{%- for key, value in software_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }}
{%- endfor %}
{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}
{% if slap_software_type in software_type %}
{% if instance_parameter_dict['slap-software-type'] in software_type %}
{% set aibcc_enabled = True %}
{% import "caucase" as caucase with context %}
{#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
{%- set SERVER_POLLUTED_KEY_LIST = ['connection-parameter-hash', 'timestamp', 'slave_title', 'slap_software_type'] -%}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
{%- set GOOD_CIPHER_LIST = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-AES256-CBC-SHA', 'ECDHE-RSA-AES128-CBC-SHA', 'ECDHE-ECDSA-AES256-CBC-SHA', 'ECDHE-ECDSA-AES128-CBC-SHA', 'RSA-AES256-CBC-SHA', 'RSA-AES128-CBC-SHA', 'ECDHE-RSA-3DES-EDE-CBC-SHA', 'RSA-3DES-EDE-CBC-SHA'] %}
{#- Allow to pass only some parameters to frontend nodes #}
{%- set FRONTEND_NODE_PASSED_KEY_LIST = [
'plain_http_port',
'port',
'apache-certificate',
'apache-key',
'domain',
'enable-http2-by-default',
'global-disable-http2',
'mpm-graceful-shutdown-timeout',
'public-ipv4',
're6st-verification-url',
'backend-connect-timeout',
'backend-connect-retries',
'ciphers',
'request-timeout',
'authenticate-to-backend',
]
%}
{% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
{% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
{# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
{% set master_partition_monitor_monitor_httpd_port = 8401 %}
{% set kedifa_partition_monitor_httpd_port = 8402 %}
{% set frontend_monitor_httpd_base_port = 8410 %}
{% set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_backend_client_port'] %}
{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_backend_client_port'] %}
{% set caucase_url = 'http://' ~ caucase_netloc %}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -20,18 +39,18 @@ rendered = ${buildout:directory}/${:filename}
extra-context =
context =
import json_module json
raw common_profile {{ common_profile }}
raw profile_common {{ software_parameter_dict['profile_common'] }}
${:extra-context}
{% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
{% set part_list = [] %}
{% set single_type_key = 'single-' %}
{% if slap_software_type == "replicate" %}
{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
{% set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
{% elif slap_software_type in ['default', 'RootSoftwareInstance'] %}
{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
{% else %}
{% set frontend_type = "%s%s" % (single_type_key, slap_software_type) %}
{% set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
{% endif %}
{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
{% set slave_list_name = 'extra_slave_instance_list' %}
......@@ -70,16 +89,19 @@ context =
{% endfor %}
{% do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %}
{% do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %}
{% do frontend_list.append(frontend_name) %}
{% do frontend_section_list.append(request_section_title) %}
{% set state_key = "-frontend-%s-state" % i %}
{% set frontend_state = slapparameter_dict.pop(state_key, None) %}
{% if frontend_state != 'destroyed' %}
{% do frontend_list.append(frontend_name) %}
{% do frontend_section_list.append(request_section_title) %}
{% endif %}
{% do part_list.append(request_section_title) %}
# Filling request dict for slave
{% set state_key = "-frontend-%s-state" % i %}
{% set request_content_dict = {
'config': config_dict,
'name': frontend_name,
'sla': sla_dict,
'state': slapparameter_dict.pop(state_key, None)
'state': frontend_state
} %}
{% set frontend_software_url_key = "-frontend-%s-software-release-url" % i %}
{% do request_content_dict.__setitem__('software-url', slapparameter_dict.get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
......@@ -92,7 +114,7 @@ context =
{% set rejected_slave_title_dict = {} %}
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% for slave in sorted(slave_instance_list) %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
{% set slave_error_list = [] %}
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
......@@ -142,7 +164,7 @@ context =
{% for url_key in ['url', 'https-url'] %}
{% if url_key in slave %}
{% set url = (slave[url_key] or '').strip() %}
{% if subprocess_module.call([caddy_backend_url_validator, url]) == 1 or not validators.url(url) %}
{% if subprocess_module.call([software_parameter_dict['caddy_backend_url_validator'], url]) == 1 or not validators.url(url) %}
{% do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
{% elif url != slave[url_key] %}
{% do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
......@@ -151,7 +173,7 @@ context =
{% endfor %}
{% if 'ssl_proxy_ca_crt' in slave %}
{% set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
{% set check_popen = popen([parameter_dict['openssl'], 'x509', '-noout']) %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(ssl_proxy_ca_crt) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
......@@ -167,8 +189,8 @@ context =
{% do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required') %}
{% endif %}
{% if slave.get('ssl_key') and slave.get('ssl_crt') %}
{% set key_popen = popen([parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{% set crt_popen = popen([parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
{% set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
{% set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
{% set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
{% set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
{% if not key_modulus or key_modulus != crt_modulus %}
......@@ -217,6 +239,13 @@ config-monitor-password = ${monitor-htpasswd:passwd}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
{#- Send only needed parameters to frontend nodes #}
{%- set base_node_configuration_dict = {} %}
{%- for key in FRONTEND_NODE_PASSED_KEY_LIST %}
{%- if key in slapparameter_dict %}
{%- do base_node_configuration_dict.__setitem__(key, slapparameter_dict[key]) %}
{%- endif %}
{%- endfor %}
{% for section, frontend_request in request_dict.iteritems() %}
{% set state = frontend_request.get('state', '') %}
[{{section}}]
......@@ -230,15 +259,18 @@ config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-infor
config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
config-backend-client-caucase-url = {{ caucase_url }}
config-master-key-download-url = ${request-kedifa:connection-master-key-download-url}
config-cluster-identification = {{ cluster_identification }}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{# Do not send additional parameters for destroyed nodes #}
{% if state != 'destroyed' %}
{% set slave_configuration_dict = slapparameter_dict %}
{% do slave_configuration_dict.update(frontend_request.get('config')) %}
{% set node_configuration_dict = {} %}
{% do node_configuration_dict.update(frontend_request.get('config')) %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
{% do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
{% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
{% do node_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
{% do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
{%- for config_key, config_value in node_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- for config_key, config_value in base_node_configuration_dict.iteritems() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{% endif %}
......@@ -260,7 +292,7 @@ sla-{{ parameter }} = {{ value }}
<= monitor-publish
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_dict | length }}
backend-client-caucase-url = {{ caucase_url }}
......@@ -327,7 +359,7 @@ config-{{ key }} = {{ dumps(slapparameter_dict[key]) }}
{%- endif %}
{%- endfor %}
config-slave-list = {{ dumps(authorized_slave_list) }}
config-cluster-identification = {{ cluster_identification }}
config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
{% set software_url_key = "-kedifa-software-release-url" %}
{% if slapparameter_dict.has_key(software_url_key) %}
......@@ -365,7 +397,7 @@ sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
[active-slave-instance]
{% set active_slave_instance_list = [] %}
{% for slave_instance in slave_instance_list %}
{% for slave_instance in instance_parameter_dict['slave-instance-list'] %}
{# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #}
{# Note: This functionality is not yet covered by tests, please modify with care #}
{% do active_slave_instance_list.append(slave_instance['slave_reference']) %}
......@@ -375,7 +407,7 @@ active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, so
[dynamic-publish-slave-information]
< = jinja2-template-base
template = {{ template_publish_slave_information }}
template = {{ software_parameter_dict['profile_replicate_publish_slave_information'] }}
filename = dynamic-publish-slave-information.cfg
extensions = jinja2.ext.do
extra-context =
......@@ -399,6 +431,8 @@ backup = ${:srv}/backup
# CAUCASE directories
caucased = ${:srv}/caucased
backup-caucased = ${:backup}/caucased
# NGINX
rejected-var = ${:var}/rejected-nginx
{% if aikc_enabled %}
[directory]
......@@ -418,11 +452,11 @@ csr_id = ${directory:aikc}/csr_id
[aikc-user-csr]
recipe = plone.recipe.command
organization = {{ cluster_identification }}
organization = {{ instance_parameter_dict['root-instance-title'] }}
organizational_unit = Automatic Internal Kedifa Caucase CSR
command =
if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:csr}
......@@ -430,6 +464,7 @@ command =
update-command = ${:command}
csr = ${aikc-config:csr}
key = ${aikc-config:key}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
......@@ -438,8 +473,8 @@ stop-on-error = True
recipe = slapos.recipe.template:jinja2
context =
key caucase_url aikc-config:caucase-url
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
exec {{ parameter_dict['bin_directory'] }}/caucase \
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
exec {{ software_parameter_dict['bin_directory'] }}/caucase \
{# raw block to use context #}
{% raw %}
--ca-url {{ caucase_url }} \
......@@ -456,7 +491,8 @@ mode = 0700
{% do part_list.append('aikc-create-user') %}
[aikc-create-user]
recipe = plone.recipe.command
stop-on-error = True
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
update-command = ${:command}
command =
if ! [ -f ${aikc-config:user-created} ] ; then
......@@ -472,7 +508,7 @@ command =
{% do part_list.append('aikc-user-caucase-updater-promise') %}
{{ caucase.updater(
prefix='aikc-user-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/aikc-user-caucase-updater',
url='${aikc-config:caucase-url}',
data_dir='${directory:srv}/caucase-updater',
......@@ -503,7 +539,7 @@ recipe = slapos.recipe.template:jinja2
context =
key csr_id_url request-{{ csr }}:connection-csr_id-url
key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
test -f ${directory:aikc}/{{ csr }}-done && exit 0
${buildout:executable} ${aikc-check-certificate:rendered} \
{# raw block to use context #}
......@@ -512,7 +548,7 @@ template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
"""{{ csr_id_certificate }}"""
{% endraw %}
if [ $? = 0 ]; then
csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
{% raw %}
{{ csr_id_url }} \
{% endraw %}
......@@ -525,7 +561,8 @@ mode = 0700
{% do part_list.append('aikc-%s' % (csr,)) %}
[aikc-{{ csr }}]
recipe = plone.recipe.command
stop-on-error = True
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
command =
${aikc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
......@@ -550,11 +587,11 @@ csr_id = ${directory:aibcc}/csr_id
[aibcc-user-csr]
recipe = plone.recipe.command
organization = {{ cluster_identification }}
organization = {{ instance_parameter_dict['root-instance-title'] }}
organizational_unit = Automatic Sign Backend Client Caucase CSR
command =
if [ ! -f ${:csr} ] && [ ! -f ${:key} ] ; then
{{ parameter_dict['openssl'] }} req -new -sha256 \
{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-out ${:csr}
......@@ -562,6 +599,7 @@ command =
update-command = ${:command}
csr = ${aibcc-config:csr}
key = ${aibcc-config:key}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
......@@ -570,8 +608,8 @@ stop-on-error = True
recipe = slapos.recipe.template:jinja2
context =
key caucase_url aibcc-config:caucase-url
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
exec {{ parameter_dict['bin_directory'] }}/caucase \
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
exec {{ software_parameter_dict['bin_directory'] }}/caucase \
{# raw block to use context #}
{% raw %}
--ca-url {{ caucase_url }} \
......@@ -590,6 +628,7 @@ mode = 0700
recipe = plone.recipe.command
# the caucase for this part is provided in this profile, so we can't fail
# as otherwise caucase will never be started...
{#- XXX: Create promise #}
stop-on-error = False
update-command = ${:command}
command =
......@@ -606,7 +645,7 @@ command =
{% do part_list.append('aibcc-user-caucase-updater-promise') %}
{{ caucase.updater(
prefix='aibcc-user-caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/aibcc-user-caucase-updater',
url='${aibcc-config:caucase-url}',
data_dir='${directory:srv}/caucase-updater',
......@@ -636,7 +675,7 @@ recipe = slapos.recipe.template:jinja2
context =
key csr_id_url request-{{ csr }}:connection-backend-client-csr_id-url
key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
test -f ${directory:aibcc}/{{ csr }}-done && exit 0
${buildout:executable} ${aibcc-check-certificate:rendered} \
{# raw block to use context #}
......@@ -645,7 +684,7 @@ template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
"""{{ csr_id_certificate }}"""
{% endraw %}
if [ $? = 0 ]; then
csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
{% raw %}
{{ csr_id_url }} \
{% endraw %}
......@@ -658,7 +697,8 @@ mode = 0700
{% do part_list.append('aibcc-%s' % (csr,)) %}
[aibcc-{{ csr }}]
recipe = plone.recipe.command
stop-on-error = True
{#- The called command is smart enough to survive errors and retry #}
stop-on-error = False
command =
${aibcc-{{ csr }}-wrapper:rendered}
update-command = ${:command}
......@@ -670,7 +710,7 @@ recipe = slapos.recipe.template:jinja2
filename = rejected-slave.json
directory = ${directory:promise-output}
rendered = ${:directory}/${:filename}
template = {{ parameter_dict['template_empty'] }}
template = {{ software_parameter_dict['template_empty'] }}
{% if rejected_slave_title_dict %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
content = {{ dumps(json_module.dumps(rejected_slave_title_dict, indent=2, sort_keys=True)) }}
......@@ -685,20 +725,15 @@ service = ${:etc}/service
promise-output = ${:srv}/promise-output
[rejected-slave-publish-configuration]
ip = {{ instance_parameter['ipv6-random'] }}
ip = {{ instance_parameter_dict['ipv6-random'] }}
port = 14455
[rejected-slave-publish]
directory = ${rejected-slave-json:directory}
url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename}
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['caddy'] }}
-conf ${rejected-slave-template:rendered}
-log stderr
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root ${:directory}
command-line = {{ software_parameter_dict['nginx'] }}
-c ${rejected-slave-template:rendered}
wrapper-path = ${directory:service}/rejected-slave-publish
hash-existing-files =
......@@ -712,6 +747,7 @@ recipe = plone.recipe.command
certificate = ${directory:etc}/rejected-slave.pem
key = ${:certificate}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
update-command = ${:command}
command =
......@@ -728,18 +764,53 @@ storage-path = ${directory:etc}/.rejected-slave.passwd
bytes = 8
user = admin
[rejected-slave-htpasswd]
recipe = plone.recipe.command
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
file = ${directory:var}/nginx-rejected.htpasswd
command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${rejected-slave-password:passwd}
update-command = ${:command}
[rejected-slave-template]
recipe = slapos.recipe.template:jinja2
var = ${directory:rejected-var}
pid = ${directory:var}/nginx-rejected.pid
template = inline:
https://:${rejected-slave-publish-configuration:port}/ {
basicauth / ${rejected-slave-password:user} ${rejected-slave-password:passwd}
tls ${rejected-slave-certificate:certificate} ${rejected-slave-certificate:key}
bind ${rejected-slave-publish-configuration:ip}
log stderr
errors stderr
daemon off;
pid ${:pid};
error_log stderr;
events {
}
http {
include {{ software_parameter_dict['nginx_mime'] }};
server {
server_name_in_redirect off;
port_in_redirect off;
error_log stderr;
access_log /dev/null;
listen [${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port} ssl;
ssl_certificate ${rejected-slave-certificate:certificate};
ssl_certificate_key ${rejected-slave-certificate:certificate};
default_type application/octet-stream;
client_body_temp_path ${:var} 1 2;
proxy_temp_path ${:var} 1 2;
fastcgi_temp_path ${:var} 1 2;
uwsgi_temp_path ${:var} 1 2;
scgi_temp_path ${:var} 1 2;
location / {
alias ${rejected-slave-json:directory}/;
autoindex off;
sendfile on;
sendfile_max_chunk 1m;
auth_basic "Rejected slave template";
auth_basic_user_file ${rejected-slave-htpasswd:file};
}
}
}
rendered = ${directory:etc}/Caddyfile-rejected-slave
rendered = ${directory:etc}/nginx-rejected-slave.conf
[promise-rejected-slave-publish-ip-port]
<= monitor-promise-base
......@@ -761,7 +832,7 @@ config-url = ${rejected-slave-publish:url}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{{ caucase.caucased(
prefix='caucased-backend-client',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
caucased_path='${directory:service}/caucased-backend-client',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
......@@ -773,8 +844,8 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[buildout]
extends =
{{ common_profile }}
{{ template_monitor }}
{{ software_parameter_dict['profile_common'] }}
{{ software_parameter_dict['profile_monitor2'] }}
parts =
monitor-base
publish-slave-information
......
{%- if slap_software_type == software_type -%}
{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
{% import "caucase" as caucase with context %}
# KeDiFa instance profile
[buildout]
extends =
{{ parameter_dict['common_profile'] }}
{{ parameter_dict['monitor_template'] }}
{{ parameter_dict['logrotate_base_instance'] }}
{{ software_parameter_dict['profile_common'] }}
{{ software_parameter_dict['profile_monitor'] }}
{{ software_parameter_dict['profile_logrotate_base'] }}
parts =
monitor-base
......@@ -25,18 +25,18 @@ parts =
# Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
# directly, and in our case it can come from the network, thus resulting
# with need to strip !py!'u'
monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter['configuration.monitor-password'] | string }}
monitor-httpd-port = {{ instance_parameter_dict['configuration.monitor-httpd-port'] | int }}
password = {{ instance_parameter_dict['configuration.monitor-password'] | string }}
[caucased]
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_port'] -%}
{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_port'] -%}
{% set caucase_url = 'http://' ~ caucase_netloc -%}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:caucased}',
......@@ -75,7 +75,8 @@ reservation = ${:srv}/reservation
# csr_id publication
csr_id = ${:srv}/csr_id
caddy-csr_id = ${:etc}/caddy-csr_id
certificate-csr_id = ${:var}/certificate-csr_id
expose-csr_id-var = ${:var}/expose-csr_id
[kedifa-csr]
recipe = plone.recipe.command
......@@ -83,22 +84,23 @@ organization = {{ slapparameter_dict['cluster-identification'] }}
organizational_unit = Kedifa Partition
command =
if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ] ; then
/bin/bash -c '{{ parameter_dict['openssl'] }} req -new -sha256 \
/bin/bash -c '{{ software_parameter_dict['openssl'] }} req -new -sha256 \
-newkey rsa:2048 -nodes -keyout ${:key} \
-subj "/O=${:organization}/OU=${:organizational_unit}" \
-reqexts SAN \
-config <(cat {{ parameter_dict['openssl_cnf'] }} \
-config <(cat {{ software_parameter_dict['openssl_cnf'] }} \
<(printf "\n[SAN]\nsubjectAltName=IP:${kedifa-config:ip}")) \
-out ${:template-csr}'
fi
update-command = ${:command}
template-csr = ${kedifa-config:template-csr}
key = ${kedifa-config:key}
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
{{ caucase.updater(
prefix='caucase-updater',
buildout_bin_directory=parameter_dict['bin_directory'],
buildout_bin_directory=software_parameter_dict['bin_directory'],
updater_path='${directory:service}/caucase-updater',
url=caucase_url,
data_dir='${directory:srv}/caucase-updater',
......@@ -119,7 +121,7 @@ csr_work_path = ${directory:tmp}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ parameter_dict['bin_directory'] }}/caucase \
{{ software_parameter_dict['bin_directory'] }}/caucase \
--ca-url {{ caucase_url }} \
--ca-crt ${kedifa-config:ca-certificate} \
--crl ${kedifa-config:crl} \
......@@ -131,20 +133,21 @@ command =
[certificate-csr_id]
recipe = plone.recipe.command
certificate = ${directory:caddy-csr_id}/certificate.pem
key = ${directory:caddy-csr_id}/key.pem
certificate = ${directory:certificate-csr_id}/certificate.pem
key = ${directory:certificate-csr_id}/key.pem
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
update-command = ${:command}
command =
if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
{{ parameter_dict['openssl'] }} req -new -newkey rsa:2048 -sha256 -subj \
"/O=${kedifa-csr:organization}/OU=${kedifa-csr:organizational_unit}/CN={{ instance_parameter['ipv6-random'] }}" \
{{ software_parameter_dict['openssl'] }} req -new -newkey rsa:2048 -sha256 -subj \
"/O=${kedifa-csr:organization}/OU=${kedifa-csr:organizational_unit}/CN={{ instance_parameter_dict['ipv6-random'] }}" \
-days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
fi
[expose-csr_id-configuration]
ip = {{ instance_parameter['ipv6-random'] }}
ip = {{ instance_parameter_dict['ipv6-random'] }}
port = 17000
key = ${certificate-csr_id:key}
certificate = ${certificate-csr_id:certificate}
......@@ -152,14 +155,40 @@ error-log = ${directory:log}/expose-csr_id.log
[expose-csr_id-template]
recipe = slapos.recipe.template:jinja2
var = ${directory:expose-csr_id-var}
pid = ${directory:var}/nginx-expose-csr_id.pid
rendered = ${directory:etc}/nginx-expose-csr_id.conf
template = inline:
https://:${expose-csr_id-configuration:port}/ {
bind ${expose-csr_id-configuration:ip}
tls ${expose-csr_id-configuration:certificate} ${expose-csr_id-configuration:key}
log ${expose-csr_id-configuration:error-log}
daemon off;
pid ${:pid};
error_log ${expose-csr_id-configuration:error-log};
events {
}
http {
include {{ software_parameter_dict['nginx_mime'] }};
server {
server_name_in_redirect off;
port_in_redirect off;
error_log ${expose-csr_id-configuration:error-log};
access_log /dev/null;
listen [${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port} ssl;
ssl_certificate ${expose-csr_id-configuration:certificate};
ssl_certificate_key ${expose-csr_id-configuration:key};
default_type application/octet-stream;
client_body_temp_path ${:var} 1 2;
proxy_temp_path ${:var} 1 2;
fastcgi_temp_path ${:var} 1 2;
uwsgi_temp_path ${:var} 1 2;
scgi_temp_path ${:var} 1 2;
location / {
alias ${directory:csr_id}/;
autoindex off;
sendfile on;
sendfile_max_chunk 1m;
}
}
}
rendered = ${directory:caddy-csr_id}/Caddyfile
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
......@@ -171,13 +200,8 @@ config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
depends = ${store-csr_id:command}
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['caddy'] }}
-conf ${expose-csr_id-template:rendered}
-log ${expose-csr_id-configuration:error-log}
-http2=true
-disable-http-challenge
-disable-tls-alpn-challenge
-root ${directory:csr_id}
command-line = {{ software_parameter_dict['nginx'] }}
-c ${expose-csr_id-template:rendered}
wrapper-path = ${directory:service}/expose-csr_id
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
......@@ -191,19 +215,19 @@ commands =
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
slapparameter_dict = {{ dumps(slapparameter_dict) }}
slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
context =
import json_module json
raw common_profile {{ parameter_dict['common_profile'] }}
raw profile_common {{ software_parameter_dict['profile_common'] }}
key slap_software_type :slap_software_type
key slapparameter_dict :slapparameter_dict
section directory directory
${:extra-context}
[kedifa-config]
ip = {{ instance_parameter['ipv6-random'] }}
port = {{ instance_parameter['configuration.kedifa_port'] }}
ip = {{ instance_parameter_dict['ipv6-random'] }}
port = {{ instance_parameter_dict['configuration.kedifa_port'] }}
db = ${directory:kedifa}/kedifa.sqlite
certificate = ${directory:etc-kedifa}/certificate.pem
key = ${:certificate}
......@@ -215,7 +239,7 @@ logfile = ${directory:log}/kedifa.log
[kedifa-reloader]
<= jinja2-template-base
template = {{ parameter_dict['template_wrapper'] }}
template = {{ software_parameter_dict['template_wrapper'] }}
rendered = ${directory:etc-run}/kedifa-reloader
command =
kill -HUP `cat ${kedifa-config:pidfile}`
......@@ -236,12 +260,12 @@ config-ca-cert-file = ${kedifa-config:ca-certificate}
<= logrotate-entry-base
name = kedifa
log = ${kedifa-config:logfile}
rotate-num = {{ instance_parameter['configuration.rotate-num'] | int }}
rotate-num = {{ instance_parameter_dict['configuration.rotate-num'] | int }}
delaycompress =
[kedifa]
recipe = slapos.cookbook:wrapper
command-line = {{ parameter_dict['kedifa'] }}
command-line = {{ software_parameter_dict['kedifa'] }}
--ip ${kedifa-config:ip}
--port ${kedifa-config:port}
--db ${kedifa-config:db}
......@@ -268,7 +292,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_}
command =
[ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
[ ! -f ${:file} ] && {{ software_parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
update-command = ${:command}
[{{ slave_reference }}-auth-random]
......@@ -283,7 +307,7 @@ commands =
recipe = plone.recipe.command
file = ${directory:reservation}/${:_buildout_section_name_}
command =
[ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
[ ! -f ${:file} ] && {{ software_parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
update-command = ${:command}
[master-auth-random]
......@@ -311,4 +335,4 @@ name = ${:_buildout_section_name_}.py
config-command =
${logrotate:wrapper-path} -d
{%- endif -%} {# if slap_software_type in software_type #}
{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}
[buildout]
extends = {{ common_profile }}
extends = {{ software_parameter_dict['profile_common'] }}
parts =
dynamic-template-caddy-replicate
switch-softwaretype
[caddyprofiledeps]
......@@ -11,76 +10,59 @@ recipe = caddyprofiledeps
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extensions = jinja2.ext.do
extra-context =
context =
import json_module json
key slap_software_type instance-parameter:slap-software-type
key slapparameter_dict instance-parameter:configuration
key slave_instance_list instance-parameter:slave-instance-list
section instance_parameter instance-parameter
section instance_parameter_dict instance-parameter
section software_parameter_dict software-parameter-section
${:extra-context}
caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
import-list =
file caucase :caucase-jinja2-library
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-template-caddy-replicate:rendered}
RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
custom-personal = ${dynamic-template-caddy-replicate:rendered}
single-default = ${dynamic-template-caddy-frontend:rendered}
single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
replicate = ${dynamic-template-caddy-replicate:rendered}
kedifa = ${dynamic-template-kedifa:rendered}
default = ${dynamic-profile-caddy-replicate:rendered}
RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered}
custom-personal = ${dynamic-profile-caddy-replicate:rendered}
single-default = ${dynamic-profile-caddy-frontend:rendered}
single-custom-personal = ${dynamic-profile-caddy-frontend:rendered}
replicate = ${dynamic-profile-caddy-replicate:rendered}
kedifa = ${dynamic-profile-kedifa:rendered}
[dynamic-template-caddy-frontend-parameters]
{% for key,value in template_frontend_parameter_dict.iteritems() %}
[software-parameter-section]
{% for key,value in software_parameter_dict.iteritems() %}
{{ key }} = {{ dumps(value) }}
{% endfor -%}
[dynamic-template-caddy-frontend]
[dynamic-profile-caddy-frontend]
< = jinja2-template-base
template = {{ template_caddy_frontend }}
template = {{ software_parameter_dict['profile_caddy_frontend'] }}
filename = instance-caddy-frontend.cfg
extensions = jinja2.ext.do
extra-context =
import furl_module furl
section parameter_dict dynamic-template-caddy-frontend-parameters
raw software_type single-custom-personal
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[dynamic-template-caddy-replicate]
[dynamic-profile-caddy-replicate]
< = jinja2-template-base
depends = ${caddyprofiledeps:recipe}
template = {{ template_caddy_replicate }}
template = {{ software_parameter_dict['profile_caddy_replicate'] }}
filename = instance-caddy-replicate.cfg
extensions = jinja2.ext.do
extra-context =
import subprocess_module subprocess
import functools_module functools
import validators validators
key cluster_identification instance-parameter:root-instance-title
raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor {{ monitor2_template }}
raw common_profile {{ common_profile }}
section parameter_dict dynamic-template-caddy-frontend-parameters
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[dynamic-template-kedifa]
[dynamic-profile-kedifa]
< = jinja2-template-base
template = {{ template_kedifa }}
template = {{ software_parameter_dict['profile_kedifa'] }}
filename = instance-kedifa.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-caddy-frontend-parameters
raw software_type kedifa
caucase-jinja2-library = {{ caucase_jinja2_library }}
import-list =
file caucase :caucase-jinja2-library
[instance-parameter]
# Fetches parameters defined in SlapOS Master for this instance.
......
[buildout]
extends = common.cfg
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/dash/buildout.cfg
../../component/caddy/buildout.cfg
../../component/gzip/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/trafficserver/buildout.cfg
../../component/6tunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/caucase/buildout.cfg
# Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg
parts +=
caucase-eggs
template
rdiff-backup
caddyprofiledeps
kedifa-develop
kedifa
[kedifa-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = d6bbd7db215e12871c1536f22a8fbf994227270c
[kedifa-develop]
recipe = zc.recipe.egg:develop
setup = ${kedifa-repository:location}
[kedifa]
recipe = zc.recipe.egg
eggs =
${python-cryptography:egg}
kedifa
[caddyprofiledeps-setup]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/setup.py
[caddyprofiledeps-dummy]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/caddyprofiledummy.py
[caddyprofiledeps-prepare]
recipe = plone.recipe.command
stop-on-error = True
location = ${buildout:parts-directory}/${:_buildout_section_name_}
update-command = ${:command}
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
cp ${caddyprofiledeps-setup:target} ${:location}/ &&
cp ${caddyprofiledeps-dummy:target} ${:location}/
[caddyprofiledeps-develop]
recipe = zc.recipe.egg:develop
setup = ${caddyprofiledeps-prepare:location}
[caddyprofiledeps]
depends = ${caddyprofiledeps-develop:recipe}
recipe = zc.recipe.egg
eggs =
caddyprofiledeps
websockify
collective.recipe.shelloutput
[profile-common]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-common.cfg.in
rendered = ${buildout:directory}/instance-common.cfg
mode = 0644
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
[software-parameter-section]
# libraries
caucase_jinja2_library = ${caucase-jinja2-library:target}
# profiles
profile_caddy_frontend = ${profile-caddy-frontend:target}
profile_caddy_replicate = ${profile-caddy-replicate:target}
profile_common = ${profile-common:rendered}
profile_kedifa = ${profile-kedifa:target}
profile_logrotate_base = ${template-logrotate-base:rendered}
profile_monitor = ${monitor-template:output}
profile_monitor2 = ${monitor2-template:rendered}
profile_replicate_publish_slave_information = ${profile-replicate-publish-slave-information:target}
profile_slave_list = ${profile-slave-list:target}
# templates
template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
template_caddy_frontend_configuration = ${profile-caddy-frontend-configuration:target}
template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
template_configuration_state_script = ${template-configuration-state-script:target}
template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
template_empty = ${template-empty:target}
template_graceful_script = ${template-graceful-script:target}
template_log_access = ${template-log-access:target}
template_not_found_html = ${template-not-found-html:target}
template_rotate_script = ${template-rotate-script:target}
template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
template_trafficserver_records_config = ${template-trafficserver-records-config:target}
template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
template_validate_script = ${template-validate-script:target}
template_wrapper = ${template-wrapper:output}
caddy_backend_url_validator = ${caddy-backend-url-validator:output}
# directories
bin_directory = ${buildout:bin-directory}
# files
sixtunnel = ${6tunnel:location}
nginx = ${nginx-output:nginx}
nginx_mime = ${nginx-output:mime}
caddy = ${caddy:output}
haproxy_executable = ${haproxy:location}/sbin/haproxy
rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
curl = ${curl:location}
dash = ${dash:location}
gzip = ${gzip:location}
logrotate = ${logrotate:location}
openssl = ${openssl:location}/bin/openssl
openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
trafficserver = ${trafficserver:location}
sha256sum = ${coreutils:location}/bin/sha256sum
kedifa = ${:bin_directory}/kedifa
kedifa-updater = ${:bin_directory}/kedifa-updater
kedifa-csr = ${:bin_directory}/kedifa-csr
xz_location = ${xz-utils:location}
htpasswd = ${:bin_directory}/htpasswd
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
context =
section software_parameter_dict software-parameter-section
[profile-caddy-frontend]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
mode = 0644
[caddy-backend-url-validator]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/caddy-backend-url-validator
mode = 0750
[profile-caddy-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
mode = 0644
[profile-kedifa]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-kedifa.cfg.in
mode = 0644
[download-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[profile-slave-list]
<=download-template
[profile-replicate-publish-slave-information]
<=download-template
[profile-caddy-frontend-configuration]
<=download-template
[template-not-found-html]
<=download-template
[template-default-slave-virtualhost]
<=download-template
[template-backend-haproxy-configuration]
<=download-template
[template-log-access]
<=download-template
[template-empty]
<=download-template
[template-slave-introspection-httpd-nginx]
<=download-template
[template-wrapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/wrapper.in
output = ${buildout:directory}/template-wrapper.cfg
mode = 0644
[template-trafficserver-records-config]
<=download-template
[template-trafficserver-storage-config]
<=download-template
[template-trafficserver-logging-yaml]
<=download-template
[template-rotate-script]
<=download-template
[template-caddy-lazy-script-call]
<=download-template
[template-graceful-script]
<=download-template
[template-validate-script]
<=download-template
[template-configuration-state-script]
<=download-template
[template-backend-haproxy-rsyslogd-conf]
<=download-template
[versions]
# Modern KeDiFa requires zc.lockfile
......
......@@ -4,21 +4,22 @@
{%- set backend_slave_list = [] %}
{%- set part_list = [] %}
{%- set cache_port = caddy_configuration.get('cache-port') %}
{%- set cache_access = "http://%s:%s" % (local_ipv4, cache_port) %}
{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) %}
{%- set backend_haproxy_http_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['http-port']) %}
{%- set backend_haproxy_https_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['https-port']) %}
{%- set cache_access = "http://%s:%s" % (instance_parameter_dict['ipv4-random'], cache_port) %}
{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (instance_parameter_dict['ipv4-random'], cache_port) %}
{%- set backend_haproxy_http_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['http-port']) %}
{%- set backend_haproxy_https_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['https-port']) %}
{%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': local_ipv4, 'http_port': http_port, 'https_port': https_port} %}
{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %}
{%- set slave_log_dict = {} %}
{%- if extra_slave_instance_list %}
{%- set slave_instance_information_list = [] %}
{%- set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) %}
{%- set slave_instance_information_list = [] %}
{%- set slave_instance_list = instance_parameter_dict['slave-instance-list'] %}
{%- if configuration['extra_slave_instance_list'] %}
{%- do slave_instance_list.extend(json_module.loads(configuration['extra_slave_instance_list'])) %}
{%- endif %}
{%- if master_key_download_url %}
{%- do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
{%- do kedifa_updater_mapping.append((master_key_download_url, caddy_configuration['master-certificate'], apache_certificate)) %}
{%- else %}
{%- do kedifa_updater_mapping.append(('notreadyyet', master_certificate, apache_certificate)) %}
{%- do kedifa_updater_mapping.append(('notreadyyet', caddy_configuration['master-certificate'], apache_certificate)) %}
{%- endif %}
{%- if kedifa_configuration['slave_kedifa_information'] %}
{%- set slave_kedifa_information = json_module.loads(kedifa_configuration['slave_kedifa_information']) %}
......@@ -30,7 +31,7 @@ recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
extra-context =
context =
raw common_profile {{ common_profile }}
raw profile_common {{ profile_common }}
${:extra-context}
# empty sections if no slaves are available
......@@ -53,7 +54,7 @@ context =
{%- if slave_ciphers %}
{%- set slave_cipher_list = ' '.join(slave_ciphers) %}
{%- else %}
{%- set slave_cipher_list = ciphers.strip() %}
{%- set slave_cipher_list = configuration['ciphers'].strip() %}
{%- endif %}
{%- do slave_instance.__setitem__('cipher_list', slave_cipher_list) %}
{#- Manage common instance parameters #}
......@@ -102,8 +103,8 @@ context =
{%- do part_list.extend([slave_logrotate_section, slave_section_title]) %}
{%- set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
{#- Pass HTTP2 switch #}
{%- do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
{%- do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
{%- do slave_instance.__setitem__('enable_http2_by_default', configuration['enable-http2-by-default']) %}
{%- do slave_instance.__setitem__('global_disable_http2', configuration['global-disable-http2']) %}
{#- Pass backend timeout values #}
{%- for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
{%- if slave_instance.get(key, '') == '' %}
......@@ -128,7 +129,7 @@ context =
{%- set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
{%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
{%- do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
{%- do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
{%- do slave_publish_dict.__setitem__('public-ipv4', configuration['public-ipv4']) %}
{%- do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %}
{#- Set slave domain if none was defined #}
{%- if slave_instance.get('custom_domain', None) == None %}
......@@ -176,9 +177,10 @@ bytes = 8
[{{ slave_htpasswd_section }}]
recipe = plone.recipe.command
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
command = {{ frontend_configuration['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
update-command = ${:command}
{#- ################################################## #}
......@@ -224,7 +226,7 @@ cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.ge
extra-context =
key content :cert-content
{%- else %}
{%- do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %}
{%- do kedifa_updater_mapping.append((key_download_url, certificate, caddy_configuration['master-certificate'])) %}
{%- endif %}
{#- BBB: SlapOS Master non-zero knowledge END #}
......@@ -233,9 +235,9 @@ extra-context =
[{{ slave_configuration_section_name }}]
certificate = {{ certificate }}
https_port = {{ dumps('' ~ https_port) }}
http_port = {{ dumps('' ~ http_port) }}
local_ipv4 = {{ dumps('' ~ local_ipv4) }}
https_port = {{ dumps('' ~ configuration['port']) }}
http_port = {{ dumps('' ~ configuration['plain_http_port']) }}
local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
{%- for key, value in slave_instance.iteritems() %}
{%- if value is not none %}
{{ key }} = {{ dumps('' ~ value) }}
......@@ -283,7 +285,7 @@ config-frequency = 720
{#- ############################### #}
{#- Publish Slave Information #}
{%- if not extra_slave_instance_list %}
{%- if not configuration['extra_slave_instance_list'] %}
{%- set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') %}
{%- do part_list.append(publish_section_title) %}
[{{ publish_section_title }}]
......@@ -315,36 +317,36 @@ recipe = slapos.cookbook:wrapper
ipv4 = ${slap-network-information:local-ipv4}
ipv6 = ${slap-network-information:global-ipv6}
wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
command-line = {{ sixtunnel_executable }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[tunnel-6to4-base-http_port]
<= tunnel-6to4-base
ipv4-port = {{ http_port }}
ipv6-port = {{ http_port }}
ipv4-port = {{ configuration['plain_http_port'] }}
ipv6-port = {{ configuration['plain_http_port'] }}
[tunnel-6to4-base-https_port]
<= tunnel-6to4-base
ipv4-port = {{ https_port }}
ipv6-port = {{ https_port }}
ipv4-port = {{ configuration['port'] }}
ipv6-port = {{ configuration['port'] }}
{#- Define log access #}
[caddy-log-access-parameters]
caddy_log_directory = {{ dumps(caddy_log_directory) }}
caddy_configuration_directory = {{ dumps(caddy_configuration_directory) }}
local_ipv4 = {{ dumps(local_ipv4) }}
local_ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
global_ipv6 = {{ dumps(global_ipv6) }}
https_port = {{ dumps(https_port) }}
http_port = {{ dumps(http_port) }}
https_port = {{ dumps(configuration['port']) }}
http_port = {{ dumps(configuration['plain_http_port']) }}
ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }}
access_log = {{ dumps(access_log) }}
error_log = {{ dumps(error_log) }}
not_found_file = {{ dumps(not_found_file) }}
access_log = {{ dumps(caddy_configuration['access-log']) }}
error_log = {{ dumps(caddy_configuration['error-log']) }}
not_found_file = {{ dumps(caddy_configuration['not-found-file']) }}
[caddy-log-access]
< = jinja2-template-base
template = {{frontend_configuration.get('template-log-access')}}
template = {{ software_parameter_dict['template_log_access'] }}
rendered = {{frontend_configuration.get('log-access-configuration')}}
extra-context =
section slave_log_directory slave-log-directory-dict
......@@ -352,11 +354,11 @@ extra-context =
section parameter_dict caddy-log-access-parameters
[slave-introspection-parameters]
local-ipv4 = {{ dumps(local_ipv4) }}
local-ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
global-ipv6 = {{ dumps(global_ipv6) }}
https-port = {{ frontend_configuration['slave-introspection-https-port'] }}
ip-access-certificate = {{ frontend_configuration.get('ip-access-certificate') }}
nginx-mime = {{ frontend_configuration['nginx_mime'] }}
nginx-mime = {{ software_parameter_dict['nginx_mime'] }}
access-log = {{ dumps(caddy_configuration['slave-introspection-access-log']) }}
error-log = {{ dumps(caddy_configuration['slave-introspection-error-log']) }}
var = {{ directory['slave-introspection-var'] }}
......@@ -364,7 +366,7 @@ pid = {{ caddy_configuration['slave-introspection-pid-file'] }}
[slave-introspection-config]
<= jinja2-template-base
template = {{ frontend_configuration['slave-introspection-template'] }}
template = {{ software_parameter_dict['template_slave_introspection_httpd_nginx'] }}
rendered = {{ frontend_configuration['slave-introspection-configuration'] }}
extra-context =
section slave_log_directory slave-log-directory-dict
......@@ -373,7 +375,7 @@ extra-context =
[slave-introspection]
recipe = slapos.cookbook:wrapper
command-line = {{ frontend_configuration['nginx'] }}
command-line = {{ software_parameter_dict['nginx'] }}
-c ${slave-introspection-config:rendered}
wrapper-path = {{ directory['service'] }}/slave-instrospection-nginx
......@@ -384,9 +386,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{#- Publish information for the instance #}
[publish-caddy-information]
recipe = slapos.cookbook:publish.serialised
public-ipv4 = {{ public_ipv4 }}
private-ipv4 = {{ local_ipv4 }}
{%- if extra_slave_instance_list %}
public-ipv4 = {{ configuration['public-ipv4'] }}
private-ipv4 = {{ instance_parameter_dict['ipv4-random'] }}
{%- if configuration['extra_slave_instance_list'] %}
{#- sort_keys are important in order to avoid shuffling parameters on each run #}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list, sort_keys=True) }}
{%- endif %}
......@@ -404,11 +406,11 @@ backend-haproxy-statistic-url = {{ statistic_url }}
[kedifa-updater]
recipe = slapos.cookbook:wrapper
command-line = {{ kedifa_configuration['kedifa-updater'] }}
command-line = {{ software_parameter_dict['kedifa-updater'] }}
--server-ca-certificate {{ kedifa_configuration['ca-certificate'] }}
--identity {{ kedifa_configuration['certificate'] }}
--master-certificate {{ master_certificate }}
--on-update "{{ frontend_graceful_reload }}"
--master-certificate {{ caddy_configuration['master-certificate'] }}
--on-update "{{ caddy_configuration['frontend-graceful-command'] }}"
${kedifa-updater-mapping:file}
{{ kedifa_configuration['kedifa-updater-state-file'] }}
......@@ -417,8 +419,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[kedifa-updater-run]
recipe = plone.recipe.command
{#- Can be stopped on error, as does not rely on self provided service but on service which comes from another partition #}
stop-on-error = True
command = {{ kedifa_configuration['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ frontend_graceful_reload }}"
command = {{ software_parameter_dict['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ caddy_configuration['frontend-graceful-command'] }}"
update-command = ${:command}
[kedifa-updater-mapping]
......@@ -452,7 +455,7 @@ extra-context =
{%- for key, value in backend_haproxy_configuration.items() %}
{{ key }} = {{ value }}
{%- endfor %}
local-ipv4 = {{ dumps('' ~ local_ipv4) }}
local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
global-ipv6 = ${slap-network-information:global-ipv6}
request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
......@@ -467,7 +470,7 @@ csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ bin_directory }}/caucase \
{{ software_parameter_dict['bin_directory'] }}/caucase \
--ca-url {{ backend_haproxy_configuration['caucase-url'] }} \
--ca-crt {{ backend_haproxy_configuration['cas-ca-certificate'] }} \
--crl {{ backend_haproxy_configuration['crl'] }} \
......@@ -479,9 +482,9 @@ command =
[buildout]
extends =
{{ common_profile }}
{{ logrotate_base_instance }}
{{ monitor_template }}
{{ profile_common }}
{{ profile_logrotate_base }}
{{ profile_monitor }}
parts +=
kedifa-updater
......@@ -511,7 +514,7 @@ csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
stop-on-error = False
update-command = ${:command}
command =
{{ bin_directory }}/caucase \
{{ software_parameter_dict['bin_directory'] }}/caucase \
--ca-url {{ kedifa_configuration['caucase-url'] }} \
--ca-crt {{ kedifa_configuration['cas-ca-certificate'] }} \
--crl {{ kedifa_configuration['crl'] }} \
......@@ -524,6 +527,7 @@ recipe = plone.recipe.command
certificate = {{ directory['caddy-csr_id'] }}/certificate.pem
key = {{ directory['caddy-csr_id'] }}/key.pem
{#- Can be stopped on error, as does not rely on self provided service #}
stop-on-error = True
update-command = ${:command}
command =
......@@ -563,7 +567,7 @@ depends =
${store-csr_id:command}
${store-backend-haproxy-csr_id:command}
recipe = slapos.cookbook:wrapper
command-line = {{ caddy_executable }}
command-line = {{ software_parameter_dict['caddy'] }}
-conf ${expose-csr_id-template:rendered}
-log ${expose-csr_id-configuration:error-log}
-http2=true
......
......@@ -16,7 +16,7 @@ $Umask 0022
$WorkDirectory {{ configuration['spool-directory'] }}
# Setup logging per slave, by extracting the slave name from the log stream
{%- set regex = ".*-backend (.*)-http.*" %}
{%- set regex = ".*-backend (.*)-http.{0,1}/" %}
template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
set $!slave_name = exec_template("extract_slave_name");
template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")
......
......@@ -14,27 +14,30 @@ defaults
timeout connect {{ configuration['backend-connect-timeout'] }}s
retries {{ configuration['backend-connect-retries'] }}
{%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
{%- macro frontend_entry(slave_instance, scheme, wildcard) %}
{#- wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
{%- set host_list = (slave_instance.get('server-alias') or '').split() %}
{%- if slave_instance.get('custom_domain') not in host_list %}
{%- do host_list.append(slave_instance.get('custom_domain')) %}
{%- endif %}
{%- set matched = {'count': 0} %}
{%- for host in host_list %}
{#- Match up to the end or optional port (starting with ':') #}
{#- Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
{%- if wildcard and host.startswith('*.') %}
{%- do matched.__setitem__('count', matched['count'] + 1) %}
{%- if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] %}
{%- set host_list = (slave_instance.get('server-alias') or '').split() %}
{%- if slave_instance.get('custom_domain') not in host_list %}
{%- do host_list.append(slave_instance.get('custom_domain')) %}
{%- endif %}
{%- set matched = {'count': 0} %}
{%- for host in host_list %}
{#- Match up to the end or optional port (starting with ':') #}
{#- Please note that this matching is quite sensitive to changes and hard to test, so avoid needless changes #}
{%- if wildcard and host.startswith('*.') %}
{%- do matched.__setitem__('count', matched['count'] + 1) %}
# match wildcard {{ host }}
acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i {{ host[2:] }}($|:.*)
{%- elif not wildcard and not host.startswith('*.') %}
{%- do matched.__setitem__('count', matched['count'] + 1) %}
{%- elif not wildcard and not host.startswith('*.') %}
{%- do matched.__setitem__('count', matched['count'] + 1) %}
acl is_{{ slave_instance['slave_reference'] }} hdr_reg(host) -i ^{{ host }}($|:.*)
{%- endif %}
{%- endfor %}
{%- if matched['count'] > 0 %}
{%- endif %}
{%- endfor %}
{%- if matched['count'] > 0 %}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
{%- endif %}
{%- endif %}
{%- endmacro %}
......@@ -49,59 +52,58 @@ frontend statistic
frontend http-backend
bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
{%- for slave_instance in backend_slave_list %}
{%- for slave_instance in backend_slave_list -%}
{{ frontend_entry(slave_instance, 'http', False) }}
{%- endfor %}
{%- for slave_instance in backend_slave_list %}
{%- for slave_instance in backend_slave_list -%}
{{ frontend_entry(slave_instance, 'http', True) }}
{%- endfor %}
frontend https-backend
bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
{%- for slave_instance in backend_slave_list %}
{%- for slave_instance in backend_slave_list -%}
{{ frontend_entry(slave_instance, 'https', False) }}
{%- endfor %}
{%- for slave_instance in backend_slave_list %}
{%- for slave_instance in backend_slave_list -%}
{{ frontend_entry(slave_instance, 'https', True) }}
{%- endfor %}
{%- for slave_instance in backend_slave_list %}
{%- for (scheme, prefix) in [('http', 'http_backend'), ('https', 'https_backend')] %}
{%- for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() %}
{%- set info_dict = slave_instance[prefix] %}
{%- if info_dict['scheme'] == 'https' %}
{%- set ssl = [] %}
{%- if slave_instance['authenticate-to-backend'] %}
{%- set ssl = ['crt %s' % (configuration['certificate'],)] %}
{%- endif %}
{%- do ssl.append('ssl verify') %}
{%- set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
{%- if slave_instance['ssl_proxy_verify'] %}
{%- if path_to_ssl_proxy_ca_crt %}
{%- do ssl.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
{%- if info_dict['hostname'] and info_dict['port'] %}
{%- set ssl_list = [] %}
{%- if info_dict['scheme'] == 'https' %}
{%- if slave_instance['authenticate-to-backend'] %}
{%- do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
{%- endif %}
{%- do ssl_list.append('ssl verify') %}
{%- set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
{%- if slave_instance['ssl_proxy_verify'] %}
{%- if path_to_ssl_proxy_ca_crt %}
{%- do ssl_list.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
{%- else %}
{#- Backend SSL shall be verified, but not CA provided, disallow connection #}
{#- Simply dropping hostname from the dict will result with ignoring it... #}
{%- do info_dict.__setitem__('hostname', '') %}
{%- endif %}
{%- else %}
{#- Backend SSL shall be verified, but not CA provided, disallow connection #}
{#- Simply dropping hostname from the dict will result with ignoring it... #}
{%- do info_dict.__setitem__('hostname', '') %}
{%- do ssl_list.append('none') %}
{%- endif %}
{%- else %}
{%- do ssl.append('none') %}
{%- endif %}
{%- set ssl = ' '.join(ssl) %}
{%- else %}
{%- set ssl = '' %}
{%- endif %}
backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- set hostname = info_dict['hostname'] %}
{%- set port = info_dict['port'] %}
{%- set path = info_dict['path'].rstrip('/') %}
{%- if hostname and port %}
{%- set hostname = info_dict['hostname'] %}
{%- set port = info_dict['port'] %}
{%- set path = info_dict['path'].rstrip('/') %}
{%- if hostname and port %}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server backend {{ hostname }}:{{ port }} {{ ssl }}
{%- if path %}
server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
{%- endif %}
{%- endif %}
{%- endfor %}
......
......@@ -75,7 +75,7 @@ log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{% endfor %}
[buildout]
extends = {{ common_profile }}
extends = {{ profile_common }}
parts =
{% for part in part_list %}
{{ ' %s' % part }}
......
......@@ -48,7 +48,6 @@ from slapos.recipe.librecipe import generateHashFromFiles
import xml.etree.ElementTree as ET
import urlparse
import socket
import sqlite3
try:
......@@ -358,7 +357,7 @@ class TestDataMixin(object):
[backend_haproxy_wrapper_path] + hash_file_list
)
for rejected_slave_publish_path in glob.glob(os.path.join(
self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave')):
self.instance_path, '*', 'etc', 'nginx-rejected-slave.conf')):
partition_id = rejected_slave_publish_path.split('/')[-3]
rejected_slave_pem_path = os.path.join(
self.instance_path, partition_id, 'etc', 'rejected-slave.pem')
......@@ -1772,7 +1771,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+ ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2}.\d{3}\] ' \
r'http-backend _Url-http\/backend ' \
r'http-backend _Url-http\/_Url-backend ' \
r'\d+/\d+\/\d+\/\d+\/\d+ ' \
r'200 \d+ - - ---- ' \
r'\d\/\d\/\d\/\d\/\d \d\/\d ' \
......@@ -1801,15 +1800,18 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
with open(backend_configuration_file) as fh:
content = fh.read()
self.assertTrue("""backend _Url-http
self.assertIn("""backend _Url-http
timeout server 12s
timeout connect 5s
retries 3""" in content)
self.assertTrue(""" timeout queue 60s
retries 3""", content)
self.assertIn(""" timeout queue 60s
timeout server 12s
timeout client 12s
timeout connect 5s
retries 3""" in content)
retries 3""", content)
# check that no needless entries are generated
self.assertIn("backend _Url-http\n", content)
self.assertNotIn("backend _Url-https\n", content)
def test_auth_to_backend(self):
parameter_dict = self.assertSlaveBase('auth-to-backend')
......@@ -6787,14 +6789,34 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
def test(self):
self.instance_parameter_dict.update({
# master partition parameters
'-frontend-quantity': 3,
'-sla-2-computer_guid': self.slap._computer_id,
'-sla-3-computer_guid': self.slap._computer_id,
'-frontend-2-state': 'stopped',
'-frontend-2-software-release-url': self.frontend_2_sr,
'-sla-3-computer_guid': self.slap._computer_id,
'-frontend-3-state': 'stopped',
'-frontend-3-software-release-url': self.frontend_3_sr,
'-kedifa-software-release-url': self.kedifa_sr,
'automatic-internal-kedifa-caucase-csr': False,
'automatic-internal-backend-client-caucase-csr': False,
# all nodes partition parameters
'apache-certificate': self.certificate_pem,
'apache-key': self.key_pem,
'domain': 'example.com',
'enable-http2-by-default': True,
'global-disable-http2': True,
'mpm-graceful-shutdown-timeout': 2,
'public-ipv4': '255.255.255.255',
're6st-verification-url': 're6st-verification-url',
'backend-connect-timeout': 2,
'backend-connect-retries': 1,
'ciphers': 'ciphers',
'request-timeout': 100,
'authenticate-to-backend': True,
# specific parameters
'-frontend-config-1-ram-cache-size': '512K',
'-frontend-config-2-ram-cache-size': '256K',
})
# re-request instance with updated parameters
......@@ -6806,43 +6828,196 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
except Exception:
pass
# inspect slapproxy, that the master correctly requested other partitions
sqlitedb_file = os.path.join(
os.path.abspath(
os.path.join(
self.slap.instance_directory, os.pardir
)
), 'var', 'proxy.db'
)
connection = sqlite3.connect(sqlitedb_file)
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
connection.row_factory = dict_factory
cursor = connection.cursor()
cursor.execute(
"select partition_reference, software_release "
"from partition14 where slap_state='busy';")
requested_partition_information = cursor.fetchall()
computer = self.slap._slap.registerComputer('local')
# state of parameters of all instances
partition_parameter_dict_dict = {}
for partition in computer.getComputerPartitionList():
if partition.getState() == 'destroyed':
continue
parameter_dict = partition.getInstanceParameterDict()
instance_title = parameter_dict['instance_title']
if '_' in parameter_dict:
# "flatten" the instance parameter
parameter_dict = json.loads(parameter_dict['_'])
partition_parameter_dict_dict[instance_title] = parameter_dict
parameter_dict[
'X-software_release_url'] = partition.getSoftwareRelease().getURI()
base_software_url = self.getSoftwareURL()
# drop some very varying parameters
def assertKeyWithPop(d, k):
self.assertIn(k, d)
d.pop(k)
assertKeyWithPop(
partition_parameter_dict_dict['caddy-frontend-1'],
'master-key-download-url')
assertKeyWithPop(
partition_parameter_dict_dict['caddy-frontend-2'],
'master-key-download-url')
assertKeyWithPop(
partition_parameter_dict_dict['caddy-frontend-3'],
'master-key-download-url')
assertKeyWithPop(
partition_parameter_dict_dict['testing partition 0'],
'timestamp')
assertKeyWithPop(
partition_parameter_dict_dict['testing partition 0'],
'ip_list')
monitor_password = partition_parameter_dict_dict[
'caddy-frontend-1'].pop('monitor-password')
self.assertEqual(
monitor_password,
partition_parameter_dict_dict[
'caddy-frontend-2'].pop('monitor-password')
)
self.assertEqual(
monitor_password,
partition_parameter_dict_dict[
'caddy-frontend-3'].pop('monitor-password')
)
self.assertEqual(
monitor_password,
partition_parameter_dict_dict[
'kedifa'].pop('monitor-password')
)
backend_client_caucase_url = u'http://[%s]:8990' % (self._ipv6_address,)
kedifa_caucase_url = u'http://[%s]:15090' % (self._ipv6_address,)
expected_partition_parameter_dict_dict = {
'caddy-frontend-1': {
'X-software_release_url': base_software_url,
u'apache-certificate': unicode(self.certificate_pem),
u'apache-key': unicode(self.key_pem),
u'authenticate-to-backend': u'True',
u'backend-client-caucase-url': backend_client_caucase_url,
u'backend-connect-retries': u'1',
u'backend-connect-timeout': u'2',
u'ciphers': u'ciphers',
u'cluster-identification': u'testing partition 0',
u'domain': u'example.com',
u'enable-http2-by-default': u'True',
u'extra_slave_instance_list': u'[]',
u'frontend-name': u'caddy-frontend-1',
u'global-disable-http2': u'True',
u'kedifa-caucase-url': kedifa_caucase_url,
u'monitor-cors-domains': u'monitor.app.officejs.com',
u'monitor-httpd-port': 8411,
u'monitor-username': u'admin',
u'mpm-graceful-shutdown-timeout': u'2',
u'plain_http_port': '11080',
u'port': '11443',
u'public-ipv4': u'255.255.255.255',
u'ram-cache-size': u'512K',
u're6st-verification-url': u're6st-verification-url',
u'request-timeout': u'100',
u'slave-kedifa-information': u'{}'
},
'caddy-frontend-2': {
'X-software_release_url': self.frontend_2_sr,
u'apache-certificate': unicode(self.certificate_pem),
u'apache-key': unicode(self.key_pem),
u'authenticate-to-backend': u'True',
u'backend-client-caucase-url': backend_client_caucase_url,
u'backend-connect-retries': u'1',
u'backend-connect-timeout': u'2',
u'ciphers': u'ciphers',
u'cluster-identification': u'testing partition 0',
u'domain': u'example.com',
u'enable-http2-by-default': u'True',
u'extra_slave_instance_list': u'[]',
u'frontend-name': u'caddy-frontend-2',
u'global-disable-http2': u'True',
u'kedifa-caucase-url': kedifa_caucase_url,
u'monitor-cors-domains': u'monitor.app.officejs.com',
u'monitor-httpd-port': 8412,
u'monitor-username': u'admin',
u'mpm-graceful-shutdown-timeout': u'2',
u'plain_http_port': u'11080',
u'port': u'11443',
u'public-ipv4': u'255.255.255.255',
u'ram-cache-size': u'256K',
u're6st-verification-url': u're6st-verification-url',
u'request-timeout': u'100',
u'slave-kedifa-information': u'{}'
},
'caddy-frontend-3': {
'X-software_release_url': self.frontend_3_sr,
u'apache-certificate': unicode(self.certificate_pem),
u'apache-key': unicode(self.key_pem),
u'authenticate-to-backend': u'True',
u'backend-client-caucase-url': backend_client_caucase_url,
u'backend-connect-retries': u'1',
u'backend-connect-timeout': u'2',
u'ciphers': u'ciphers',
u'cluster-identification': u'testing partition 0',
u'domain': u'example.com',
u'enable-http2-by-default': u'True',
u'extra_slave_instance_list': u'[]',
u'frontend-name': u'caddy-frontend-3',
u'global-disable-http2': u'True',
u'kedifa-caucase-url': kedifa_caucase_url,
u'monitor-cors-domains': u'monitor.app.officejs.com',
u'monitor-httpd-port': 8413,
u'monitor-username': u'admin',
u'mpm-graceful-shutdown-timeout': u'2',
u'plain_http_port': u'11080',
u'port': u'11443',
u'public-ipv4': u'255.255.255.255',
u're6st-verification-url': u're6st-verification-url',
u'request-timeout': u'100',
u'slave-kedifa-information': u'{}'
},
'kedifa': {
'X-software_release_url': self.kedifa_sr,
u'caucase_port': u'15090',
u'cluster-identification': u'testing partition 0',
u'kedifa_port': u'15080',
u'monitor-cors-domains': u'monitor.app.officejs.com',
u'monitor-httpd-port': u'8402',
u'monitor-username': u'admin',
u'slave-list': []
},
'testing partition 0': {
'-frontend-2-software-release-url': self.frontend_2_sr,
'-frontend-2-state': 'stopped',
'-frontend-3-software-release-url': self.frontend_3_sr,
'-frontend-3-state': 'stopped',
'-frontend-config-1-ram-cache-size': '512K',
'-frontend-config-2-ram-cache-size': '256K',
'-frontend-quantity': '3',
'-kedifa-software-release-url': self.kedifa_sr,
'-sla-2-computer_guid': 'local',
'-sla-3-computer_guid': 'local',
'X-software_release_url': base_software_url,
'apache-certificate': unicode(self.certificate_pem),
'apache-key': unicode(self.key_pem),
'authenticate-to-backend': 'True',
'automatic-internal-backend-client-caucase-csr': 'False',
'automatic-internal-kedifa-caucase-csr': 'False',
'backend-connect-retries': '1',
'backend-connect-timeout': '2',
'caucase_port': '15090',
'ciphers': 'ciphers',
'domain': 'example.com',
'enable-http2-by-default': 'True',
'full_address_list': [],
'global-disable-http2': 'True',
'instance_title': 'testing partition 0',
'kedifa_port': '15080',
'mpm-graceful-shutdown-timeout': '2',
'plain_http_port': '11080',
'port': '11443',
'public-ipv4': '255.255.255.255',
're6st-verification-url': 're6st-verification-url',
'request-timeout': '100',
'root_instance_title': 'testing partition 0',
'slap_software_type': 'RootSoftwareInstance',
'slave_instance_list': []
}
}
self.assertEqual(
requested_partition_information,
[
{'software_release': base_software_url,
'partition_reference': 'testing partition 0'},
{'software_release': self.kedifa_sr,
'partition_reference': 'kedifa'},
# that one is base, as expected
{'software_release': base_software_url,
'partition_reference': 'caddy-frontend-1'},
{'software_release': self.frontend_2_sr,
'partition_reference': 'caddy-frontend-2'},
{'software_release': self.frontend_3_sr,
'partition_reference': 'caddy-frontend-3'}]
expected_partition_parameter_dict_dict,
partition_parameter_dict_dict
)
......@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
......
......@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
......
......@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
......
......@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
T-2/var/log/httpd/_dummy-cached_backend_log
T-2/var/log/httpd/_dummy-cached_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
T-2/var/log/httpd/_enable-http2-default_error_log
T-2/var/log/httpd/_enable-http2-false_access_log
T-2/var/log/httpd/_enable-http2-false_backend_log
T-2/var/log/httpd/_enable-http2-false_error_log
T-2/var/log/httpd/_enable-http2-true_access_log
T-2/var/log/httpd/_enable-http2-true_backend_log
T-2/var/log/httpd/_enable-http2-true_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
......
......@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_backend_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
T-2/var/log/httpd/_custom_domain_backend_log
......@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
T-2/var/log/httpd/_disabled-cookie-list_backend_log
T-2/var/log/httpd/_disabled-cookie-list_error_log
T-2/var/log/httpd/_empty_access_log
T-2/var/log/httpd/_empty_backend_log
T-2/var/log/httpd/_empty_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
......@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_backend_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
T-2/var/log/httpd/_monitor-ipv6-test_backend_log
T-2/var/log/httpd/_monitor-ipv6-test_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log
......
......@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
T-2/var/log/httpd/_auth-to-backend_backend_log
T-2/var/log/httpd/_auth-to-backend_error_log
T-2/var/log/httpd/_ciphers_access_log
T-2/var/log/httpd/_ciphers_backend_log
T-2/var/log/httpd/_ciphers_error_log
T-2/var/log/httpd/_custom_domain_access_log
T-2/var/log/httpd/_custom_domain_backend_log
......@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
T-2/var/log/httpd/_disabled-cookie-list_backend_log
T-2/var/log/httpd/_disabled-cookie-list_error_log
T-2/var/log/httpd/_empty_access_log
T-2/var/log/httpd/_empty_backend_log
T-2/var/log/httpd/_empty_error_log
T-2/var/log/httpd/_enable-http2-default_access_log
T-2/var/log/httpd/_enable-http2-default_backend_log
......@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
T-2/var/log/httpd/_https-only_backend_log
T-2/var/log/httpd/_https-only_error_log
T-2/var/log/httpd/_monitor-ipv4-test_access_log
T-2/var/log/httpd/_monitor-ipv4-test_backend_log
T-2/var/log/httpd/_monitor-ipv4-test_error_log
T-2/var/log/httpd/_monitor-ipv6-test_access_log
T-2/var/log/httpd/_monitor-ipv6-test_backend_log
T-2/var/log/httpd/_monitor-ipv6-test_error_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment