{%- if software_type == slap_software_type %} {%- set kedifa_updater_mapping = [] %} {%- set cached_server_dict = {} %} {%- set backend_slave_list = [] %} {%- set part_list = [] %} {%- set cache_port = caddy_configuration.get('cache-port') %} {%- set cache_access = "http://%s:%s" % (local_ipv4, cache_port) %} {%- set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) %} {%- set backend_haproxy_http_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['http-port']) %} {%- set backend_haproxy_https_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['https-port']) %} {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %} {%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': local_ipv4, 'http_port': http_port, 'https_port': https_port} %} {%- set slave_log_dict = {} %} {%- if extra_slave_instance_list %} {%- set slave_instance_information_list = [] %} {%- set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) %} {%- endif %} {%- if master_key_download_url %} {%- do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %} {%- else %} {%- do kedifa_updater_mapping.append(('notreadyyet', master_certificate, apache_certificate)) %} {%- endif %} {%- if kedifa_configuration['slave_kedifa_information'] %} {%- set slave_kedifa_information = json_module.loads(kedifa_configuration['slave_kedifa_information']) %} {%- else %} {%- set slave_kedifa_information = {} %} {%- endif -%} [jinja2-template-base] recipe = slapos.recipe.template:jinja2 extensions = jinja2.ext.do extra-context = context = raw common_profile {{ common_profile }} ${:extra-context} # empty sections if no slaves are available [slave-log-directory-dict] [slave-password] {#- Loop thought slave list to set up slaves #} {%- set DEFAULT_PORT = {'http': 80, 'https': 443, '': None} %} {%- for slave_instance in slave_instance_list %} {#- prepare backend parameters #} {%- for key, prefix in [('url', 'http_backend'), ('https-url', 'https_backend')] %} {%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %} {%- set info_dict = {'scheme': parsed.scheme, 'hostname': parsed.hostname, 'port': parsed.port or DEFAULT_PORT[parsed.scheme], 'path': parsed.path, 'fragment': parsed.fragment} %} {%- do slave_instance.__setitem__(prefix, info_dict) %} {%- endfor %} {%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %} {#- Manage ciphers #} {%- set slave_ciphers = slave_instance.get('ciphers', '').strip().split() %} {%- if slave_ciphers %} {%- set slave_cipher_list = ' '.join(slave_ciphers) %} {%- else %} {%- set slave_cipher_list = ciphers.strip() %} {%- endif %} {%- do slave_instance.__setitem__('cipher_list', slave_cipher_list) %} {#- Manage common instance parameters #} {%- set slave_type = slave_instance.get('type', '') %} {%- set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_type != 'redirect') %} {%- set slave_reference = slave_instance.get('slave_reference') %} {%- set slave_kedifa = slave_kedifa_information.get(slave_reference) %} {#- Setup backend URLs for front facing Caddy #} {%- if slave_type == 'redirect' %} {%- do slave_instance.__setitem__('backend-http-url', slave_instance.get('url', '').rstrip('/')) %} {%- if slave_instance.get('https-url') %} {%- do slave_instance.__setitem__('backend-https-url', slave_instance.get['https-url'].rstrip('/')) %} {%- endif %} {%- elif enable_cache %} {%- if 'domain' in slave_instance %} {%- if not slave_instance.get('custom_domain') %} {%- do slave_instance.__setitem__('custom_domain', slave_instance.get('domain')) %} {%- endif %} {%- endif %} {%- do slave_instance.__setitem__('backend-http-url', cache_access) %} {%- if slave_instance.get('https-url') %} {%- do slave_instance.__setitem__('backend-https-url', ssl_cache_access) %} {%- endif %} {%- do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) %} {%- else %} {%- do slave_instance.__setitem__('backend-http-url', backend_haproxy_http_url) %} {%- if slave_instance.get('https-url') %} {%- do slave_instance.__setitem__('backend-https-url', backend_haproxy_https_url) %} {%- endif %} {%- endif %} {%- if slave_kedifa %} {%- set key_download_url = slave_kedifa.get('key-download-url') %} {%- else %} {%- set key_download_url = 'notreadyyet' %} {%- endif %} {%- set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference %} {%- set slave_parameter_dict = generic_instance_parameter_dict.copy() %} {%- set slave_publish_dict = {} %} {%- set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %} {%- set slave_logrotate_section = slave_reference + "-logs" %} {%- set slave_password_section = slave_reference + "-password" %} {%- set slave_ln_section = slave_reference + "-ln" %} {#- extend parts #} {%- do part_list.extend([slave_ln_section]) %} {%- do part_list.extend([slave_logrotate_section, slave_section_title]) %} {%- set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %} {#- Pass HTTP2 switch #} {%- do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %} {%- do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %} {#- Pass backend timeout values #} {%- for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %} {%- if slave_instance.get(key, '') == '' %} {%- do slave_instance.__setitem__(key, configuration[key]) %} {%- endif %} {%- endfor %} {%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %} {#- Set Up log files #} {%- do slave_parameter_dict.__setitem__('access_log', '/'.join([caddy_log_directory, '%s_access_log' % slave_reference])) %} {%- do slave_parameter_dict.__setitem__('error_log', '/'.join([caddy_log_directory, '%s_error_log' % slave_reference])) %} {%- do slave_instance.__setitem__('access_log', slave_parameter_dict.get('access_log')) %} {%- do slave_instance.__setitem__('error_log', slave_parameter_dict.get('error_log')) %} {#- Add slave log directory to the slave log access dict #} {%- do slave_log_dict.__setitem__(slave_reference, slave_log_folder) %} {%- set slave_log_access_url = 'https://' + slave_reference.lower() + ':${'+ slave_password_section +':passwd}@[' + frontend_configuration.get('caddy-ipv6') + ']:' + frontend_configuration.get('caddy-https-port') + '/' + slave_reference.lower() + '/' %} {%- do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %} {%- do slave_publish_dict.__setitem__('slave-reference', slave_reference) %} {%- do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %} {%- do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %} {#- Set slave domain if none was defined #} {%- if slave_instance.get('custom_domain', None) == None %} {%- set domain_prefix = slave_instance.get('slave_reference').replace("-", "").replace("_", "").lower() %} {%- do slave_instance.__setitem__('custom_domain', "%s.%s" % (domain_prefix, slapparameter_dict.get('domain'))) %} {%- endif %} {%- do slave_publish_dict.__setitem__('domain', slave_instance.get('custom_domain')) %} {%- do slave_publish_dict.__setitem__('url', "http://%s" % slave_instance.get('custom_domain')) %} {%- do slave_publish_dict.__setitem__('site_url', "http://%s" % slave_instance.get('custom_domain')) %} {%- do slave_publish_dict.__setitem__('secure_access', 'https://%s' % slave_instance.get('custom_domain')) %} [slave-log-directory-dict] {{slave_reference}} = {{ slave_log_folder }} [slave-password] {{ slave_reference }} = {{ '${' + slave_password_section + ':passwd}' }} {#- Set slave logrotate entry #} [{{slave_logrotate_section}}] <= logrotate-entry-base name = ${:_buildout_section_name_} log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}} backup = {{ slave_log_folder }} {#- integrate current logs inside #} [{{slave_ln_section}}] recipe = plone.recipe.command stop-on-error = false update-command = ${:command} command = ln -sf {{slave_parameter_dict.get('error_log')}} {{ slave_log_folder }}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} {{ slave_log_folder }}/access.log {#- Set password for slave #} [{{slave_password_section}}] recipe = slapos.cookbook:generate.password storage-path = {{caddy_configuration_directory}}/.{{slave_reference}}.passwd bytes = 8 {#- ################################################## #} {#- Set Slave Certificates if needed #} {#- Set certificate key for custom configuration #} {%- set cert_name = slave_reference.replace('-','.') + '.pem' %} {%- set certificate = '%s/%s' % (autocert, cert_name) %} {%- do slave_parameter_dict.__setitem__('certificate', certificate )%} {#- Set ssl certificates for each slave #} {%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%} {%- if cert_name in slave_instance %} {%- set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %} {%- set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %} {%- do part_list.append(cert_title) %} {%- do slave_parameter_dict.__setitem__(cert_name, cert_file) %} {%- do slave_instance.__setitem__('path_to_' + cert_name, cert_file) %} {#- Store certificates on fs #} [{{ cert_title }}] < = jinja2-template-base template = {{ empty_template }} rendered = {{ cert_file }} extra-context = key content {{ cert_title + '-config:value' }} {#- BBB: SlapOS Master non-zero knowledge BEGIN #} {#- Store certificate in config #} [{{ cert_title + '-config' }}] value = {{ dumps(slave_instance.get(cert_name)) }} {%- endif %} {%- endfor %} {#- Set Up Certs #} {%- if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance %} {%- set cert_title = '%s-crt' % (slave_reference) %} {%- set cert_file = '/'.join([directory['bbb-ssl-dir'], cert_title.replace('-','.')]) %} {%- do kedifa_updater_mapping.append((key_download_url, certificate, cert_file)) %} {%- do part_list.append(cert_title) %} {%- do slave_parameter_dict.__setitem__("ssl_crt", cert_file) %} [{{cert_title}}] < = jinja2-template-base template = {{ empty_template }} rendered = {{ cert_file }} cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.get('ssl_ca_crt', '') + '\n' + slave_instance.get('ssl_key')) }} extra-context = key content :cert-content {%- else %} {%- do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %} {%- endif %} {#- BBB: SlapOS Master non-zero knowledge END #} {#- ########################################## #} {#- Set Slave Configuration #} [{{ slave_configuration_section_name }}] certificate = {{ certificate }} https_port = {{ dumps('' ~ https_port) }} http_port = {{ dumps('' ~ http_port) }} local_ipv4 = {{ dumps('' ~ local_ipv4) }} {%- for key, value in slave_instance.iteritems() %} {%- if value is not none %} {{ key }} = {{ dumps('' ~ value) }} {%- endif %} {%- endfor %} [{{ slave_section_title }}] < = jinja2-template-base rendered = {{ caddy_configuration_directory }}/${:filename} template = {{ template_default_slave_configuration }} extra-context = section slave_parameter {{ slave_configuration_section_name }} import urllib_module urllib filename = {{ '%s.conf' % slave_reference }} {{ '\n' }} {%- set monitor_ipv6_test = slave_instance.get('monitor-ipv6-test', '') %} {%- if monitor_ipv6_test %} {%- set monitor_ipv6_section_title = 'check-%s-ipv6-packet-list-test' % slave_instance.get('slave_reference') %} {%- do part_list.append(monitor_ipv6_section_title) %} [{{ monitor_ipv6_section_title }}] <= monitor-promise-base module = check_icmp_packet_lost name = {{ monitor_ipv6_section_title }}.py config-address = {{ dumps(monitor_ipv6_test) }} # promise frequency in minutes (2 times/day) config-frequency = 720 {%- endif %} {%- set monitor_ipv4_test = slave_instance.get('monitor-ipv4-test', '') %} {%- if monitor_ipv4_test %} {%- set monitor_ipv4_section_title = 'check-%s-ipv4-packet-list-test' % slave_instance.get('slave_reference') %} {%- do part_list.append(monitor_ipv4_section_title) %} [{{ monitor_ipv4_section_title }}] <= monitor-promise-base module = check_icmp_packet_lost name = {{ monitor_ipv4_section_title }}.py config-address = {{ dumps(monitor_ipv4_test) }} config-ipv4 = true # promise frequency in minutes (2 times/day) config-frequency = 720 {%- endif %} {#- ############################### #} {#- Publish Slave Information #} {%- if not extra_slave_instance_list %} {%- set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') %} {%- do part_list.append(publish_section_title) %} [{{ publish_section_title }}] recipe = slapos.cookbook:publish {%- for key, value in slave_publish_dict.iteritems() %} {{ key }} = {{ value }} {%- endfor %} {%- else %} {%- do slave_instance_information_list.append(slave_publish_dict) %} {%- endif %} {%- if slave_type != 'redirect' %} {%- do backend_slave_list.append(slave_instance) %} {%- endif %} {%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #} [slave-log-directories] <= slave-log-directory-dict recipe = slapos.cookbook:mkdirectory {%- do part_list.append('slave-log-directories') %} {%- do part_list.append('caddy-log-access') %} {#- ############################################## #} {#- ## Prepare virtualhost for slaves using cache #} {#- Define IPv6 to IPV4 tunneling #} [tunnel-6to4-base] recipe = slapos.cookbook:wrapper ipv4 = ${slap-network-information:local-ipv4} ipv6 = ${slap-network-information:global-ipv6} wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port} command-line = {{ sixtunnel_executable }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port} hash-existing-files = ${buildout:directory}/software_release/buildout.cfg [tunnel-6to4-base-http_port] <= tunnel-6to4-base ipv4-port = {{ http_port }} ipv6-port = {{ http_port }} [tunnel-6to4-base-https_port] <= tunnel-6to4-base ipv4-port = {{ https_port }} ipv6-port = {{ https_port }} {#- Define log access #} [caddy-log-access-parameters] caddy_log_directory = {{ dumps(caddy_log_directory) }} caddy_configuration_directory = {{ dumps(caddy_configuration_directory) }} local_ipv4 = {{ dumps(local_ipv4) }} global_ipv6 = {{ dumps(global_ipv6) }} https_port = {{ dumps(https_port) }} http_port = {{ dumps(http_port) }} ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }} access_log = {{ dumps(access_log) }} error_log = {{ dumps(error_log) }} not_found_file = {{ dumps(not_found_file) }} [caddy-log-access] < = jinja2-template-base template = {{frontend_configuration.get('template-log-access')}} rendered = {{frontend_configuration.get('log-access-configuration')}} extra-context = section slave_log_directory slave-log-directory-dict section slave_password slave-password section parameter_dict caddy-log-access-parameters {#- Publish information for the instance #} [publish-caddy-information] recipe = slapos.cookbook:publish.serialised public-ipv4 = {{ public_ipv4 }} private-ipv4 = {{ local_ipv4 }} {%- if extra_slave_instance_list %} {#- sort_keys are important in order to avoid shuffling parameters on each run #} slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list, sort_keys=True) }} {%- endif %} monitor-base-url = {{ monitor_base_url }} csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/csr_id.txt backend-client-csr_id-url = https://[${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port}/backend-haproxy-csr_id.txt csr_id-certificate = ${get-csr_id-certificate:certificate} [kedifa-updater] recipe = slapos.cookbook:wrapper command-line = {{ kedifa_configuration['kedifa-updater'] }} --server-ca-certificate {{ kedifa_configuration['ca-certificate'] }} --identity {{ kedifa_configuration['certificate'] }} --master-certificate {{ master_certificate }} --on-update "{{ frontend_graceful_reload }}" ${kedifa-updater-mapping:file} {{ kedifa_configuration['kedifa-updater-state-file'] }} wrapper-path = {{ directory['service'] }}/kedifa-updater hash-existing-files = ${buildout:directory}/software_release/buildout.cfg [kedifa-updater-run] recipe = plone.recipe.command stop-on-error = True command = {{ kedifa_configuration['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ frontend_graceful_reload }}" update-command = ${:command} [kedifa-updater-mapping] recipe = slapos.recipe.template:jinja2 file = {{ kedifa_configuration['kedifa-updater-mapping-file'] }} template = inline: {%- for mapping in kedifa_updater_mapping %} {{ mapping[0] }} {{ mapping[1] }} {{ mapping[2] }} {%- endfor %} rendered = ${:file} [caddy-log-access-empty] # Caddy refuse to start if an `import`ed file is empty, so we prepend a header # so that the file is never empty. < = jinja2-template-base template = inline: # This file contain directives to serve directories with log files for shared instances, but no shared instances are defined yet. rendered = {{frontend_configuration.get('log-access-configuration')}} ##<Backend haproxy> [backend-haproxy-configuration] < = jinja2-template-base template = {{ template_backend_haproxy_configuration }} rendered = ${backend-haproxy-config:file} backend_slave_list = {{ dumps(sorted(backend_slave_list)) }} extra-context = key backend_slave_list :backend_slave_list section configuration backend-haproxy-config [backend-haproxy-config] file = {{ backend_haproxy_configuration['file'] }} pid-file = {{ backend_haproxy_configuration['pid-file'] }} log-socket = {{ backend_haproxy_configuration['log-socket'] }} local-ipv4 = {{ dumps('' ~ local_ipv4) }} http-port = {{ ('' ~ backend_haproxy_configuration['http-port']) }} https-port = {{ ('' ~ backend_haproxy_configuration['https-port']) }} request-timeout = {{ dumps('' ~ configuration['request-timeout']) }} backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }} backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }} certificate = {{ dumps('' ~ backend_haproxy_configuration['certificate']) }} [store-backend-haproxy-csr_id] recipe = plone.recipe.command csr_id_path = {{ directory['csr_id'] }}/backend-haproxy-csr_id.txt csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_} stop-on-error = False update-command = ${:command} command = {{ bin_directory }}/caucase \ --ca-url {{ backend_haproxy_configuration['caucase-url'] }} \ --ca-crt {{ backend_haproxy_configuration['cas-ca-certificate'] }} \ --crl {{ backend_haproxy_configuration['crl'] }} \ --mode service \ --send-csr {{ backend_haproxy_configuration['csr'] }} > ${:csr_work_path} && \ cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path} ##<Backend haproxy> [buildout] extends = {{ common_profile }} {{ logrotate_base_instance }} {{ monitor_template }} parts += kedifa-updater kedifa-updater-run backend-haproxy-configuration {%- for part in part_list %} {{ ' %s' % part }} {%- endfor %} {%- if 'caddy-log-access' not in part_list %} caddy-log-access-empty {%- endif %} publish-caddy-information tunnel-6to4-base-http_port tunnel-6to4-base-https_port expose-csr_id promise-expose-csr_id-ip-port cache-access = {{ cache_access }} [store-csr_id] recipe = plone.recipe.command csr_id_path = {{ directory['csr_id'] }}/csr_id.txt csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_} stop-on-error = False update-command = ${:command} command = {{ bin_directory }}/caucase \ --ca-url {{ kedifa_configuration['caucase-url'] }} \ --ca-crt {{ kedifa_configuration['cas-ca-certificate'] }} \ --crl {{ kedifa_configuration['crl'] }} \ --mode service \ --send-csr {{ kedifa_configuration['csr'] }} > ${:csr_work_path} && \ cut -d ' ' -f 1 ${:csr_work_path} > ${:csr_id_path} [certificate-csr_id] recipe = plone.recipe.command certificate = {{ directory['caddy-csr_id'] }}/certificate.pem key = {{ directory['caddy-csr_id'] }}/key.pem stop-on-error = True update-command = ${:command} command = if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then openssl req -new -newkey rsa:2048 -sha256 -subj \ "/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-network-information:global-ipv6}" \ -days 5 -nodes -x509 -keyout ${:key} -out ${:certificate} fi [expose-csr_id-configuration] ip = ${slap-network-information:global-ipv6} port = 17001 key = ${certificate-csr_id:key} certificate = ${certificate-csr_id:certificate} error-log = {{ directory['caddy-csr_id-log'] }}/expose-csr_id.log [expose-csr_id-template] recipe = slapos.recipe.template:jinja2 template = inline: https://:${expose-csr_id-configuration:port}/ { bind ${expose-csr_id-configuration:ip} tls ${expose-csr_id-configuration:certificate} ${expose-csr_id-configuration:key} log ${expose-csr_id-configuration:error-log} } rendered = {{ directory['caddy-csr_id'] }}/Caddyfile [promise-expose-csr_id-ip-port] <= monitor-promise-base module = check_port_listening name = expose-csr_id-ip-port-listening.py config-hostname = ${expose-csr_id-configuration:ip} config-port = ${expose-csr_id-configuration:port} [expose-csr_id] depends = ${store-csr_id:command} ${store-backend-haproxy-csr_id:command} recipe = slapos.cookbook:wrapper command-line = {{ caddy_executable }} -conf ${expose-csr_id-template:rendered} -log ${expose-csr_id-configuration:error-log} -http2=true -disable-http-challenge -disable-tls-alpn-challenge -root {{ directory['csr_id'] }} wrapper-path = {{ directory['service'] }}/expose-csr_id hash-existing-files = ${buildout:directory}/software_release/buildout.cfg [get-csr_id-certificate] recipe = collective.recipe.shelloutput commands = certificate = cat ${certificate-csr_id:certificate} {%- endif %} {# if software_type == slap_software_type #}