{% import "caucase" as caucase with context %} {% set part_list = [] -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%} {% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%} {# XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6 per partition. No more (undefined result), no less (IndexError). -#} {% set ipv4 = (ipv4_set | list)[0] -%} {% set apache_ip_list = [ipv4] -%} {% if ipv6_set -%} {% set ipv6 = (ipv6_set | list)[0] -%} {% do apache_ip_list.append('[' ~ ipv6 ~ ']') -%} {% endif -%} [jinja2-template-base] recipe = slapos.recipe.template:jinja2 mode = 644 [balancer-csr-template-config] < = jinja2-template-base template = inline: [req] prompt = no req_extensions = req_ext distinguished_name = dn [ dn ] CN = example.com [ req_ext ] subjectAltName = @alt_names [ alt_names ] IP.1 = {{ ipv4 }} {% if ipv6_set -%} IP.2 = {{ ipv6 }} {% endif %} rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt [balancer-csr-template] recipe = plone.recipe.command command = if [ ! -f '${:csr}' ] ; then {{ parameter_dict["openssl"] }}/bin/openssl req \ -newkey rsa:2048 \ -batch \ -new \ -nodes \ -keyout /dev/null \ -config '${balancer-csr-template-config:rendered}' \ -out '${:csr}' fi stop-on-error = true csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem {{ caucase.updater( prefix='caucase-updater', buildout_bin_directory=parameter_dict['bin-directory'], updater_path='${directory:services-on-watch}/caucase-updater', url=ssl_parameter_dict['caucase-url'], data_dir='${directory:srv}/caucase-updater', crt_path='${apache-conf-ssl:cert}', ca_path='${directory:srv}/caucase-updater/ca.crt', crl_path='${directory:srv}/caucase-updater/crl.pem', key_path='${apache-conf-ssl:key}', on_renew='${apache-graceful:output}', max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0), template_csr_pem=ssl_parameter_dict.get('csr'), template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-template:csr}', openssl=parameter_dict['openssl'] ~ '/bin/openssl', )}} {% do section('caucase-updater') -%} {% do section('caucase-updater-promise') -%} {% set frontend_caucase_url_hash_list = [] -%} {% for frontend_caucase_url in frontend_caucase_url_list -%} {% set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%} {% do frontend_caucase_url_hash_list.append(hash) -%} {% set data_dir = '${directory:srv}/client-cert-ca/%s' % hash -%} {{ caucase.updater( prefix='caucase-updater-%s' % hash, buildout_bin_directory=parameter_dict['bin-directory'], updater_path='${directory:services-on-watch}/caucase-updater-%s' % hash, url=frontend_caucase_url, data_dir=data_dir, ca_path='%s/ca.crt' % data_dir, crl_path='%s/crl.pem' % data_dir, on_renew='${caucase-updater-housekeeper:output}; ${apache-graceful:output}', max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0), openssl=parameter_dict['openssl'] ~ '/bin/openssl', )}} {% do section('caucase-updater-%s' % hash) -%} {% endfor -%} {% if frontend_caucase_url_hash_list -%} [caucase-updater-housekeeper] recipe = collective.recipe.template output = ${directory:bin}/caucase-updater-housekeeper mode = 700 input = inline: #!${buildout:executable} import glob import os import subprocess hash_list = {{ repr(frontend_caucase_url_hash_list) }} crt_list = ['%s.crt' % e for e in hash_list] crl_list = ['%s.crl' % e for e in hash_list] for path in glob.glob('${apache-conf-ssl:ca-cert-dir}/*.crt'): if os.path.basename(path) not in crt_list: os.unlink(path) for path in glob.glob('${apache-conf-ssl:crl-dir}/*.crl'): if os.path.basename(path) not in crl_list: os.unlink(path) for hash in hash_list: crt = '${directory:srv}/client-cert-ca/%s/ca.crt' % hash crt_link = '${apache-conf-ssl:ca-cert-dir}/%s.crt' % hash crl = '${directory:srv}/client-cert-ca/%s/crl.pem' % hash crl_link = '${apache-conf-ssl:crl-dir}/%s.crl' % hash if os.path.isfile(crt) and not os.path.islink(crt_link): os.symlink(crt, crt_link) if os.path.isfile(crl) and not os.path.islink(crl_link): os.symlink(crl, crl_link) subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:ca-cert-dir}']) subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:crl-dir}']) [caucase-updater-housekeeper-run] recipe = plone.recipe.command command = ${caucase-updater-housekeeper:output} update-command = ${:command} {% endif -%} {% set haproxy_dict = {} -%} {% set apache_dict = {} -%} {% set zope_virtualhost_monster_backend_dict = {} %} {% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% for family_name, parameter_id_list in sorted( slapparameter_dict['zope-family-dict'].iteritems()) -%} {% set zope_family_address_list = [] -%} {% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%} {% set has_webdav = [] -%} {% for parameter_id in parameter_id_list -%} {% set zope_address_list = slapparameter_dict[parameter_id] -%} {% for zope_address, maxconn, webdav in zope_address_list -%} {% if webdav -%} {% do has_webdav.append(None) %} {% endif -%} {% set zope_effective_address = zope_address -%} {% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%} {% endfor -%} {# # Generate entries with rewrite rule for test runnners #} {% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %} {% if test_runner_address_list -%} {% set test_runner_backend_mapping = {} %} {% set test_runner_apache_url_list = [] %} {% set test_runner_external_port = next_port() %} {% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %} {% do test_runner_backend_mapping.__setitem__( 'unit_test_' ~ i, 'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %} {% do test_runner_apache_url_list.append( 'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %} {% endfor %} {% do zope_virtualhost_monster_backend_dict.__setitem__( (ipv4, test_runner_external_port), ( ssl_authentication, test_runner_backend_mapping ) ) -%} {% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%} {% endif -%} {% endfor -%} {# Make rendering fail artificially if any family has no known backend. # This is useful as haproxy's hot-reconfiguration mechanism is # supervisord-incompatible. # As jinja2 postpones KeyError until place-holder value is actually used, # do a no-op getitem. -#} {% do zope_family_address_list[0][0] -%} {% set haproxy_port = next_port() -%} {% set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%} {% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%} {% if has_webdav -%} {% set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#} {% set external_scheme = 'webdavs' -%} {% else %} {% set internal_scheme = 'http' -%} {% set external_scheme = 'https' -%} {% endif -%} {% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, slapparameter_dict['ssl-authentication-dict'].get(family_name, False))) -%} {% endfor -%} [haproxy-cfg-parameter-dict] socket-path = ${directory:run}/haproxy.sock server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }} backend-dict = {{ dumps(haproxy_dict) }} ip = {{ ipv4 }} [haproxy-cfg] < = jinja2-template-base template = {{ parameter_dict['template-haproxy-cfg'] }} rendered = ${directory:etc}/haproxy.cfg context = section parameter_dict haproxy-cfg-parameter-dict extensions = jinja2.ext.do [{{ section('haproxy') }}] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:services}/haproxy command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:rendered}" hash-files = ${haproxy-cfg:rendered} [apache-conf-ssl] cert = ${directory:apache-conf}/apache.crt key = ${directory:apache-conf}/apache.pem {% if frontend_caucase_url_list -%} depends = ${caucase-updater-housekeeper-run:recipe} ca-cert-dir = ${directory:apache-ca-cert-dir} crl-dir = ${directory:apache-crl-dir} {%- endif %} [apache-conf-parameter-dict] backend-list = {{ dumps(apache_dict.values()) }} zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }} ip-list = {{ dumps(apache_ip_list) }} pid-file = ${directory:run}/apache.pid log-dir = ${directory:log} error-log = ${directory:log}/apache-error.log access-log = ${directory:log}/apache-access.log # Apache 2.4's default value (60 seconds) can be a bit too short timeout = 300 # Basic SSL server configuration cert = ${apache-conf-ssl:cert} key = ${apache-conf-ssl:key} cipher = ssl-session-cache = ${directory:log}/apache-ssl-session-cache {% if frontend_caucase_url_list -%} # Client x509 auth ca-cert-dir = ${apache-conf-ssl:ca-cert-dir} crl-dir = ${apache-conf-ssl:crl-dir} {%- endif %} [apache-conf] < = jinja2-template-base template = {{ parameter_dict['template-apache-conf'] }} rendered = ${directory:apache-conf}/apache.conf context = section parameter_dict apache-conf-parameter-dict [{{ section('apache') }}] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:services}/apache command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND wait-for-files = ${apache-conf-ssl:cert} ${apache-conf-ssl:key} [apache-graceful] recipe = collective.recipe.template output = ${directory:bin}/apache-httpd-graceful mode = 700 input = inline: #!/bin/sh kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')" [{{ section('apache-promise') }}] <= monitor-promise-base # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently module = check_port_listening name = apache.py config-hostname = {{ ipv4 }} config-port = {{ apache_dict.values()[0][0] }} [{{ section('publish') }}] recipe = slapos.cookbook:publish.serialised {% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%} {{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %} {{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }} {% endfor -%} {% for family_name, test_runner_url_list in test_runner_url_dict.items() -%} {{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }} {% endfor -%} monitor-base-url = ${monitor-publish-parameters:monitor-base-url} [{{ section('logrotate-apache') }}] < = logrotate-entry-base name = apache log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log} post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1 [directory] recipe = slapos.cookbook:mkdirectory apache-conf = ${:etc}/apache {% if frontend_caucase_url_list -%} apache-ca-cert-dir = ${:apache-conf}/ssl.crt apache-crl-dir = ${:apache-conf}/ssl.crl {% endif -%} bin = ${buildout:directory}/bin etc = ${buildout:directory}/etc services = ${:etc}/run services-on-watch = ${:etc}/service var = ${buildout:directory}/var run = ${:var}/run log = ${:var}/log srv = ${buildout:directory}/srv apachedex = ${monitor-directory:private}/apachedex [{{ section('resiliency-exclude-file') }}] # Generate rdiff exclude file in case of resiliency < = jinja2-template-base template = {{ 'inline:{{ "${directory:log}/**\\n" }}' }} rendered = ${directory:srv}/exporter.exclude [{{ section('monitor-generate-apachedex-report') }}] recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = generate-apachedex-report # The goal is to be executed before logrotate log rotation. # Here, logrotate-entry-base:frequency = daily, so we run at 23 o'clock every day. frequency = 0 23 * * * command = ${monitor-generate-apachedex-report-wrapper:wrapper-path} [monitor-generate-apachedex-report-wrapper] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:bin}/${:command} command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}" command = generate-apachedex-report [apachedex-parameters] # XXX - Sample log file with curent date: apache_access.log-%(date)s.gz # which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12 apache-log-list = ${apache-conf-parameter-dict:access-log} configuration = {{ slapparameter_dict['apachedex-configuration'] }} promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }} [{{ section('monitor-promise-apachedex-result') }}] <= monitor-promise-base module = check_command_execute name = check-apachedex-result.py config-command = "{{ parameter_dict['promise-check-apachedex-result'] }}" --apachedex_path "${directory:apachedex}" --status_file ${monitor-directory:private}/apachedex.report.json --threshold "${apachedex-parameters:promise-threshold}" [{{ section('promise-check-computer-memory') }}] <= monitor-promise-base module = check_command_execute name = check-computer-memory.py config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent [monitor-instance-parameter] monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-port = {{ next_port() }} monitor-title = {{ slapparameter_dict['name'] }} password = {{ slapparameter_dict['monitor-passwd'] }} [buildout] extends = {{ template_monitor }} parts += {{ part_list | join('\n ') }}