slapos-master: Follow up changes on ERP5 stack related to caucase

software/slapos-master/apache-backend.conf.in

@@ -187,6 +187,7 @@ Listen {{ ip }}:{{ port }}
 Listen {{ ip }}:{{ port }}
 <VirtualHost {{ ip }}:{{ port }}>
   SSLEngine on
+  Timeout 3600
 {%   if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
   SSLVerifyClient require
   SSLCACertificateFile {{ parameter_dict['ca-cert'] }}

software/slapos-master/buildout.hash.cfg

@@ -14,12 +14,12 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = a617197b7be3cc40d97a97c75f923a48
+md5sum = edf81a602137858cd5835c050ac6e08c
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 0f0ff00eca0b2ba1f1626b6415acb719
+md5sum = 356cb73670ea4599ad608b29fb86b278
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in
-md5sum = 7d1d13f4a888c2735ff8d7a4a961d9b2
+md5sum = c16342bd6af60296aa4325677f5827c4

software/slapos-master/instance-balancer.cfg.in

+{% import "caucase" as caucase with context %}
 {% set part_list = [] -%}
-{% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
-{% set caucase_url = slapparameter_dict.get('caucase-url', '') -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
-{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
+{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
 {% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -22,80 +21,23 @@ per partition. No more (undefined result), no less (IndexError).
 recipe = slapos.recipe.template:jinja2
 mode = 644
 
-[simplefile]
-< = jinja2-template-base
-template = inline:{{ '{{ content }}' }}
-
-{% macro simplefile(section_name, file_path, content, mode='') -%}
-{%   set content_section_name = section_name ~ '-content' -%}
-[{{  content_section_name }}]
-content = {{ dumps(content) }}
-
-[{{  section(section_name) }}]
-< = simplefile
-rendered = {{ file_path }}
-context = key content {{content_section_name}}:content
-mode = {{ mode }}
-{%- endmacro %}
-
-[certificate-request-base]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:bin}/request-instance-certificate
-command-line = {{ parameter_dict['bin-directory'] }}/caucase-cliweb
-  --crt-file ${apache-conf-ssl:cert}
-  --key-file ${apache-conf-ssl:key}
-  --crl-file ${apache-conf-ssl:crl}
-  --ca-url {{ caucase_url }}
-  --ca-crt-file ${apache-conf-ssl:ca-cert}
-
-{% macro request_cert(name, common_name) -%}
-{% set get_crl_periodicity = slapparameter_dict.get('crl-update-periodicity', 'daily') -%}
-
-[{{ section(name ~ '-certificate-request') }}]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:services}/request-{{ name }}-certificate
-command-line =
-  ${certificate-request-base:wrapper-path}
-  --cn {{ common_name }}
-  --request
-
-[{{ section(name ~ '-renew-cron-entry') }}]
-recipe = slapos.cookbook:cron.d
-cron-entries = ${cron:cron-entries}
-name = {{ name }}-certificate-auto-renew
-time = weekly
-# 2592000 = 30*24*60*60  equivalent to one month in seconds
-command = ${certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${apache-graceful:output}"
-
-[{{ section(name ~ '-download-crl') }}]
-# download the crl for the first time
-recipe = plone.recipe.command
-command = 
-  if [ ! -s "${apache-conf-ssl:crl}" ]; then
-    ${certificate-request-base:wrapper-path} --update-crl
-  fi
-update-command = ${:command}
-stop-on-error = true
-
-[{{ section(name ~ '-update-crl-cron-entry') }}]
-recipe = slapos.cookbook:cron.d
-cron-entries = ${cron:cron-entries}
-name = {{ name }}-update-crl
-time = {{ get_crl_periodicity }}
-# XXX - Update crl call apache graceful restart, it's not recommended to check crl too often, Apache
-# has an issue with reload and can be frozen and stop responding. Default periodicity time = daily
-command = ${certificate-request-base:wrapper-path} --update-crl --on-crl-update="${apache-graceful:output}"
-{%- endmacro %}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=parameter_dict['bin-directory'],
+     updater_path='${directory:services-on-watch}/caucase-updater',
+     url=ssl_parameter_dict['caucase-url'],
+     data_dir='${directory:srv}/caucase-updater',
+     crt_path='${apache-conf-ssl:cert}',
+     ca_path='${apache-conf-ssl:ca-cert}',
+     crl_path='${apache-conf-ssl:crl}',
+     key_path='${apache-conf-ssl:key}',
+     on_renew='${apache-graceful:output}',
+     max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
+     template_csr_pem=ssl_parameter_dict.get('csr'),
+     openssl=parameter_dict['openssl'] ~ '/bin/openssl',
+)}}
+{% do section('caucase-updater') -%}
 
-{% if use_ipv6 -%}
-[zope-tunnel-base]
-recipe = slapos.cookbook:ipv4toipv6
-runner-path = ${directory:services}/${:base-name}
-6tunnel-path = {{ parameter_dict['6tunnel'] }}/bin/6tunnel
-shell-path = {{ parameter_dict['dash'] }}/bin/dash
-ipv4 = {{ ipv4 }}
-
-{% endif -%}
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
 {% set zope_virtualhost_monster_backend_dict = {} %}
@@ -112,18 +54,7 @@ ipv4 = {{ ipv4 }}
 {%       if webdav -%}
 {%         do has_webdav.append(None) %}
 {%       endif -%}
-{%       if use_ipv6 -%}
-{%         set current_port = next_port() -%}
-[{{ section('zope-tunnel-' ~ current_port) }}]
-< = zope-tunnel-base
-base-name = {{ 'zeo-tunnel-' ~ current_port }}
-ipv4-port = {{ current_port }}
-ipv6-port = {{ zope_address.split(']:')[1] }}
-ipv6 = {{ zope_address.split(']:')[0][1:] }}
-{%         set zope_effective_address = ipv4 ~ ":" ~ current_port -%}
-{%       else -%}
 {%       set zope_effective_address = zope_address -%}
-{%       endif -%}
 {%       do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
 
 {#       # Generate entries with rewrite rule for test runnners #}
@@ -163,7 +94,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
+{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, slapparameter_dict['ssl-authentication-dict'].get(family_name, False))) -%}
 {% endfor -%}
 
 [haproxy-cfg-parameter-dict]
@@ -185,8 +116,6 @@ wrapper-path = ${directory:services}/haproxy
 command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:rendered}"
 hash-files = ${haproxy-cfg:rendered}
 
-{# TODO: build socat and wrap it as "${directory:bin}/haproxy-ctl" to connect to "${haproxy-cfg-parameter-dict:socket-path}" #}
-
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt
 key = ${directory:apache-conf}/apache.pem
@@ -204,13 +133,13 @@ access-log = ${directory:log}/apache-access.log
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
-cert = ${apache-ssl:cert}
-key = ${apache-ssl:key}
+cert = ${apache-conf-ssl:cert}
+key = ${apache-conf-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 # Client x509 auth
-ca-cert = ${apache-ssl-client:cert}
-crl = ${apache-ssl-client:crl}
+ca-cert = ${apache-conf-ssl:ca-cert}
+crl = ${apache-conf-ssl:crl}
 
 {% if shared_ca_path -%}
 shared-ca-cert = {{ shared_ca_path }}/cacert.pem
@@ -233,13 +162,12 @@ wait-for-files =
 
 [apache-graceful]
 recipe = collective.recipe.template
+output = ${directory:bin}/apache-httpd-graceful
+mode = 700
 input = inline:
   #!/bin/sh
   kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
 
-output = ${directory:bin}/apache-httpd-graceful
-mode = 700
-
 [{{ section('apache-promise') }}]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
 recipe = slapos.cookbook:check_port_listening
@@ -259,39 +187,6 @@ recipe = slapos.cookbook:publish.serialised
 
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
-[apache-ssl]
-{% if ssl_parameter_dict.get('key') -%}
-key = ${apache-ssl-key:rendered}
-cert = ${apache-ssl-cert:rendered}
-{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
-{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
-{% elif caucase_url -%}
-key = ${apache-conf-ssl:key}
-cert = ${apache-conf-ssl:cert}
-
-{{ request_cert('erp5', 'instance.apache@erp5') }}
-{% else %}
-recipe = plone.recipe.command
-command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
-key = ${apache-conf-ssl:key}
-cert = ${apache-conf-ssl:cert}
-{%- endif %}
-
-[apache-ssl-client]
-{% if ssl_parameter_dict.get('ca-cert') -%}
-cert = ${apache-ssl-ca:rendered}
-crl = ${apache-ssl-crl:rendered}
-{{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }}
-{{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }}
-{% elif caucase_url -%}
-cert = ${apache-conf-ssl:ca-cert}
-crl = ${apache-conf-ssl:crl}
-
-{% else %}
-cert =
-crl =
-{%- endif %}
-
 {% set apache_service_log_list = {} -%}
 {% for family_name, (_, _, _, authentication) in apache_dict.items() -%}
 {%   if authentication -%}
@@ -319,6 +214,7 @@ bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
 promise = ${:etc}/promise
 services = ${:etc}/run
+services-on-watch = ${:etc}/service
 var = ${buildout:directory}/var
 run = ${:var}/run
 log = ${:var}/log

software/slapos-master/instance-erp5.cfg.in

 {% import "root_common" as root_common with context -%}
+{% import "caucase" as caucase with context %}
 {% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
 {% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
 {% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
@@ -19,13 +20,13 @@
 {%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
 {% endif -%}
 {% set monitor_base_url_dict = {} -%}
-{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
-{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
+{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }}
+{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
 
 {% macro request(name, software_type, config_key, config, ret={'url': True}, key_config={}) -%}
 {% do config.update(slapparameter_dict.get(config_key, {})) -%}
@@ -53,6 +54,44 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
 config-name = {{ name }}
 {% endmacro -%}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+promise = ${:etc}/promise
+service-on-watch = ${:etc}/service
+srv = ${buildout:directory}/srv
+backup-caucased = ${:srv}/backup/caucased
+
+{% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
+{% set caucase_url = caucase_dict.get('url') -%}
+{% if not caucase_url -%}
+{%   if use_ipv6 -%}
+{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
+{%-  else -%}
+{%     set caucase_host = (ipv4_set | list)[0] %}
+{%-  endif %}
+{%   set caucase_port = caucase_dict.get('base-port', 8890) -%}
+{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_port -%}
+{%   set caucase_url = 'http://' ~ caucase_netloc -%}
+{{   caucase.caucased(
+       prefix='caucased',
+       buildout_bin_directory=bin_directory,
+       caucased_path='${directory:service-on-watch}/caucased',
+       backup_dir='${directory:backup-caucased}',
+       data_dir='${directory:srv}/caucased',
+       netloc=caucase_netloc,
+       service_auto_approve_count=caucase_dict.get('service-auto-approve-amount', 1),
+       user_auto_approve_count=caucase_dict.get('user-auto-approve-amount', 0),
+       key_len=caucase_dict.get('key-length', 2048),
+       promise='${directory:promise}/caucased',
+)}}
+{% do root_common.section('caucased') -%}
+{% do root_common.section('caucased-promise') -%}
+{% endif -%}
+{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
+{% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
+{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%}
+
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
@@ -64,14 +103,6 @@ config-name = {{ name }}
 connection-url = smtp://127.0.0.2:0/
 {%- endif %}
 
-{% if caucase_url -%}
-{%   do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
-[request-caucase]
-connection-http-url = {{ caucase_url }}
-{%- else %}
-{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891, 'auto-sign-csr-amount': 2}, {'http-url': True, 'https-url': False}) }}
-{% endif -%}
-
 {# ZODB -#}
 {% set zodb_dict = {} -%}
 {% set storage_dict = {} -%}
@@ -124,14 +155,16 @@ return =
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
-config-caucase-url = ${request-caucase:connection-http-url}
 config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
+config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }}
 config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
 config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
 config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
+config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
+config-zope-longrequest-logger-maximum-delay = {{ dumps(monitor_dict.get('zope-longrequest-logger-maximum-delay', 0)) }}
 config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${publish-early:inituser-password}
 config-kumofs-url = ${request-memcached-persistent:connection-url}
@@ -282,7 +315,6 @@ command =
   chmod 644 ${apache-certificate-authority:ca-dir}/openssl.cnf
 update-command = ${:command}
 
-{% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
 [request-balancer]
 <= request-common
 name = balancer
@@ -298,7 +330,6 @@ return =
   {% endif %}
 {% endfor -%}
 {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
-
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
@@ -309,11 +340,9 @@ config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':conn
 {% endfor -%}
 # XXX: should those really be same for all families ?
 config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
-config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
 config-monitor-passwd = ${monitor-htpasswd:passwd}
+config-ssl = {{ dumps(balancer_dict['ssl']) }}
 config-name = ${:name}
-config-caucase-url = ${request-caucase:connection-http-url}
-config-crl-update-periodicity = {{ crl_update_period }}
 config-shared-certificate-authority-path = ${directory:ca-dir} 
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
@@ -444,4 +473,3 @@ password = ${monitor-htpasswd:passwd}
 {% for key, value in monitor_base_url_dict.items() -%}
 {{ key }} = {{ value }}
 {% endfor %}
-