stack/erp5: use caucase managed certificate for balancer

This reverts commit 620c9332 (stack/erp5: stop using caucase managed
certificate for balancer, 2020-11-10) with an updated design. We add a
caucase service for balancer in the balancer partition. The caucase
service from the root partition (that was not used) is removed.

The underlying idea is that the default configuration should use multiple
caucases with limited scope, here we have one caucase to manage the
certificate used by haproxy server in the balancer partition, so we put
one caucase to manage this certificate and the caucase is configured to
auto-accept one certificate only. The plan is that when we will add a
certificate for mariadb server, we'll add another caucase inside this
mariadb server.

For more advanced usage and also to support the cases where a new
certificate needs to be re-emitted for some reason, users can request
with an existing caucase URL. In that case, they will have to accept
the certificate requests.

Notable changes:

balancer/ssl/caucase-url is no longer documented in parameters, this is
an internal parameter, users can pass one global caucase service to
manage all partition

CAUCASE environment variable is no longer set when running zope. There
was no identified use case and with this new approach of multiple
caucases, the term "caucase" alone became ambiguous.

software/erp5/README.rst

@@ -90,7 +90,6 @@ This software release assigns the following port ranges by default:
   balancer              2150-2199
   zope                  2200-*
   jupyter               8888
-  caucase               8890,8891
   ====================  ==========
 
 Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to

software/erp5/instance-erp5-input-schema.json

@@ -514,7 +514,7 @@
           "properties": {
             "url": {
               "title": "Caucase URL",
-              "description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
+              "description": "URL of existing caucase instance to use. If empty, caucase instances will be deployed inside partitions.",
               "default": "",
               "type": "string",
               "format": "uri"
@@ -721,15 +721,9 @@
               },
               "uniqueItems": true
             },
-            "caucase-url": {
-              "title": "Caucase URL",
-              "description": "URL of caucase service to use. If not set, global setting will be used.",
-              "type": "string",
-              "format": "uri"
-            },
             "csr": {
               "title": "csr",
-              "description": "PEM-encoded certificate signature request to request server certificate with. If not provided, HTTPS will be disabled.",
+              "description": "PEM-encoded certificate signature request to request server certificate with.",
               "type": "string"
             },
             "max-crl-update-delay": {

software/erp5/instance-erp5-output-schema.json

@@ -69,7 +69,7 @@
       "type": "string"
     },
     "caucase-http-url": {
-      "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.",
+      "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port.",
       "pattern": "^http://",
       "type": "string"
     }

software/erp5/test/test/test_balancer.py

 import glob
+import ipaddress
 import json
 import logging
 import os
 import re
+import socket
 import subprocess
+import sqlite3
+import tempfile
 import time
 import urllib.parse
 from http.server import BaseHTTPRequestHandler
 from unittest import mock
 
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+import OpenSSL.SSL
 import pexpect
 import psutil
 import requests
 
+from slapos.proxy.db_version import DB_VERSION
 from slapos.testing.utils import CrontabMixin, ManagedHTTPServer
 
 from . import CaucaseCertificate, CaucaseService, ERP5InstanceTestCase, default, matrix, setUpModule
@@ -100,9 +111,7 @@ class BalancerTestCase(ERP5InstanceTestCase):
         },
         'dummy_http_server': [[cls.getManagedResource("backend_web_server", EchoHTTPServer).netloc, 1, False]],
         'ssl-authentication-dict': {'default': False},
-        'ssl': {
-            'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
-        },
+        'ssl': {},
         'timeout-dict': {'default': None},
         'family-path-routing-dict': {},
         'path-routing-list': [],
@@ -492,6 +501,155 @@ class TestHTTP(BalancerTestCase):
       ])
 
 
+class TestServerTLSEmbeddedCaucase(BalancerTestCase):
+  """Check Server TLS with embedded caucase
+  """
+  __partition_reference__ = 's'
+  def _getCaucaseCACertificatePath(self) -> str:
+    """Returns the path of the caucase certificate on file system.
+    """
+    ca_cert = tempfile.NamedTemporaryFile(
+      prefix="ca.crt.pem",
+      mode="w",
+      delete=False,
+    )
+    ca_cert.write(
+      requests.get(
+        urllib.parse.urljoin(
+          self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+          '/cas/crt/ca.crt.pem',
+        )).text)
+    ca_cert.flush()
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
+  def _getServerCertificate(self, hostname: str, port: int) -> x509.base.Certificate:
+    sock = socket.socket(socket.AF_INET6 if ':' in hostname else socket.AF_INET)
+    sock.connect((hostname, port))
+    ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+    sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
+    sock_ssl.set_connect_state()
+    sock_ssl.set_tlsext_host_name(hostname.encode())
+    sock_ssl.do_handshake()
+    cert = sock_ssl.get_peer_certificate()
+    crypto_cert = cert.to_cryptography()
+    sock_ssl.close()
+    sock.close()
+    return crypto_cert
+
+  def test_certificate_validates_with_caucase_ca(self) -> None:
+    requests.get(self.default_balancer_url, verify=self._getCaucaseCACertificatePath())
+
+  def test_certificate_renewal(self) -> None:
+    balancer_parsed_url = urllib.parse.urlparse(self.default_balancer_url)
+    certificate_before_renewal = self._getServerCertificate(
+      balancer_parsed_url.hostname,
+      balancer_parsed_url.port)
+
+    # run caucase updater in the future, so that certificate is renewed
+    caucase_updater, = glob.glob(
+      os.path.join(
+        self.computer_partition_root_path,
+        'etc',
+        'service',
+        'caucase-updater-haproxy-certificate-*',
+      ))
+    process = pexpect.spawnu("faketime +90days " + caucase_updater)
+    logger = self.logger
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+    process.logfile = DebugLogFile()
+    process.expect("Renewing .*\nNext wake-up.*")
+    process.terminate()
+    process.wait()
+
+    # wait for server to use new certificate
+    for _ in range(30):
+      certificate_after_renewal = self._getServerCertificate(
+        balancer_parsed_url.hostname,
+        balancer_parsed_url.port)
+      if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
+        break
+      time.sleep(.5)
+
+    self.assertGreater(
+      certificate_after_renewal.not_valid_before,
+      certificate_before_renewal.not_valid_before,
+    )
+
+    # requests are served properly after certificate renewal
+    self.test_certificate_validates_with_caucase_ca()
+
+
+class TestServerTLSExternalCaucase(TestServerTLSEmbeddedCaucase):
+  """Check Server TLS with external caucase
+  """
+  @classmethod
+  def _getInstanceParameterDict(cls) -> dict:
+    parameter_dict = super()._getInstanceParameterDict()
+    parameter_dict['ssl']['caucase-url'] = cls.getManagedResource(
+      "caucase", CaucaseService).url
+    return parameter_dict
+
+  def test_published_caucase_http_url_parameter(self) -> None:
+    self.assertEqual(
+      self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+      self.getManagedResource("caucase", CaucaseService).url,
+    )
+
+
+class TestServerTLSCSRTemplateParameter(TestServerTLSExternalCaucase):
+  """Check Server TLS with a CSR template passed as parameter
+  """
+  @classmethod
+  def _getInstanceParameterDict(cls) -> dict:
+    # use a CSR template with this subject, we'll assert that the
+    # certificate used by haproxy has same subject.
+    cls.csr_subject = subject = x509.Name(
+      [x509.NameAttribute(NameOID.COMMON_NAME, cls.__name__)])
+
+    # Add all IPs of the computer in SubjectAlternativeName, we don't
+    # know what will be the IP of the balancer partition.
+    with sqlite3.connect(cls.slap._proxy_database) as db:
+      ip_address_list = [
+        x509.IPAddress(ipaddress.ip_address(r)) for (r, ) in db.execute(
+          f"SELECT address FROM partition_network{DB_VERSION}").fetchall()
+      ]
+    assert ip_address_list
+
+    csr = x509.CertificateSigningRequestBuilder().subject_name(
+      subject).add_extension(
+        x509.SubjectAlternativeName(ip_address_list),
+        critical=True,
+      ).sign(
+        rsa.generate_private_key(
+          public_exponent=65537,
+          key_size=2048,
+          backend=default_backend(),
+        ),
+        hashes.SHA256(),
+        default_backend(),
+      )
+
+    parameter_dict = super()._getInstanceParameterDict()
+    parameter_dict['ssl']['csr'] = csr.public_bytes(serialization.Encoding.PEM).decode()
+    return parameter_dict
+
+  def test_certificate_validates_with_caucase_ca(self) -> None:
+    super().test_certificate_validates_with_caucase_ca()
+    balancer_parsed_url = urllib.parse.urlparse(self.default_balancer_url)
+    cert = self._getServerCertificate(
+      balancer_parsed_url.hostname,
+      balancer_parsed_url.port,
+    )
+    self.assertEqual(
+      cert.subject.rfc4514_string(),
+      self.csr_subject.rfc4514_string())
+
+
 class ContentTypeHTTPServer(ManagedHTTPServer):
   """An HTTP/1.1 Server which reply with content type from path.
 
@@ -722,8 +880,8 @@ class TestClientTLS(BalancerTestCase):
       self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
                        client_certificate_name)
 
-      # We have two services, in charge of updating CRL and CA certificates for
-      # each frontend CA
+      # We have two services in charge of updating CRL and CA certificates for
+      # each frontend CA, plus the one for the balancer's own certificate
       caucase_updater_list = glob.glob(
           os.path.join(
               self.computer_partition_root_path,
@@ -731,7 +889,7 @@ class TestClientTLS(BalancerTestCase):
               'service',
               'caucase-updater-*',
           ))
-      self.assertEqual(len(caucase_updater_list), 2)
+      self.assertEqual(len(caucase_updater_list), 3)
 
       # find the one corresponding to this caucase
       for caucase_updater_candidate in caucase_updater_list:

software/erp5/test/test/test_erp5.py

@@ -50,7 +50,7 @@ import urllib3
 from slapos.testing.utils import CrontabMixin
 import zc.buildout.configparser
 
-from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule
+from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule
 
 setUpModule # pyflakes
 
@@ -98,6 +98,22 @@ class TestPublishedURLIsReachableMixin:
       self.assertEqual(r.status_code, requests.codes.ok)
       self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+      prefix="ca.crt.pem",
+      mode="w",
+      delete=False,
+    )
+    ca_cert.write(
+      requests.get(
+        urllib.parse.urljoin(
+          self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+          '/cas/crt/ca.crt.pem',
+        )).text)
+    ca_cert.flush()
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
@@ -105,7 +121,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default-v6'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_family_default_v4_is_reachable(self):
@@ -115,7 +131,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['family-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
   def test_published_frontend_default_is_reachable(self):
@@ -125,7 +141,7 @@ class TestPublishedURLIsReachableMixin:
     self._checkERP5IsReachable(
       param_dict['url-frontend-default'],
       param_dict['site-id'],
-      verify=False,
+      self._getCaucaseServiceCACertificate(),
     )
 
 
@@ -158,6 +174,68 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
       self.getRootPartitionConnectionParameterDict()['url-frontend-default'])
 
 
+class TestExternalCaucase(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test providing the URL of an external caucase in parameters.
+  """
+  __partition_reference__ = 'ec'
+
+  @classmethod
+  def getInstanceParameterDict(cls) -> dict:
+    caucase_url = cls.getManagedResource("caucase", CaucaseService).url
+    return {'_': json.dumps({'caucase': {'url': caucase_url}})}
+
+  def test_published_caucase_http_url_parameter(self) -> None:
+    self.assertEqual(
+      self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+      self.getManagedResource("caucase", CaucaseService).url,
+    )
+
+
+class TestReinstantiateWithExternalCaucase(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test providing the URL of an external caucase in parameters after
+  the initial instantiation.
+  """
+  __partition_reference__ = 'sc'
+
+  def test_switch_to_external_caucase(self) -> None:
+    # this also waits that ERP5 is fully ready
+    self.test_published_frontend_default_is_reachable()
+
+    external_caucase_url = self.getManagedResource("caucase", CaucaseService).url
+    partition_parameter_kw = {
+      '_': json.dumps({'caucase': {
+        'url': external_caucase_url
+      }})
+    }
+    def rerequest():
+      return self.slap.request(
+        software_release=self.getSoftwareURL(),
+        software_type=self.getInstanceSoftwareType(),
+        partition_reference=self.default_partition_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        state='started')
+
+    rerequest()
+    self.slap.waitForInstance(max_retry=10)
+
+    self.assertEqual(
+      json.loads(rerequest().getConnectionParameterDict()['_'])['caucase-http-url'],
+      external_caucase_url)
+
+    with tempfile.NamedTemporaryFile(mode="w") as ca_cert:
+      ca_cert.write(
+        requests.get(
+          urllib.parse.urljoin(
+            external_caucase_url,
+            '/cas/crt/ca.crt.pem',
+          )).text)
+      ca_cert.flush()
+
+      requests.get(
+        self.getRootPartitionConnectionParameterDict()['url-frontend-default'],
+        verify=ca_cert.name).raise_for_status()
+
+
 class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 Jupyter notebook
   """

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 55232eae0bcdb68a7cb2598d2ba9d60c
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 1b9daa97ed74969f227aba07da0d97ec
+md5sum = d48b6b1568aaa08fc1a3b18a65740036
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,11 +86,11 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 2439b90d6f707f47050fc9074fa4d810
+md5sum = d8a18108165a00c8f2c8f2fca4e4274e
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d38f0716e3f9402be0d82a2748c7defb
+md5sum = dbd17fbde7a1a3b1a12e3ea1db25baa1
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -2,6 +2,20 @@
 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
+{% set caucase_url = ssl_parameter_dict.get('caucase-url') %}
+{% set with_embedded_caucased = caucase_url is none %}
+{% if not caucase_url -%}
+{%   if ipv6_set -%}
+{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
+{%-  else -%}
+{%     set caucase_host = (ipv4_set | list)[0] %}
+{%-  endif %}
+{%   set caucase_http_port = 2198 -%}{# caucase_https_port is implicitly 2199 #}
+{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_http_port -%}
+{%   set caucase_url = 'http://' ~ caucase_netloc %}
+{% endif %}
+{# Caucase related parts include a hash of the caucase_url, to make is possible to re-request with another caucase url. #}
+{% set caucase_haproxy_certificate_suffix = '-' + hashlib.md5(six.ensure_binary(caucase_url)).hexdigest()[:6] -%}
 {% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
 {#
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
@@ -15,25 +29,103 @@ per partition. No more (undefined result), no less (IndexError).
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 
+[simplefile]
+< = jinja2-template-base
+inline = {{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+output = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
+
+{% if with_embedded_caucased -%}
+{{   caucase.caucased(
+       prefix='caucased-haproxy-certificate',
+       buildout_bin_directory=bin_directory,
+       caucased_path='${directory:services-on-watch}/caucased' + caucase_haproxy_certificate_suffix,
+       backup_dir='${directory:backup-caucased-haproxy-certificate}',
+       data_dir='${directory:srv-caucased-haproxy-certificate}',
+       netloc=caucase_netloc,
+       tmp='${directory:tmp}',
+       service_auto_approve_count=ssl_parameter_dict.get('service-auto-approve-amount', 1),
+       user_auto_approve_count=ssl_parameter_dict.get('user-auto-approve-amount', 0),
+       key_len=ssl_parameter_dict.get('key-length', 2048),
+)}}
+{%   do section('caucased-haproxy-certificate') -%}
+{%   do section('caucased-haproxy-certificate-promise') -%}
+{% endif -%}
+
+[haproxy-certificate]
+cert-and-key-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}-cert-and-key.pem
+ca-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.ca.crt
+crl-file = ${directory:etc}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.crl
+
+[haproxy-certificate-csr-config]
+recipe = slapos.recipe.template
+inline =
+  [ req ]
+  prompt = no
+  req_extensions = req_ext
+  distinguished_name = dn
+  [ dn ]
+  CN = haproxy
+  [ req_ext ]
+  subjectAltName = @alt_names
+  [ alt_names ]
+  IP.1 = {{ ipv4 }}
+{% if ipv6_set %}
+  IP.2 = {{ ipv6 }}
+{% endif %}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[haproxy-certificate-csr]
+recipe = plone.recipe.command
+command =
+  if [ ! -f '${:csr}' ] ; then
+   {{ parameter_dict['openssl'] }}/bin/openssl req \
+      -newkey rsa \
+      -batch \
+      -new \
+      -nodes \
+      -keyout /dev/null \
+      -config '${haproxy-certificate-csr-config:output}' \
+      -out '${:csr}'
+  fi
+stop-on-error = true
+csr = ${directory:srv}/${:_buildout_section_name_}{{ caucase_haproxy_certificate_suffix }}.csr.pem
+
+{% if ssl_parameter_dict.get('csr') %}
+{{ simplefile(
+    'haproxy-certificate-csr-from-parameters',
+    '${directory:etc}/haproxy-certificate-csr-from-parameters.pem',
+    ssl_parameter_dict['csr']) }}
+[haproxy-certificate-csr]
+csr = ${haproxy-certificate-csr-from-parameters:output}
+{% endif %}
+
 {{ caucase.updater(
-     prefix='caucase-updater',
+     prefix='caucase-updater-haproxy-certificate',
      buildout_bin_directory=parameter_dict['bin-directory'],
-     updater_path='${directory:services-on-watch}/caucase-updater',
-     url=ssl_parameter_dict['caucase-url'],
-     data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:caucase-cert}',
-     ca_path='${directory:srv}/caucase-updater/ca.crt',
-     crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:caucase-key}',
+     updater_path='${directory:services-on-watch}/caucase-updater-haproxy-certificate' + caucase_haproxy_certificate_suffix,
+     url=caucase_url,
+     data_dir='${directory:caucase-updater-haproxy-certificate}',
+     crt_path='${haproxy-certificate:cert-and-key-file}',
+     ca_path='${haproxy-certificate:ca-file}',
+     crl_path='${haproxy-certificate:crl-file}',
+     key_path='${haproxy-certificate:cert-and-key-file}',
      on_renew='${haproxy-reload:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
-     template_csr_pem=ssl_parameter_dict.get('csr'),
+     template_csr='${haproxy-certificate-csr:csr}',
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
-{# XXX we don't use caucase yet.
-{% do section('caucase-updater') -%}
-{% do section('caucase-updater-promise') -%}
-#}
+{% do section('caucase-updater-haproxy-certificate') -%}
 
 {% set frontend_caucase_url_hash_list = [] -%}
 {% for frontend_caucase_url in frontend_caucase_url_list -%}
@@ -221,28 +313,6 @@ wrapper-path = ${directory:services-on-watch}/haproxy
 command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:output}"
 hash-files = ${haproxy-cfg:output}
 
-[apache-conf-ssl]
-# XXX caucase is/was buggy and this certificate does not match key for instances
-# that were updated, so don't use it yet.
-caucase-cert = ${directory:apache-conf}/apache-caucase.crt
-caucase-key = ${directory:apache-conf}/apache-caucase.pem
-
-[simplefile]
-< = jinja2-template-base
-inline = {{ '{{ content }}' }}
-
-{% macro simplefile(section_name, file_path, content, mode='') -%}
-{%   set content_section_name = section_name ~ '-content' -%}
-[{{  content_section_name }}]
-content = {{ dumps(content) }}
-
-[{{  section(section_name) }}]
-< = simplefile
-output = {{ file_path }}
-context = key content {{content_section_name}}:content
-mode = {{ mode }}
-{%- endmacro %}
-
 [{{ section('haproxy-socat-stats')}}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/${:_buildout_section_name_}
@@ -276,7 +346,7 @@ config-command = test -S ${rsyslogd-cfg-parameter-dict:log-socket}
 
 
 [haproxy-conf-ssl]
-certificate = ${build-certificate-and-key:certificate-and-key}
+certificate = ${haproxy-certificate:cert-and-key-file}
 {% if frontend_caucase_url_list -%}
 ca-cert = ${directory:etc}/frontend-ca.pem
 ca-cert-dir = ${directory:ca-cert}
@@ -285,18 +355,15 @@ crl-dir = ${directory:crl}
 depends = ${caucase-updater-housekeeper-run:recipe}
 {%- endif %}
 
-[build-certificate-and-key]
+# BBB cert and key as ssl parameters. Use caucase instead.
 {% if ssl_parameter_dict.get('key') -%}
-certificate-and-key = ${tls-certificate-and-key-from-parameters:output}
 {{ simplefile(
-    'tls-certificate-and-key-from-parameters',
+    'haproxy-conf-ssl-certificate-and-key-from-parameters',
     '${directory:etc}/certificate-and-key-from-parameters.pem',
     ssl_parameter_dict['cert'] ~ "\n" ~ ssl_parameter_dict['key']) }}
-{% else %}
-recipe = plone.recipe.command
-command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:certificate-and-key}" -out "${:certificate-and-key}"
-certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
-{%- endif %}
+[haproxy-conf-ssl]
+certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output}
+{% endif %}
 
 [{{ section('haproxy-promise') }}]
 <= monitor-promise-base
@@ -315,6 +382,7 @@ recipe = slapos.cookbook:publish.serialised
 {% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
 {{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
 {% endfor -%}
+caucase-http-url = {{ caucase_url }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [{{ section('logrotate-rsyslogd') }}]
@@ -334,6 +402,12 @@ var = ${buildout:directory}/var
 run = ${:var}/run
 log = ${:var}/log
 srv = ${buildout:directory}/srv
+{% if with_embedded_caucased %}
+srv-caucased-haproxy-certificate = ${:srv}/caucased-haproxy-certificate{{ caucase_haproxy_certificate_suffix }}/
+backup-caucased-haproxy-certificate = ${:srv}/backup/caucased{{ caucase_haproxy_certificate_suffix }}
+{% endif %}
+caucase-updater-haproxy-certificate = ${:srv}/caucase-updater-haproxy-certificate
+tmp = ${buildout:directory}/tmp
 apachedex = ${monitor-directory:private}/apachedex
 rsyslogd-spool = ${:run}/rsyslogd-spool
 {% if frontend_caucase_url_list -%}

stack/erp5/instance-erp5.cfg.in

@@ -79,35 +79,30 @@ bin = ${buildout:directory}/bin
 service-on-watch = ${buildout:directory}/etc/service
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
-backup-caucased = ${:srv}/backup/caucased
 
-{% if not caucase_url -%}
-{%   if use_ipv6 -%}
-{%     set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
-{%-  else -%}
-{%     set caucase_host = (ipv4_set | list)[0] %}
-{%-  endif %}
-{%   set caucase_port = caucase_dict.get('base-port', 8890) -%}
-{%   set caucase_netloc = caucase_host ~ ':' ~ caucase_port -%}
-{%   set caucase_url = 'http://' ~ caucase_netloc -%}
-{{   caucase.caucased(
-       prefix='caucased',
-       buildout_bin_directory=bin_directory,
-       caucased_path='${directory:service-on-watch}/caucased',
-       backup_dir='${directory:backup-caucased}',
-       data_dir='${directory:srv}/caucased',
-       netloc=caucase_netloc,
-       tmp='${directory:tmp}',
-       service_auto_approve_count=caucase_dict.get('service-auto-approve-amount', 1),
-       user_auto_approve_count=caucase_dict.get('user-auto-approve-amount', 0),
-       key_len=caucase_dict.get('key-length', 2048),
-)}}
-{% do root_common.section('caucased') -%}
-{% do root_common.section('caucased-promise') -%}
-{% endif -%}
-{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
 {% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%}
-{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%}
+{% do balancer_dict.setdefault('ssl', {}) %}
+{% if caucase_url %}
+{%   do balancer_dict['ssl'].setdefault('caucase-url', caucase_url) -%}
+
+[balancer-updated-caucase-url]
+# Promise to wait for balancer partition to use the provided URL, for cases where the
+# caucase URL is changed
+recipe = slapos.cookbook:check_parameter
+value = ${publish:caucase-http-url}
+expected-not-value =
+expected-value = {{ caucase_url }}
+path = ${directory:bin}/${:_buildout_section_name_}
+
+[balancer-updated-caucase-url-promise]
+<= monitor-promise-base
+promise = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command = ${balancer-updated-caucase-url:path}
+{% do root_common.section("balancer-updated-caucase-url-promise") -%}
+
+{% endif %}
+
 {% do balancer_dict.setdefault('tcpv4-port', 2150) -%}
 {% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%}
 {% set routing_path_template_field_dict = {"site-id": site_id} -%}
@@ -241,7 +236,6 @@ return =
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
-config-caucase-url = {{ dumps(caucase_url) }}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
 config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }}
@@ -444,7 +438,7 @@ config-allow-redirects = 0
 
 {%- endif %}
 
-{% set balancer_ret_dict = {'monitor-base-url': False} -%}
+{% set balancer_ret_dict = {'monitor-base-url': False, 'caucase-http-url': False} -%}
 {% for family in zope_family_dict -%}
 {%   do balancer_ret_dict.__setitem__(family, False) -%}
 {%   do balancer_ret_dict.__setitem__(family + '-v6', False) -%}
@@ -492,7 +486,7 @@ config-allow-redirects = 0
   ret=balancer_ret_dict,
   key_config=balancer_key_config_dict,
 ) }}
-
+{% do publish_dict.__setitem__('caucase-http-url', '${request-balancer:connection-caucase-http-url}' ) -%}
 
 {% endif -%}{# if zope_partition_dict -#}
 

stack/erp5/instance-zope.cfg.in

@@ -81,7 +81,6 @@ environment +=
   MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
   PYTHONUNBUFFERED=1
   INSTANCE_HOME=${:instance-home}
-  CAUCASE={{ slapparameter_dict['caucase-url'] }}
   FONTCONFIG_FILE=${fontconfig-conf:output}
   JUPYTER_PATH=${directory:jupyter-dir}
   JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir}