software/erp5: remove httpd and use haproxy instead

software/erp5/test/setup.py

@@ -53,6 +53,7 @@ setup(name=name,
         'pexpect',
         'pyOpenSSL',
         'typing; python_version<"3"',
+        'waitress',
       ],
       test_suite='test',
     )

software/erp5/test/test/test_balancer.py

@@ -2,6 +2,7 @@ import glob
 import hashlib
 import json
 import logging
+import multiprocessing
 import os
 import re
 import shutil
@@ -19,6 +20,7 @@ import OpenSSL.SSL
 import pexpect
 import psutil
 import requests
+import waitress
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
@@ -55,6 +57,28 @@ class EchoHTTPServer(ManagedHTTPServer):
     log_message = logging.getLogger(__name__ + '.HeaderEchoHandler').info
 
 
+class WaitressServer(ManagedHTTPServer):
+  """A managed http server using waitress, which keeps http connections open.
+  """
+  def _makeServer(self):
+    def wsgiapp(environ, start_response):
+      start_response('200 OK', [('content-type', 'text/plain')])
+      return ["OK"]
+
+    hostname = self.hostname
+    port = self.port
+    class Waitress_BaseHTTPServer:
+      """Expose minimal API compatilibty to act as a BaseHTTPServer
+      """
+      def serve_forever(self):
+        return waitress.serve(
+            wsgiapp,
+            host=hostname,
+            port=port
+        )
+    return Waitress_BaseHTTPServer()
+
+
 class CaucaseService(ManagedResource):
   """A caucase service.
   """
@@ -196,10 +220,10 @@ class TestAccessLog(BalancerTestCase, CrontabMixin):
         verify=False,
     )
     with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
-      access_line = access_log_file.read()
+      access_line = access_log_file.read().splitlines()[-1]
     self.assertIn('/url_path', access_line)
 
-    # last \d is the request time in micro seconds, since this SlowHTTPServer
+    # last \d is the request time in milli seconds, since this SlowHTTPServer
     # sleeps for 3 seconds, it should take between 3 and 4 seconds to process
     # the request - but our test machines can be slow sometimes, so we tolerate
     # it can take up to 20 seconds.
@@ -210,8 +234,8 @@ class TestAccessLog(BalancerTestCase, CrontabMixin):
     self.assertTrue(match)
     assert match
     request_time = int(match.groups()[-1])
-    self.assertGreater(request_time, 3 * 1000 * 1000)
-    self.assertLess(request_time, 20 * 1000 * 1000)
+    self.assertGreater(request_time, 3 * 1000)
+    self.assertLess(request_time, 20 * 1000)
 
   def test_access_log_apachedex_report(self):
     # type: () -> None
@@ -351,16 +375,28 @@ class TestBalancer(BalancerTestCase):
         'backend_web_server1')
 
 
-class TestHTTP(BalancerTestCase):
-  """Check HTTP protocol
+
+class TestHTTP10(BalancerTestCase):
+  """Check HTTP protocol with a HTTP/1.0 backend
   """
   __partition_reference__ = 'h'
 
   def test_http_version(self):
     # type: () -> None
-    # https://stackoverflow.com/questions/37012486/python-3-x-how-to-get-http-version-using-requests-library/37012810
     self.assertEqual(
-        requests.get(self.default_balancer_url, verify=False).raw.version, 11)
+        subprocess.check_output([
+            'curl',
+            '--silent',
+            '--show-error',
+            '--output',
+            '/dev/null',
+            '--insecure',
+            '--write-out',
+            '%{http_version}',
+            self.default_balancer_url,
+        ]),
+        '2',
+    )
 
   def test_keep_alive(self):
     # type: () -> None
@@ -387,6 +423,18 @@ class TestHTTP(BalancerTestCase):
     ])
 
 
+class TestHTTP11(TestHTTP10):
+  """Check HTTP protocol with a HTTP/1.1 backend
+  """
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestHTTP11, cls)._getInstanceParameterDict()
+    # use a HTTP/1.1 server instead
+    parameter_dict['dummy_http_server'] = [[cls.getManagedResource("waitress_server", WaitressServer).netloc, 1, False]]
+    return parameter_dict
+
+
 class TestTLS(BalancerTestCase):
   """Check TLS
   """

software/erp5/test/test/test_erp5.py

@@ -169,33 +169,22 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         3 + 5,
         len([p for p in all_process_info if p['name'].startswith('zope-')]))
 
-  def test_apache_listen(self):
-    # We have 2 families, apache should listen to a total of 3 ports per family
+  def test_haproxy_listen(self):
+    # We have 2 families, haproxy should listen to a total of 3 ports per family
     # normal access on ipv4 and ipv6 and test runner access on ipv4 only
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
+    process_info, = [p for p in all_process_info if p['name'].startswith('haproxy-')]
+    haproxy_master_process = psutil.Process(process_info['pid'])
+    haproxy_worker_process, = haproxy_master_process.children()
     self.assertEqual(
         sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
         sorted([
             c.family
-            for c in apache_process.connections()
+            for c in haproxy_worker_process.connections()
             if c.status == 'LISTEN'
         ]))
 
-  def test_haproxy_listen(self):
-    # There is one haproxy per family
-    with self.slap.instance_supervisor_rpc as supervisor:
-      all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [
-        p for p in all_process_info if p['name'].startswith('haproxy-')
-    ]
-    haproxy_process = psutil.Process(process_info['pid'])
-    self.assertEqual([socket.AF_INET, socket.AF_INET], [
-        c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
-    ])
-
 
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instanciated without test runner.

software/slapos-sr-testing/software.cfg

@@ -300,7 +300,7 @@ chardet = 3.0.4
 # ipaddress is patching IPAddress so IPv6 match works
 ipaddress = 1.0.22
 
-# cacuase and its dependencies
+# caucase and its dependencies
 caucase = 0.9.4
 pem = 18.2.0
 PyJWT = 1.6.4
@@ -319,3 +319,4 @@ mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
 typing = 3.7.4.3
+waitress = 1.3.0

stack/erp5/buildout.cfg

@@ -11,6 +11,7 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/haproxy/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
   ../../component/findutils/buildout.cfg
   ../../component/librsvg/buildout.cfg
   ../../component/imagemagick/buildout.cfg
@@ -179,6 +180,7 @@ context =
     key gzip_location gzip:location
     key xz_utils_location xz-utils:location
     key haproxy_location haproxy:location
+    key rsyslogd_location rsyslogd:location
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
     key jupyter_enable_default erp5-defaults:jupyter-enable-default
@@ -208,6 +210,7 @@ context =
     key template_balancer template-balancer:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_rsyslogd_cfg template-rsyslogd-cfg:target
     key template_jupyter_cfg instance-jupyter-notebook:rendered
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
@@ -273,6 +276,9 @@ fontconfig-includes =
 [template-haproxy-cfg]
 <= download-base
 
+[template-rsyslogd-cfg]
+<= download-base
+
 [erp5-bin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/erp5-bin.git

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = 5c5250112b87a3937f939028f9594b85
+md5sum = 694221ac8ef893f4bbc50ab33649ada2
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 82dc695e212be124d60ceb1143e56b0d
+md5sum = 7552dd291b209811f6a50903da0b58ec
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,8 +90,12 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = ecf119142e6b5cd85a2ba397552d2142
+md5sum = 3749c954919647e97d52829d63fe3a1a
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = fec6a312e4ef84b02837742992aaf495
+md5sum = cab75aeeae8d9ab092999d128ff22f32
+
+[template-rsyslogd-cfg]
+filename = rsyslogd.cfg.in
+md5sum = 11c3130c98af90b70e51ac9aeddce341

stack/erp5/haproxy.cfg.in

+
+{# This file configures haproxy to redirect requests from ports to specific urls.
+ # It provides TLS support for server and optionnaly for client.
+ #
+ # All parameters are given through the `parameter_dict` variable, see the
+ # list entries :
+ #
+ #     parameter_dict = { 
+ #       #  Certificate and keys in PEM format
+ #       "cert": "<file_path>",
+ #
+ #       #  CA to verify client certificates in PEM format.
+ #       #  If set, client certificates will be verified with these CAs.
+ #       #  If not set, client certificates are not verified.
+ #       "ca-cert": "<file_path>",
+ # 
+ #       #  An optional CRL in PEM format (the file can contain multiple CRL)
+ #       #  This is required if ca-cert is passed.
+ #       "crl": "<file_path>",
+ #
+ #       #  AF_UNIX socket for logs. Syslog must be listening on this socket.
+ #       "log-socket": "<file_path>",
+ #
+ #       #  AF_UNIX socket for statistics and control.
+ #       #  Haproxy will listen on this socket.
+ #       "stats-socket": "<file_path>",
+ #
+ #     #  The mapping of backends, keyed by name
+ #     "backend-dict": 
+ #        "family": {
+ #            # (ip, port, path) on balancer  -> backend famliy
+ #            ('127.0.0.1', 443, path): {
+ #                # bool
+ #                "enable-authentication": True,
+ #                "backend-list": [
+ #                    URL, connection_count, is_web_dav
+ #                 ]
+ #                }
+ #            ]
+ #    }
+ # name, (port, backend_list)
+ # backend_list => 
+ # for address, connection_count, webdav in backend_list
+
+ #       #  The list of ip which haproxy will listen to.
+ #       "ip-list": [
+ #         "0.0.0.0",
+ #         "[::1]",
+ #       ],
+ #
+ # TODO: merge these 2 in one
+ 
+ #       #  The list of backends.
+ #       "backend-list": [
+ #         # (port, unused, internal_url, enable_authentication)
+ #         (8000, _, "http://10.0.0.10:8001", True),
+ #         (8002, _, "http://10.0.0.10:8003", False),
+ #       ],
+ #
+ #       # The mapping of zope paths.
+ #       # This is a Zope specific feature.
+ #       # `enable_authentication` has same meaning as for `backend-list`.
+ #       "zope-virtualhost-monster-backend-dict": {
+ #          # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) }
+ #          ('[::1]', 8004): (
+ #            True, {
+ #              'zope-1': 'http://10.0.0.10:8001',
+ #              'zope-2': 'http://10.0.0.10:8002',
+ #            },
+ #          ),
+ #        },
+ #     }
+ #
+ #  This sample of `parameter_dict` will make haproxy listening to :
+ #  From to `backend-list`:
+ #   - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
+ #   - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
+ #  only accepting requests from clients who provide a valid TLS certificate trusted by any of the `ca-cert-list`.
+ #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
+ #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
+ #  accepting requests from any client.
+ #  In both cases, X-Forwarded-For will be stripped unless client presents a verified certificate.
+ #
+ # From zope-virtualhost-monster-backend-dict`:
+ #   - [::1]:8004 with some path based rewrite-rules redirecting to:
+ #     * http://10.0.0.10/8001 when path matches /zope-1(.*)
+ #     * http://10.0.0.10/8002 when path matches /zope-2(.*)
+ #   with some VirtualHostMonster rewrite rules so zope writes URLs with
+ #  [::1]:8004 as server name.
+ #  For more details, refer to
+ #  https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
+-#}
+
 {% set server_check_path = parameter_dict['server-check-path'] -%}
 global
   maxconn 4096
-  stats socket {{ parameter_dict['socket-path'] }} level admin
+  master-worker
+  pidfile {{ parameter_dict['pidfile'] }}
+
+  # SSL configuration was generated with mozilla SSL Configuration Generator
+  # generated 2020-10-28, Mozilla Guideline v5.6, HAProxy 2.1, OpenSSL 1.1.1g, modern configuration
+  # https://ssl-config.mozilla.org/#server=haproxy&version=2.1&config=modern&openssl=1.1.1g&guideline=5.6
+  ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+  ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+  ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+  ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+
+  stats socket {{ parameter_dict['stats-socket'] }} level admin
+
 
 defaults
   mode http
   retries 1
   option redispatch
   maxconn 2000
-  cookie SERVERID rewrite
   balance roundrobin
+
+  # TODO disable
   stats uri /haproxy
   stats realm Global\ statistics
-  # it is useless to have timeout much bigger than the one of apache.
-  # By default apache use 300s, so we set slightly more in order to
-  # make sure that apache will first stop the connection.
-  timeout server 305s
-  # Stop waiting in queue for a zope to become available.
-  # If no zope can be reached after one minute, consider the request will
-  # never succeed.
+
+  # TODO ?  
+#  option abortonclose
+
+  timeout check 5s
+  timeout connect 10s
   timeout queue 60s
-  # The connection should be immediate on LAN,
-  # so we should not set more than 5 seconds, and it could be already too much
-  timeout connect 5s
-  # As requested in haproxy doc, make this "at least equal to timeout server".
+  timeout server 305s
   timeout client 305s
-  # Use "option httpclose" to not preserve client & server persistent connections
-  # while handling every incoming request individually, dispatching them one after
-  # another to servers, in HTTP close mode. This is really needed when haproxy
-  # is configured with maxconn to 1, without this option browsers are unable
-  # to render a page
-  option httpclose
-
-{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
-listen {{ name }}
-  bind {{ parameter_dict['ip'] }}:{{ port }}
+
+	option http-server-close
+
+  # compress some text content types
+  compression algo gzip
+  compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
+
+  # access logs ( TODO: adjust level ?)
+  log {{ parameter_dict['log-socket'] }} local0
+
+{% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~  ' alpn h2,http/1.1' %}
+
+{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
+listen family_{{ name }}
+{%-  if parameter_dict.get('ca-cert') -%}
+{%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
+{%-  else %}
+{%-    set ssl_auth = '' %}
+{%-  endif %}
+  bind {{ parameter_dict['ipv4'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
+  bind {{ parameter_dict['ipv6'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
+  cookie SERVERID rewrite
   http-request set-header X-Balancer-Current-Cookie SERVERID
+
+  # remove X-Forwarded-For unless client presented a verified certificate
+  acl client_cert_verified ssl_c_used ssl_c_verify 0
+  http-request del-header X-Forwarded-For unless client_cert_verified
+  # set Remote-User if client presented a verified certificate
+  http-request del-header Remote-User
+  http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if client_cert_verified
+
+  # logs
+  capture request header Referer len 512
+  capture request header User-Agent len 512
+  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
+
 {%   set has_webdav = [] -%}
 {%   for address, connection_count, webdav in backend_list -%}
 {%     if webdav %}{% do has_webdav.append(None) %}{% endif -%}
-{%     set server_name = name ~ '-' ~ loop.index0 -%}
+{%     set server_name = name ~ '-' ~ loop.index0 %}
   server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}
-{%   endfor -%}
+{%-  endfor -%}
 {%-  if not has_webdav and server_check_path %}
   option httpchk GET {{ server_check_path }}
-{%   endif -%}
+{%-   endif %}
+
+{% endfor %}
+
+
+{% for (ip, port), (_, backend_dict) in sorted(parameter_dict['zope-virtualhost-monster-backend-dict'].iteritems()) -%}
+{%   set group_name = 'testrunner_' ~ loop.index0 %}
+frontend frontend_{{ group_name }}
+  bind {{ ip }}:{{ port }} {{ bind_ssl_crt }}
+{%   for name in sorted(backend_dict.keys()) %}
+  use_backend backend_{{ group_name }}_{{ name }} if { path -m beg /{{ name }} }
+{%-   endfor %}
+
+{%   for name, url in sorted(backend_dict.items()) %}
+backend backend_{{ group_name }}_{{ name }}
+  server {{ name }} {{ urlparse.urlparse(url).netloc }}
+{%-  endfor %}
 {% endfor %}

stack/erp5/instance-balancer.cfg.in

@@ -43,7 +43,7 @@ command = {{ parameter_dict["openssl"] }}/bin/openssl req \
   -batch \
   -new \
   -nodes \
-  -keyout '${apache-conf-ssl:key}' \
+  -keyout '${tls:certificate}' \
   -config '${balancer-csr-request-config:rendered}' \
   -out '${:csr}'
 stop-on-error = true
@@ -56,11 +56,11 @@ csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
      updater_path='${directory:services-on-watch}/caucase-updater',
      url=ssl_parameter_dict['caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:cert}',
+     crt_path='${tls:certificate}',
      ca_path='${directory:srv}/caucase-updater/ca.crt',
      crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:key}',
-     on_renew='${apache-graceful:output}',
+     key_path='${tls:certificate}',
+     on_renew='${haproxy-reload:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
      template_csr_pem=ssl_parameter_dict.get('csr'),
      template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-request:csr}',
@@ -73,7 +73,7 @@ csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
 {% for frontend_caucase_url in frontend_caucase_url_list -%}
 {%   set hash = hashlib.md5(frontend_caucase_url).hexdigest() -%}
 {%   do frontend_caucase_url_hash_list.append(hash) -%}
-{%   set data_dir = '${directory:srv}/client-cert-ca/%s' % hash -%}
+{%   set data_dir = '${directory:client-cert-ca}/%s' % hash -%}
 {{   caucase.updater(
        prefix='caucase-updater-%s' % hash,
        buildout_bin_directory=parameter_dict['bin-directory'],
@@ -82,13 +82,14 @@ csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
        data_dir=data_dir,
        ca_path='%s/ca.crt' % data_dir,
        crl_path='%s/crl.pem' % data_dir,
-       on_renew='${caucase-updater-housekeeper:output}; ${apache-graceful:output}',
+       on_renew='${caucase-updater-housekeeper:output}',
        max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
        openssl=parameter_dict['openssl'] ~ '/bin/openssl',
      )}}
 {%   do section('caucase-updater-%s' % hash) -%}
 {% endfor -%}
 
+
 {% if frontend_caucase_url_hash_list -%}
 [caucase-updater-housekeeper]
 recipe = collective.recipe.template
@@ -100,26 +101,51 @@ input =
   import glob
   import os
   import subprocess
+
   hash_list = {{ repr(frontend_caucase_url_hash_list) }}
+
   crt_list = ['%s.crt' % e for e in hash_list]
-  crl_list = ['%s.crl' % e for e in hash_list]
-  for path in glob.glob('${apache-conf-ssl:ca-cert-dir}/*.crt'):
+  for path in glob.glob('${tls:ca-cert-dir}/*.crt'):
     if os.path.basename(path) not in crt_list:
       os.unlink(path)
-  for path in glob.glob('${apache-conf-ssl:crl-dir}/*.crl'):
+  crl_list = ['%s.crl' % e for e in hash_list]
+  for path in glob.glob('${tls:crl-dir}/*.crl'):
     if os.path.basename(path) not in crl_list:
       os.unlink(path)
+
   for hash in hash_list:
-    crt = '${directory:srv}/client-cert-ca/%s/ca.crt' % hash
-    crt_link = '${apache-conf-ssl:ca-cert-dir}/%s.crt' % hash
-    crl = '${directory:srv}/client-cert-ca/%s/crl.pem' % hash
-    crl_link = '${apache-conf-ssl:crl-dir}/%s.crl' % hash
+    crt = '${directory:client-cert-ca}/%s/ca.crt' % hash
+    crt_link = '${tls:ca-cert-dir}/%s.crt' % hash
+    crl = '${directory:client-cert-ca}/%s/crl.pem' % hash
+    crl_link = '${tls:crl-dir}/%s.crl' % hash
     if os.path.isfile(crt) and not os.path.islink(crt_link):
       os.symlink(crt, crt_link)
     if os.path.isfile(crl) and not os.path.islink(crl_link):
       os.symlink(crl, crl_link)
-  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:ca-cert-dir}'])
-  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${apache-conf-ssl:crl-dir}'])
+  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${tls:ca-cert-dir}'])
+  subprocess.check_call(['{{ parameter_dict["openssl"] }}/bin/c_rehash', '${tls:crl-dir}'])
+
+  # assemble all CA and all CRLs in one file for haproxy
+  with open('${tls:ca-cert}.tmp', 'w') as f:
+    for path in glob.glob('${tls:ca-cert-dir}/*.crt'):
+      with open(path) as in_f:
+        f.write('#{}\n'.format(path))
+        f.write(in_f.read() + '\n')
+  with open('${tls:crl}.tmp', 'w') as f:
+    for path in glob.glob('${tls:crl-dir}/*.crl'):
+      with open(path) as in_f:
+        f.write('#{}\n'.format(path))
+        f.write(in_f.read() + '\n')
+
+  if os.path.exists('${tls:ca-cert}'):
+    os.unlink('${tls:ca-cert}')
+  if os.path.exists('${tls:crl}'):
+    os.unlink('${tls:crl}')
+  os.rename('${tls:ca-cert}.tmp', '${tls:ca-cert}')
+  os.rename('${tls:crl}.tmp', '${tls:crl}')
+
+  subprocess.check_call(['${haproxy-reload:output}'])
+
 
 [caucase-updater-housekeeper-run]
 recipe = plone.recipe.command
@@ -174,47 +200,94 @@ update-command = ${:command}
  # do a no-op getitem.
 -#}
 {%   do zope_family_address_list[0][0] -%}
+
 {%   set haproxy_port = next_port() -%}
 {%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
-{%   do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
 {%   if has_webdav -%}
-{%     set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
 {%     set external_scheme = 'webdavs' -%}
 {%   else %}
-{%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, slapparameter_dict['ssl-authentication-dict'].get(family_name, False))) -%}
+{%   do haproxy_dict.__setitem__(family_name, (haproxy_port, external_scheme, slapparameter_dict['ssl-authentication-dict'].get(family_name, False), zope_family_address_list)) -%}
 {% endfor -%}
 
 [haproxy-cfg-parameter-dict]
-socket-path = ${directory:run}/haproxy.sock
+ipv4 = {{ ipv4 }}
+ipv6 = {{ ipv6 }}
+cert = ${tls:certificate}
+{% if frontend_caucase_url_list -%}
+ca-cert = ${tls:ca-cert}
+crl = ${tls:crl}
+{% endif %}
+stats-socket = ${directory:run}/haproxy.sock
+pidfile = ${directory:run}/haproxy.pid
+log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
 server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
 backend-dict = {{ dumps(haproxy_dict) }}
-ip = {{ ipv4 }}
+zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
+
 
 [haproxy-cfg]
 < = jinja2-template-base
 template = {{ parameter_dict['template-haproxy-cfg'] }}
 rendered = ${directory:etc}/haproxy.cfg
-context = section parameter_dict haproxy-cfg-parameter-dict
+context =
+  section parameter_dict haproxy-cfg-parameter-dict
+  import urlparse urlparse
 extensions = jinja2.ext.do
 
+[haproxy-reload]
+recipe = collective.recipe.template
+output = ${directory:bin}/${:_buildout_section_name_}
+mode = 700
+input =
+  inline:
+  #!/bin/sh
+  kill -HUP $(cat "${haproxy-cfg-parameter-dict:pidfile}")
+
 [{{ section('haproxy') }}]
 recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:services}/haproxy
+wrapper-path = ${directory:services-on-watch}/haproxy
 command-line = "{{ parameter_dict['haproxy'] }}/sbin/haproxy" -f "${haproxy-cfg:rendered}"
 hash-files = ${haproxy-cfg:rendered}
 
-[apache-conf-ssl]
-cert = ${directory:apache-conf}/apache.crt
-key = ${directory:apache-conf}/apache.pem
+[rsyslogd-cfg-parameter-dict]
+log-socket = ${directory:run}/log.sock
+log-file = ${directory:log}/apache-access.log
+pid-file = ${directory:run}/rsyslogd.pid
+spool-directory = ${directory:rsyslogd-spool}
+
+[rsyslogd-cfg]
+<= jinja2-template-base
+template = {{ parameter_dict['template-rsyslogd-cfg'] }}
+rendered = ${directory:etc}/rsyslogd.conf
+context = section parameter_dict rsyslogd-cfg-parameter-dict
+
+[{{ section ('rsyslogd') }}]
+recipe = slapos.cookbook:wrapper
+command-line = {{ parameter_dict['rsyslogd'] }}/sbin/rsyslogd -i ${rsyslogd-cfg-parameter-dict:pid-file} -n -f ${rsyslogd-cfg:rendered}
+wrapper-path = ${directory:services-on-watch}/rsyslogd
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+hash-files = ${rsyslogd-cfg:rendered}
+
+[{{ section ('rsyslogd-listen-promise') }}]
+<= monitor-promise-base
+module = check_command_execute
+name = rsyslogd_listen_promise.py
+config-command = test -S ${rsyslogd-cfg-parameter-dict:log-socket}
+
+
+[tls]
+certificate = ${directory:etc}/certificate-and-key.pem
 {% if frontend_caucase_url_list -%}
+ca-cert = ${directory:etc}/frontend-ca.pem
+ca-cert-dir = ${directory:ca-cert}
+crl = ${directory:etc}/frontend-crl.pem
+crl-dir = ${directory:crl}
 depends = ${caucase-updater-housekeeper-run:recipe}
-ca-cert-dir = ${directory:apache-ca-cert-dir}
-crl-dir = ${directory:apache-crl-dir}
 {%- endif %}
 
+
 [simplefile]
 < = jinja2-template-base
 template = inline:{{ '{{ content }}' }}
@@ -231,62 +304,20 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
-[apache-conf-parameter-dict]
-backend-list = {{ dumps(apache_dict.values()) }}
-zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
-ip-list = {{ dumps(apache_ip_list) }}
-pid-file = ${directory:run}/apache.pid
-log-dir = ${directory:log}
-error-log = ${directory:log}/apache-error.log
-access-log = ${directory:log}/apache-access.log
-# Apache 2.4's default value (60 seconds) can be a bit too short
-timeout = 300
-# Basic SSL server configuration
-cert = ${apache-conf-ssl:cert}
-key = ${apache-conf-ssl:key}
-cipher =
-ssl-session-cache = ${directory:log}/apache-ssl-session-cache
-{% if frontend_caucase_url_list -%}
-# Client x509 auth
-ca-cert-dir = ${apache-conf-ssl:ca-cert-dir}
-crl-dir = ${apache-conf-ssl:crl-dir}
-{%- endif %}
 
-[apache-conf]
-< = jinja2-template-base
-template = {{ parameter_dict['template-apache-conf'] }}
-rendered = ${directory:apache-conf}/apache.conf
-context = section parameter_dict apache-conf-parameter-dict
-
-[{{ section('apache') }}]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:services}/apache
-command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
-wait-for-files =
-  ${apache-conf-ssl:cert}
-  ${apache-conf-ssl:key}
-
-[apache-graceful]
-recipe = collective.recipe.template
-output = ${directory:bin}/apache-httpd-graceful
-mode = 700
-input = inline:
-  #!/bin/sh
-  kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
-
-[{{ section('apache-promise') }}]
+[{{ section('haproxy-promise') }}]
 <= monitor-promise-base
-# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
+# Check any haproxy port in ipv4, expect other ports and ipv6 to behave consistently
 module = check_port_listening
-name = apache.py
+name = haproxy.py
 config-hostname = {{ ipv4 }}
-config-port = {{ apache_dict.values()[0][0] }}
+config-port = {{ haproxy_dict.values()[0][0] }}
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
-{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
-{{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
-{{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
+{% for family_name, (port, scheme, _, _) in haproxy_dict.items() -%}
+{{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ port }}{% endif %}
+{{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ port }}
 {% endfor -%}
 {% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
 {{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
@@ -294,19 +325,15 @@ recipe = slapos.cookbook:publish.serialised
 
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
-[{{ section('logrotate-apache') }}]
+[{{ section('logrotate-rsyslogd') }}]
 < = logrotate-entry-base
 name = apache
-log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
-post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
+log = ${rsyslogd-cfg-parameter-dict:log-file}
+post = test ! -s ${rsyslogd-cfg-parameter-dict:pid-file} || kill -HUP $(cat ${rsyslogd-cfg-parameter-dict:pid-file})
+
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-apache-conf = ${:etc}/apache
-{% if frontend_caucase_url_list -%}
-apache-ca-cert-dir = ${:apache-conf}/ssl.crt
-apache-crl-dir = ${:apache-conf}/ssl.crl
-{% endif -%}
 bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
 services = ${:etc}/run
@@ -316,6 +343,12 @@ run = ${:var}/run
 log = ${:var}/log
 srv = ${buildout:directory}/srv
 apachedex = ${monitor-directory:private}/apachedex
+rsyslogd-spool = ${:run}/rsyslogd-spool
+{% if frontend_caucase_url_list -%}
+ca-cert = ${:etc}/ssl.crt
+crl = ${:etc}/ssl.crl
+client-cert-ca = ${:srv}/client-cert-ca
+{% endif -%}
 
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency
@@ -339,9 +372,7 @@ command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_di
 command = generate-apachedex-report
 
 [apachedex-parameters]
-# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
-# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
-apache-log-list = ${apache-conf-parameter-dict:access-log}
+apache-log-list = ${rsyslogd-cfg-parameter-dict:log-file}
 configuration = {{ slapparameter_dict['apachedex-configuration'] }}
 promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
 

stack/erp5/instance-erp5.cfg.in

@@ -333,7 +333,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet --logformat=\'%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %msT\'')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/instance.cfg.in

@@ -59,10 +59,13 @@ openssl-location = {{ openssl_location }}
 apache = {{ apache_location }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
+rsyslogd = {{ rsyslogd_location }}
 apachedex-location = {{ bin_directory }}/apachedex
 run-apachedex-location = {{ bin_directory }}/runApacheDex
 promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
 template-haproxy-cfg = {{ template_haproxy_cfg }}
+template-rsyslogd-cfg = {{ template_rsyslogd_cfg }}
+# TODO drop
 template-apache-conf = {{ template_apache_conf }}
 
 [dynamic-template-balancer]

stack/erp5/rsyslogd.cfg.in

+module(
+    load="imuxsock"
+    SysSock.Name="{{ parameter_dict['log-socket'] }}")
+
+# Just simply output the raw line without any additional information, as
+# haproxy emits enough information by itself
+# Also cut out first empty space in msg, which is related to rsyslogd
+# internal and end up cutting on 8k, as it's default of $MaxMessageSize
+template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+$ActionFileDefaultTemplate rawoutput
+
+$FileCreateMode 0600
+$DirCreateMode 0700
+$Umask 0022
+
+$WorkDirectory {{ parameter_dict['spool-directory'] }}
+
+*.* {{ parameter_dict['log-file'] }}