software/erp5: remove httpd and use haproxy instead

software/erp5/test/setup.py

@@ -53,6 +53,7 @@ setup(name=name,
         'pexpect',
         'pyOpenSSL',
         'typing; python_version<"3"',
+        'waitress',
       ],
       test_suite='test',
     )

software/erp5/test/test/test_balancer.py

@@ -2,6 +2,7 @@ import glob
 import hashlib
 import json
 import logging
+import multiprocessing
 import os
 import re
 import shutil
@@ -19,6 +20,7 @@ import OpenSSL.SSL
 import pexpect
 import psutil
 import requests
+import waitress
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
@@ -55,6 +57,28 @@ class EchoHTTPServer(ManagedHTTPServer):
     log_message = logging.getLogger(__name__ + '.HeaderEchoHandler').info
 
 
+class WaitressServer(ManagedHTTPServer):
+  """A managed http server using waitress, which keeps http connections open.
+  """
+  def _makeServer(self):
+    def wsgiapp(environ, start_response):
+      start_response('200 OK', [('content-type', 'text/plain')])
+      return ["OK"]
+
+    hostname = self.hostname
+    port = self.port
+    class Waitress_BaseHTTPServer:
+      """Expose minimal API compatilibty to act as a BaseHTTPServer
+      """
+      def serve_forever(self):
+        return waitress.serve(
+            wsgiapp,
+            host=hostname,
+            port=port
+        )
+    return Waitress_BaseHTTPServer()
+
+
 class CaucaseService(ManagedResource):
   """A caucase service.
   """
@@ -196,10 +220,10 @@ class TestAccessLog(BalancerTestCase, CrontabMixin):
         verify=False,
     )
     with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
-      access_line = access_log_file.read()
+      access_line = access_log_file.read().splitlines()[-1]
     self.assertIn('/url_path', access_line)
 
-    # last \d is the request time in micro seconds, since this SlowHTTPServer
+    # last \d is the request time in milli seconds, since this SlowHTTPServer
     # sleeps for 3 seconds, it should take between 3 and 4 seconds to process
     # the request - but our test machines can be slow sometimes, so we tolerate
     # it can take up to 20 seconds.
@@ -210,8 +234,8 @@ class TestAccessLog(BalancerTestCase, CrontabMixin):
     self.assertTrue(match)
     assert match
     request_time = int(match.groups()[-1])
-    self.assertGreater(request_time, 3 * 1000 * 1000)
-    self.assertLess(request_time, 20 * 1000 * 1000)
+    self.assertGreater(request_time, 3 * 1000)
+    self.assertLess(request_time, 20 * 1000)
 
   def test_access_log_apachedex_report(self):
     # type: () -> None
@@ -351,16 +375,28 @@ class TestBalancer(BalancerTestCase):
         'backend_web_server1')
 
 
-class TestHTTP(BalancerTestCase):
-  """Check HTTP protocol
+
+class TestHTTP10(BalancerTestCase):
+  """Check HTTP protocol with a HTTP/1.0 backend
   """
   __partition_reference__ = 'h'
 
   def test_http_version(self):
     # type: () -> None
-    # https://stackoverflow.com/questions/37012486/python-3-x-how-to-get-http-version-using-requests-library/37012810
     self.assertEqual(
-        requests.get(self.default_balancer_url, verify=False).raw.version, 11)
+        subprocess.check_output([
+            'curl',
+            '--silent',
+            '--show-error',
+            '--output',
+            '/dev/null',
+            '--insecure',
+            '--write-out',
+            '%{http_version}',
+            self.default_balancer_url,
+        ]),
+        '2',
+    )
 
   def test_keep_alive(self):
     # type: () -> None
@@ -387,6 +423,18 @@ class TestHTTP(BalancerTestCase):
     ])
 
 
+class TestHTTP11(TestHTTP10):
+  """Check HTTP protocol with a HTTP/1.1 backend
+  """
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestHTTP11, cls)._getInstanceParameterDict()
+    # use a HTTP/1.1 server instead
+    parameter_dict['dummy_http_server'] = [[cls.getManagedResource("waitress_server", WaitressServer).netloc, 1, False]]
+    return parameter_dict
+
+
 class TestTLS(BalancerTestCase):
   """Check TLS
   """

software/erp5/test/test/test_erp5.py

@@ -169,33 +169,22 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
         3 + 5,
         len([p for p in all_process_info if p['name'].startswith('zope-')]))
 
-  def test_apache_listen(self):
-    # We have 2 families, apache should listen to a total of 3 ports per family
+  def test_haproxy_listen(self):
+    # We have 2 families, haproxy should listen to a total of 3 ports per family
     # normal access on ipv4 and ipv6 and test runner access on ipv4 only
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
-    apache_process = psutil.Process(process_info['pid'])
+    process_info, = [p for p in all_process_info if p['name'].startswith('haproxy-')]
+    haproxy_master_process = psutil.Process(process_info['pid'])
+    haproxy_worker_process, = haproxy_master_process.children()
     self.assertEqual(
         sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
         sorted([
             c.family
-            for c in apache_process.connections()
+            for c in haproxy_worker_process.connections()
             if c.status == 'LISTEN'
         ]))
 
-  def test_haproxy_listen(self):
-    # There is one haproxy per family
-    with self.slap.instance_supervisor_rpc as supervisor:
-      all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [
-        p for p in all_process_info if p['name'].startswith('haproxy-')
-    ]
-    haproxy_process = psutil.Process(process_info['pid'])
-    self.assertEqual([socket.AF_INET, socket.AF_INET], [
-        c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
-    ])
-
 
 class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """Test ERP5 can be instanciated without test runner.

software/slapos-sr-testing/software.cfg

@@ -300,7 +300,7 @@ chardet = 3.0.4
 # ipaddress is patching IPAddress so IPv6 match works
 ipaddress = 1.0.22
 
-# cacuase and its dependencies
+# caucase and its dependencies
 caucase = 0.9.4
 pem = 18.2.0
 PyJWT = 1.6.4
@@ -319,3 +319,4 @@ mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
 typing = 3.7.4.3
+waitress = 1.3.0

stack/erp5/buildout.cfg

@@ -11,6 +11,7 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/haproxy/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
   ../../component/findutils/buildout.cfg
   ../../component/librsvg/buildout.cfg
   ../../component/imagemagick/buildout.cfg
@@ -179,6 +180,7 @@ context =
     key gzip_location gzip:location
     key xz_utils_location xz-utils:location
     key haproxy_location haproxy:location
+    key rsyslogd_location rsyslogd:location
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
     key jupyter_enable_default erp5-defaults:jupyter-enable-default
@@ -208,6 +210,7 @@ context =
     key template_balancer template-balancer:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
+    key template_rsyslogd_cfg template-rsyslogd-cfg:target
     key template_jupyter_cfg instance-jupyter-notebook:rendered
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
@@ -273,6 +276,9 @@ fontconfig-includes =
 [template-haproxy-cfg]
 <= download-base
 
+[template-rsyslogd-cfg]
+<= download-base
+
 [erp5-bin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/erp5-bin.git

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = 5c5250112b87a3937f939028f9594b85
+md5sum = 694221ac8ef893f4bbc50ab33649ada2
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 82dc695e212be124d60ceb1143e56b0d
+md5sum = 7552dd291b209811f6a50903da0b58ec
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,8 +90,12 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = ecf119142e6b5cd85a2ba397552d2142
+md5sum = 3749c954919647e97d52829d63fe3a1a
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in
-md5sum = fec6a312e4ef84b02837742992aaf495
+md5sum = cab75aeeae8d9ab092999d128ff22f32
+
+[template-rsyslogd-cfg]
+filename = rsyslogd.cfg.in
+md5sum = 11c3130c98af90b70e51ac9aeddce341

stack/erp5/haproxy.cfg.in

+
+{# This file configures haproxy to redirect requests from ports to specific urls.
+ # It provides TLS support for server and optionnaly for client.
+ #
+ # All parameters are given through the `parameter_dict` variable, see the
+ # list entries :
+ #
+ #     parameter_dict = { 
+ #       #  Certificate and keys in PEM format
+ #       "cert": "<file_path>",
+ #
+ #       #  CA to verify client certificates in PEM format.
+ #       #  If set, client certificates will be verified with these CAs.
+ #       #  If not set, client certificates are not verified.
+ #       "ca-cert": "<file_path>",
+ # 
+ #       #  An optional CRL in PEM format (the file can contain multiple CRL)
+ #       #  This is required if ca-cert is passed.
+ #       "crl": "<file_path>",
+ #
+ #       #  AF_UNIX socket for logs. Syslog must be listening on this socket.
+ #       "log-socket": "<file_path>",
+ #
+ #       #  AF_UNIX socket for statistics and control.
+ #       #  Haproxy will listen on this socket.
+ #       "stats-socket": "<file_path>",
+ #
+ #     #  The mapping of backends, keyed by name
+ #     "backend-dict": 
+ #        "family": {
+ #            # (ip, port, path) on balancer  -> backend famliy
+ #            ('127.0.0.1', 443, path): {
+ #                # bool
+ #                "enable-authentication": True,
+ #                "backend-list": [
+ #                    URL, connection_count, is_web_dav
+ #                 ]
+ #                }
+ #            ]
+ #    }
+ # name, (port, backend_list)
+ # backend_list => 
+ # for address, connection_count, webdav in backend_list
+
+ #       #  The list of ip which haproxy will listen to.
+ #       "ip-list": [
+ #         "0.0.0.0",
+ #         "[::1]",
+ #       ],
+ #
+ # TODO: merge these 2 in one
+ 
+ #       #  The list of backends.
+ #       "backend-list": [
+ #         # (port, unused, internal_url, enable_authentication)
+ #         (8000, _, "http://10.0.0.10:8001", True),
+ #         (8002, _, "http://10.0.0.10:8003", False),
+ #       ],
+ #
+ #       # The mapping of zope paths.
+ #       # This is a Zope specific feature.
+ #       # `enable_authentication` has same meaning as for `backend-list`.
+ #       "zope-virtualhost-monster-backend-dict": {
+ #          # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) }
+ #          ('[::1]', 8004): (
+ #            True, {
+ #              'zope-1': 'http://10.0.0.10:8001',
+ #              'zope-2': 'http://10.0.0.10:8002',
+ #            },
+ #          ),
+ #        },
+ #     }
+ #
+ #  This sample of `parameter_dict` will make haproxy listening to :
+ #  From to `backend-list`:
+ #   - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
+ #   - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
+ #  only accepting requests from clients who provide a valid TLS certificate trusted by any of the `ca-cert-list`.
+ #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
+ #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
+ #  accepting requests from any client.
+ #  In both cases, X-Forwarded-For will be stripped unless client presents a verified certificate.
+ #
+ # From zope-virtualhost-monster-backend-dict`:
+ #   - [::1]:8004 with some path based rewrite-rules redirecting to:
+ #     * http://10.0.0.10/8001 when path matches /zope-1(.*)
+ #     * http://10.0.0.10/8002 when path matches /zope-2(.*)
+ #   with some VirtualHostMonster rewrite rules so zope writes URLs with
+ #  [::1]:8004 as server name.
+ #  For more details, refer to
+ #  https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
+-#}
+
 {% set server_check_path = parameter_dict['server-check-path'] -%}
 global
   maxconn 4096
-  stats socket {{ parameter_dict['socket-path'] }} level admin
+  master-worker
+  pidfile {{ parameter_dict['pidfile'] }}
+
+  # SSL configuration was generated with mozilla SSL Configuration Generator
+  # generated 2020-10-28, Mozilla Guideline v5.6, HAProxy 2.1, OpenSSL 1.1.1g, modern configuration
+  # https://ssl-config.mozilla.org/#server=haproxy&version=2.1&config=modern&openssl=1.1.1g&guideline=5.6
+  ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+  ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+  ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+  ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+
+  stats socket {{ parameter_dict['stats-socket'] }} level admin
+
 
 defaults
   mode http
   retries 1
   option redispatch
   maxconn 2000
-  cookie SERVERID rewrite
   balance roundrobin
+
+  # TODO disable
   stats uri /haproxy
   stats realm Global\ statistics
-  # it is useless to have timeout much bigger than the one of apache.
-  # By default apache use 300s, so we set slightly more in order to
-  # make sure that apache will first stop the connection.
-  timeout server 305s
-  # Stop waiting in queue for a zope to become available.
-  # If no zope can be reached after one minute, consider the request will
-  # never succeed.
+
+  # TODO ?  
+#  option abortonclose
+
+  timeout check 5s
+  timeout connect 10s
   timeout queue 60s
-  # The connection should be immediate on LAN,
-  # so we should not set more than 5 seconds, and it could be already too much
-  timeout connect 5s
-  # As requested in haproxy doc, make this "at least equal to timeout server".
+  timeout server 305s
   timeout client 305s
-  # Use "option httpclose" to not preserve client & server persistent connections
-  # while handling every incoming request individually, dispatching them one after
-  # another to servers, in HTTP close mode. This is really needed when haproxy
-  # is configured with maxconn to 1, without this option browsers are unable
-  # to render a page
-  option httpclose
-
-{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
-listen {{ name }}
-  bind {{ parameter_dict['ip'] }}:{{ port }}
+
+	option http-server-close
+
+  # compress some text content types
+  compression algo gzip
+  compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
+
+  # access logs ( TODO: adjust level ?)
+  log {{ parameter_dict['log-socket'] }} local0
+
+{% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~  ' alpn h2,http/1.1' %}
+
+{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
+listen family_{{ name }}
+{%-  if parameter_dict.get('ca-cert') -%}
+{%-    set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
+{%-  else %}
+{%-    set ssl_auth = '' %}
+{%-  endif %}
+  bind {{ parameter_dict['ipv4'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
+  bind {{ parameter_dict['ipv6'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
+  cookie SERVERID rewrite
   http-request set-header X-Balancer-Current-Cookie SERVERID
+
+  # remove X-Forwarded-For unless client presented a verified certificate
+  acl client_cert_verified ssl_c_used ssl_c_verify 0
+  http-request del-header X-Forwarded-For unless client_cert_verified
+  # set Remote-User if client presented a verified certificate
+  http-request del-header Remote-User
+  http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if client_cert_verified
+
+  # logs
+  capture request header Referer len 512
+  capture request header User-Agent len 512
+  log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
+
 {%   set has_webdav = [] -%}
 {%   for address, connection_count, webdav in backend_list -%}
 {%     if webdav %}{% do has_webdav.append(None) %}{% endif -%}
-{%     set server_name = name ~ '-' ~ loop.index0 -%}
+{%     set server_name = name ~ '-' ~ loop.index0 %}
   server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}
-{%   endfor -%}
+{%-  endfor -%}
 {%-  if not has_webdav and server_check_path %}
   option httpchk GET {{ server_check_path }}
-{%   endif -%}
+{%-   endif %}
+
+{% endfor %}
+
+
+{% for (ip, port), (_, backend_dict) in sorted(parameter_dict['zope-virtualhost-monster-backend-dict'].iteritems()) -%}
+{%   set group_name = 'testrunner_' ~ loop.index0 %}
+frontend frontend_{{ group_name }}
+  bind {{ ip }}:{{ port }} {{ bind_ssl_crt }}
+{%   for name in sorted(backend_dict.keys()) %}
+  use_backend backend_{{ group_name }}_{{ name }} if { path -m beg /{{ name }} }
+{%-   endfor %}
+
+{%   for name, url in sorted(backend_dict.items()) %}
+backend backend_{{ group_name }}_{{ name }}
+  server {{ name }} {{ urlparse.urlparse(url).netloc }}
+{%-  endfor %}
 {% endfor %}

stack/erp5/instance-erp5.cfg.in

@@ -333,7 +333,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
+'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet --logformat=\'%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %msT\'')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/instance.cfg.in

@@ -59,10 +59,13 @@ openssl-location = {{ openssl_location }}
 apache = {{ apache_location }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
+rsyslogd = {{ rsyslogd_location }}
 apachedex-location = {{ bin_directory }}/apachedex
 run-apachedex-location = {{ bin_directory }}/runApacheDex
 promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
 template-haproxy-cfg = {{ template_haproxy_cfg }}
+template-rsyslogd-cfg = {{ template_rsyslogd_cfg }}
+# TODO drop
 template-apache-conf = {{ template_apache_conf }}
 
 [dynamic-template-balancer]

stack/erp5/rsyslogd.cfg.in

+module(
+    load="imuxsock"
+    SysSock.Name="{{ parameter_dict['log-socket'] }}")
+
+# Just simply output the raw line without any additional information, as
+# haproxy emits enough information by itself
+# Also cut out first empty space in msg, which is related to rsyslogd
+# internal and end up cutting on 8k, as it's default of $MaxMessageSize
+template(name="rawoutput" type="string" string="%msg:2:8192%\n")
+$ActionFileDefaultTemplate rawoutput
+
+$FileCreateMode 0600
+$DirCreateMode 0700
+$Umask 0022
+
+$WorkDirectory {{ parameter_dict['spool-directory'] }}
+
+*.* {{ parameter_dict['log-file'] }}