Commit bd3c2b18 authored by Jérome Perrin's avatar Jérome Perrin

Remove httpd from ERP5 and use haproxy instead

See merge request nexedi/slapos!858
parents aeece80c 82a249b6
......@@ -7,6 +7,7 @@ parts =
[socat]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz
version = 1.7.3.2
md5sum = aec3154f7854580cfab0c2d81e910519
......
......@@ -48,10 +48,6 @@ def setUpModule():
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
# ERP5 instanciation needs to run several times before being ready, as
# the root instance request more instances.
instance_max_retry = 7 # XXX how many times ?
def getRootPartitionConnectionParameterDict(self):
"""Return the output paramters from the root partition"""
return json.loads(
......
This diff is collapsed.
......@@ -43,23 +43,44 @@ setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, base_url, site_id, verify):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
# What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable when zope is not started yet, with 404 when
# erp5 site is not created, with 500 when mysql is not yet reachable, so we
# retry in a loop until we get a succesful response.
for i in range(1, 60):
r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
if r.status_code != requests.codes.ok:
delay = i * 2
self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay)
continue
r.raise_for_status()
break
# configure this requests session to retry.
# XXX we should probably add a promise instead
session = requests.Session()
session.mount(
base_url,
requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry(
total=60,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self):
......@@ -67,14 +88,20 @@ class TestPublishedURLIsReachableMixin(object):
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
param_dict['family-default-v6'],
param_dict['site-id'],
verify=False,
)
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
param_dict['family-default'],
param_dict['site-id'],
verify=False,
)
class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......@@ -93,7 +120,7 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
return {'_': json.dumps({'wsgi': False})}
class TestApacheBalancerPorts(ERP5InstanceTestCase):
class TestBalancerPorts(ERP5InstanceTestCase):
"""Instanciate with two zope families, this should create for each family:
- a balancer entry point with corresponding haproxy
- a balancer entry point for test runner
......@@ -151,33 +178,22 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
3 + 5,
len([p for p in all_process_info if p['name'].startswith('zope-')]))
def test_apache_listen(self):
# We have 2 families, apache should listen to a total of 3 ports per family
def test_haproxy_listen(self):
# We have 2 families, haproxy should listen to a total of 3 ports per family
# normal access on ipv4 and ipv6 and test runner access on ipv4 only
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
process_info, = [p for p in all_process_info if p['name'].startswith('haproxy-')]
haproxy_master_process = psutil.Process(process_info['pid'])
haproxy_worker_process, = haproxy_master_process.children()
self.assertEqual(
sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
sorted([
c.family
for c in apache_process.connections()
for c in haproxy_worker_process.connections()
if c.status == 'LISTEN'
]))
def test_haproxy_listen(self):
# There is one haproxy per family
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [
p for p in all_process_info if p['name'].startswith('haproxy-')
]
haproxy_process = psutil.Process(process_info['pid'])
self.assertEqual([socket.AF_INET, socket.AF_INET], [
c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
])
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instanciated without test runner.
......@@ -199,20 +215,22 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for test runner
def test_no_haproxy_testrunner_port(self):
# Haproxy only listen on two ports, there is no haproxy ports allocated for test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
process_info, = [p for p in all_process_info if p['name'].startswith('haproxy')]
haproxy_master_process = psutil.Process(process_info['pid'])
haproxy_worker_process, = haproxy_master_process.children()
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
for c in haproxy_worker_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
......
......@@ -23,3 +23,7 @@ md5sum = 4998e62351f54700ee23a2ca8cd89329
[template-apache-backend-conf]
filename = apache-backend.conf.in
md5sum = 9d7104ce18f79a7a84988efc11f5ed23
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = fec6a312e4ef84b02837742992aaf495
{% set server_check_path = parameter_dict['server-check-path'] -%}
global
maxconn 4096
stats socket {{ parameter_dict['socket-path'] }} level admin
defaults
mode http
retries 1
option redispatch
maxconn 2000
cookie SERVERID rewrite
balance roundrobin
stats uri /haproxy
stats realm Global\ statistics
# it is useless to have timeout much bigger than the one of apache.
# By default apache use 300s, so we set slightly more in order to
# make sure that apache will first stop the connection.
timeout server 305s
# Stop waiting in queue for a zope to become available.
# If no zope can be reached after one minute, consider the request will
# never succeed.
timeout queue 60s
# The connection should be immediate on LAN,
# so we should not set more than 5 seconds, and it could be already too much
timeout connect 5s
# As requested in haproxy doc, make this "at least equal to timeout server".
timeout client 305s
# Use "option httpclose" to not preserve client & server persistent connections
# while handling every incoming request individually, dispatching them one after
# another to servers, in HTTP close mode. This is really needed when haproxy
# is configured with maxconn to 1, without this option browsers are unable
# to render a page
option httpclose
{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
listen {{ name }}
bind {{ parameter_dict['ip'] }}:{{ port }}
http-request set-header X-Balancer-Current-Cookie SERVERID
{% set has_webdav = [] -%}
{% for address, connection_count, webdav in backend_list -%}
{% if webdav %}{% do has_webdav.append(None) %}{% endif -%}
{% set server_name = name ~ '-' ~ loop.index0 -%}
server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}
{% endfor -%}
{%- if not has_webdav and server_check_path %}
option httpchk GET {{ server_check_path }}
{% endif -%}
{% endfor %}
......@@ -63,7 +63,9 @@ filename = instance-balancer.cfg.in
[template-apache-backend-conf]
url = ${:_profile_base_location_}/${:filename}
filename = apache-backend.conf.in
[template-haproxy-cfg]
url = ${:_profile_base_location_}/${:filename}
[versions]
python-memcached = 1.47
......
......@@ -11,6 +11,8 @@ extends =
../../component/gzip/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/socat/buildout.cfg
../../component/rsyslogd/buildout.cfg
../../component/findutils/buildout.cfg
../../component/librsvg/buildout.cfg
../../component/imagemagick/buildout.cfg
......@@ -180,6 +182,8 @@ context =
key gzip_location gzip:location
key xz_utils_location xz-utils:location
key haproxy_location haproxy:location
key socat_location socat:location
key rsyslogd_location rsyslogd:location
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
......@@ -209,6 +213,7 @@ context =
key template_balancer template-balancer:target
key template_erp5 template-erp5:target
key template_haproxy_cfg template-haproxy-cfg:target
key template_rsyslogd_cfg template-rsyslogd-cfg:target
key template_jupyter_cfg instance-jupyter-notebook:rendered
key template_kumofs template-kumofs:target
key template_mariadb template-mariadb:target
......@@ -274,6 +279,9 @@ fontconfig-includes =
[template-haproxy-cfg]
<= download-base
[template-rsyslogd-cfg]
<= download-base
[erp5-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
......
......@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template]
filename = instance.cfg.in
md5sum = 5c5250112b87a3937f939028f9594b85
md5sum = 2ccfd6e2eb803a0d5e23e36a5e6c50ad
[monitor-template-dummy]
filename = dummy.cfg
......@@ -90,8 +90,12 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 4ba93d28d93bd066d5d19f4f74fc13d7
md5sum = 4a119083eab1eadbaf44468eb4f3381f
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = fec6a312e4ef84b02837742992aaf495
md5sum = 8de18a61607bd66341a44b95640d293f
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
md5sum = 7030e42b50e03f24e036b7785bd6159f
{# This file configures haproxy to redirect requests from ports to specific urls.
# It provides TLS support for server and optionnaly for client.
#
# All parameters are given through the `parameter_dict` variable, see the
# list entries :
#
# parameter_dict = {
# # Path of the PID file. HAProxy will write its own PID to this file
# # Sending USR2 signal to this pid will cause haproxy to reload
# # its configuration.
# "pidfile": "<file_path>",
#
# # AF_UNIX socket for logs. Syslog must be listening on this socket.
# "log-socket": "<file_path>",
#
# # AF_UNIX socket for statistics and control.
# # Haproxy will listen on this socket.
# "stats-socket": "<file_path>",
#
# # IPv4 to listen on
# # All backends from `backend-dict` will listen on this IP.
# "ipv4": "0.0.0.0",
#
# # IPv6 to listen on
# # All backends from `backend-dict` will listen on this IP.
# "ipv6": "::1",
#
# # Certificate and key in PEM format. All ports will serve TLS using
# # this certificate.
# "cert": "<file_path>",
#
# # CA to verify client certificates in PEM format.
# # If set, client certificates will be verified with these CAs.
# # If not set, client certificates are not verified.
# "ca-cert": "<file_path>",
#
# # An optional CRL in PEM format (the file can contain multiple CRL)
# # This is required if ca-cert is passed.
# "crl": "<file_path>",
#
# # Path to use for HTTP health check on backends from `backend-dict`.
# "server-check-path": "/",
#
# # The mapping of backends, keyed by family name
# "backend-dict": {
# "family-secure": {
# ( 8000, # port int
# 'https', # proto str
# True, # ssl_required bool
# [ # backends
# '10.0.0.10:8001', # netloc str
# 1, # max_connection_count int
# False, # is_web_dav bool
# ],
# ),
# },
# "family-default": {
# ( 8002, # port int
# 'https', # proto str
# False, # ssl_required bool
# [ # backends
# '10.0.0.10:8003', # netloc str
# 1, # max_connection_count int
# False, # is_web_dav bool
# ],
# ),
# },
#
# # The mapping of zope paths.
# # This is a Zope specific feature.
# # `enable_authentication` has same meaning as for `backend-list`.
# "zope-virtualhost-monster-backend-dict": {
# # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) }
# ('[::1]', 8004): (
# True, {
# 'zope-1': 'http://10.0.0.10:8001',
# 'zope-2': 'http://10.0.0.10:8002',
# },
# ),
# },
# }
#
# This sample of `parameter_dict` will make haproxy listening to :
# From to `backend-list`:
# For "family-secure":
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
# - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
# only accepting requests from clients providing a verified TLS certificate
# emitted by a CA from `ca-cert` and not revoked in `crl`.
# For "family-default":
# - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
# - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
# accepting requests from any client.
#
# For both families, X-Forwarded-For header will be stripped unless
# client presents a certificate that can be verified with `ca-cert` and `crl`.
#
# From zope-virtualhost-monster-backend-dict`:
# - [::1]:8004 with some path based rewrite-rules redirecting to:
# * http://10.0.0.10/8001 when path matches /zope-1(.*)
# * http://10.0.0.10/8002 when path matches /zope-2(.*)
# with some VirtualHostMonster rewrite rules so zope writes URLs with
# [::1]:8004 as server name.
# For more details, refer to
# https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
-#}
{% set server_check_path = parameter_dict['server-check-path'] -%}
global
maxconn 4096
stats socket {{ parameter_dict['socket-path'] }} level admin
master-worker
pidfile {{ parameter_dict['pidfile'] }}
# SSL configuration was generated with mozilla SSL Configuration Generator
# generated 2020-10-28, Mozilla Guideline v5.6, HAProxy 2.1, OpenSSL 1.1.1g, modern configuration
# https://ssl-config.mozilla.org/#server=haproxy&version=2.1&config=modern&openssl=1.1.1g&guideline=5.6
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
stats socket {{ parameter_dict['stats-socket'] }} level admin
defaults
mode http
retries 1
option redispatch
maxconn 2000
cookie SERVERID rewrite
balance roundrobin
stats uri /haproxy
stats realm Global\ statistics
# it is useless to have timeout much bigger than the one of apache.
# By default apache use 300s, so we set slightly more in order to
# make sure that apache will first stop the connection.
timeout server 305s
# Stop waiting in queue for a zope to become available.
# If no zope can be reached after one minute, consider the request will
# never succeed.
timeout connect 10s
timeout queue 60s
# The connection should be immediate on LAN,
# so we should not set more than 5 seconds, and it could be already too much
timeout connect 5s
# As requested in haproxy doc, make this "at least equal to timeout server".
timeout server 305s
timeout client 305s
# Use "option httpclose" to not preserve client & server persistent connections
# while handling every incoming request individually, dispatching them one after
# another to servers, in HTTP close mode. This is really needed when haproxy
# is configured with maxconn to 1, without this option browsers are unable
# to render a page
option httpclose
{% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
listen {{ name }}
bind {{ parameter_dict['ip'] }}:{{ port }}
option http-server-close
# compress some content types
compression algo gzip
compression type application/font-woff application/font-woff2 application/hal+json application/javascript application/json application/rss+xml application/wasm application/x-font-opentype application/x-font-ttf application/x-javascript application/xml image/svg+xml text/cache-manifest text/css text/html text/javascript text/plain text/xml
log {{ parameter_dict['log-socket'] }} local0 info
{% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~ ' alpn h2,http/1.1' %}
{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
listen family_{{ name }}
{%- if parameter_dict.get('ca-cert') -%}
{%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
{%- else %}
{%- set ssl_auth = '' %}
{%- endif %}
bind {{ parameter_dict['ipv4'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
bind {{ parameter_dict['ipv6'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }}
cookie SERVERID rewrite
http-request set-header X-Balancer-Current-Cookie SERVERID
# remove X-Forwarded-For unless client presented a verified certificate
acl client_cert_verified ssl_c_used ssl_c_verify 0
http-request del-header X-Forwarded-For unless client_cert_verified
# set Remote-User if client presented a verified certificate
http-request del-header Remote-User
http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if client_cert_verified
# logs
capture request header Referer len 512
capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
{% set has_webdav = [] -%}
{% for address, connection_count, webdav in backend_list -%}
{% if webdav %}{% do has_webdav.append(None) %}{% endif -%}
{% set server_name = name ~ '-' ~ loop.index0 -%}
{% set server_name = name ~ '-' ~ loop.index0 %}
server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}
{% endfor -%}
{%- endfor -%}
{%- if not has_webdav and server_check_path %}
option httpchk GET {{ server_check_path }}
{% endif -%}
{%- endif %}
{% endfor %}
{% for (ip, port), (_, backend_dict) in sorted(parameter_dict['zope-virtualhost-monster-backend-dict'].iteritems()) -%}
{% set group_name = 'testrunner_' ~ loop.index0 %}
frontend frontend_{{ group_name }}
bind {{ ip }}:{{ port }} {{ bind_ssl_crt }}
timeout client 8h
# logs
capture request header Referer len 512
capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
{% for name in sorted(backend_dict.keys()) %}
use_backend backend_{{ group_name }}_{{ name }} if { path -m beg /{{ name }} }
{%- endfor %}
{% for name, url in sorted(backend_dict.items()) %}
backend backend_{{ group_name }}_{{ name }}
http-request replace-path ^/{{ name }}(.*) /VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ name }}\1
timeout server 8h
server {{ name }} {{ urlparse.urlparse(url).netloc }}
{%- endfor %}
{% endfor %}
This diff is collapsed.
......@@ -56,13 +56,17 @@ openssl-location = {{ openssl_location }}
[dynamic-template-balancer-parameters]
<= default-dynamic-template-parameters
apache = {{ apache_location }}
openssl = {{ openssl_location }}
haproxy = {{ haproxy_location }}
rsyslogd = {{ rsyslogd_location }}
socat = {{ socat_location }}
apachedex-location = {{ bin_directory }}/apachedex
run-apachedex-location = {{ bin_directory }}/runApacheDex
promise-check-apachedex-result = {{ bin_directory }}/check-apachedex-result
template-haproxy-cfg = {{ template_haproxy_cfg }}
template-rsyslogd-cfg = {{ template_rsyslogd_cfg }}
# XXX: only used in software/slapos-master:
apache = {{ apache_location }}
template-apache-conf = {{ template_apache_conf }}
[dynamic-template-balancer]
......
module(
load="imuxsock"
SysSock.Name="{{ parameter_dict['log-socket'] }}")
# Just simply output the raw line without any additional information, as
# haproxy emits enough information by itself
# Also cut out first empty space in msg, which is related to rsyslogd
# internal and end up cutting on 8k, as it's default of $MaxMessageSize
template(name="rawoutput" type="string" string="%msg:2:8192%\n")
$ActionFileDefaultTemplate rawoutput
$FileCreateMode 0600
$DirCreateMode 0700
$Umask 0022
$WorkDirectory {{ parameter_dict['spool-directory'] }}
local0.=info {{ parameter_dict['access-log-file'] }}
local0.warning {{ parameter_dict['error-log-file'] }}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment