Commit 2fc522bf authored by Jérome Perrin's avatar Jérome Perrin

stack/erp5: use slapos.recipe.build to manage haproxy parameters

and save the already allocated ports in a state file, so that requesting
new families does not change already allocated ports.
parent d49914a6
...@@ -305,6 +305,13 @@ class TestBalancerPorts(ERP5InstanceTestCase): ...@@ -305,6 +305,13 @@ class TestBalancerPorts(ERP5InstanceTestCase):
param_dict[f'family-{family_name}']) param_dict[f'family-{family_name}'])
self.checkValidHTTPSURL( self.checkValidHTTPSURL(
param_dict[f'family-{family_name}-v6']) param_dict[f'family-{family_name}-v6'])
# ports are allocated in alphabetical order and are "stable", ie. is not supposed
# to change after updating software release, because there is typically a rapid-cdn
# frontend pointing to this port.
self.assertEqual(urllib.parse.urlparse(param_dict['family-family1']).port, 2152)
self.assertEqual(urllib.parse.urlparse(param_dict['family-family1-v6']).port, 2152)
self.assertEqual(urllib.parse.urlparse(param_dict['family-family2']).port, 2154)
self.assertEqual(urllib.parse.urlparse(param_dict['family-family2-v6']).port, 2154)
def test_published_test_runner_url(self): def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per family # each family's also a list of test test runner URLs, by default 3 per family
...@@ -313,6 +320,7 @@ class TestBalancerPorts(ERP5InstanceTestCase): ...@@ -313,6 +320,7 @@ class TestBalancerPorts(ERP5InstanceTestCase):
family_test_runner_url_list = param_dict[ family_test_runner_url_list = param_dict[
f'{family_name}-test-runner-url-list'] f'{family_name}-test-runner-url-list']
self.assertEqual(3, len(family_test_runner_url_list)) self.assertEqual(3, len(family_test_runner_url_list))
self.assertEqual(3, len(set(family_test_runner_url_list)))
for url in family_test_runner_url_list: for url in family_test_runner_url_list:
self.checkValidHTTPSURL(url) self.checkValidHTTPSURL(url)
...@@ -353,6 +361,61 @@ class TestBalancerPorts(ERP5InstanceTestCase): ...@@ -353,6 +361,61 @@ class TestBalancerPorts(ERP5InstanceTestCase):
]) ])
class TestBalancerPortsStable(ERP5InstanceTestCase):
"""Instantiate with two one families and a frontend, then
re-request with one more family and one more frontend, the ports
should not change
"""
__partition_reference__ = 'ap'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps(
{
"frontend": {
"zzz": {
"zope-family": "zzz"
}
},
"zope-partition-dict": {
"zzz": {
"instance-count": 1,
"family": "zzz"
},
},
})
}
def test_same_balancer_ports_when_adding_zopes_or_frontends(self):
param_dict_before = self.getRootPartitionConnectionParameterDict()
# re-request with one more frontend and one more backend, that are before
# the existing ones when sorting alphabetically
instance_parameter_dict = json.loads(self.getInstanceParameterDict()['_'])
instance_parameter_dict['frontend']['aaa'] = {"zope-family": "aaa"}
instance_parameter_dict['zope-partition-dict']['aaa'] = {
"instance-count": 2,
"family": "aaa"
}
def rerequest():
return self.slap.request(
software_release=self.getSoftwareURL(),
software_type=self.getInstanceSoftwareType(),
partition_reference=self.default_partition_reference,
partition_parameter_kw={'_': json.dumps(instance_parameter_dict)},
state='started')
rerequest()
self.slap.waitForInstance(max_retry=10)
param_dict_after = json.loads(rerequest().getConnectionParameterDict()['_'])
self.assertEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-zzz-v6'])
self.assertEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-zzz'])
self.assertNotEqual(param_dict_before['family-zzz-v6'], param_dict_after['family-aaa-v6'])
self.assertNotEqual(param_dict_before['url-frontend-zzz'], param_dict_after['url-frontend-aaa'])
class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin): class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with selenium server for test runner. """Test ERP5 can be instantiated with selenium server for test runner.
""" """
......
...@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04 ...@@ -70,7 +70,7 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 5e0e9565227fe190c420a7bbcd0f7b93 md5sum = 55463b0abdbe0118ef1c27e6b71c3324
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
...@@ -90,11 +90,11 @@ md5sum = 6178ba7b42848f9e2412ab898a7b026c ...@@ -90,11 +90,11 @@ md5sum = 6178ba7b42848f9e2412ab898a7b026c
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = dbd17fbde7a1a3b1a12e3ea1db25baa1 md5sum = 42cb68905f92e7df38cc5c64b94be3de
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
md5sum = 85a8c0dadf7b648ef9748b6199dcfeb6 md5sum = 9988a14c4108e3bce3f871e34673cdd5
[template-rsyslogd-cfg] [template-rsyslogd-cfg]
filename = rsyslogd.cfg.in filename = rsyslogd.cfg.in
......
{# This file configures haproxy to redirect requests from ports to specific urls. {# This file configures haproxy to redirect requests from ports to specific urls.
# It provides TLS support for server and optionnaly for client. # It provides TLS support for server and optionnaly for client.
# #
...@@ -46,7 +45,6 @@ ...@@ -46,7 +45,6 @@
# "backend-dict": { # "backend-dict": {
# "family-secure": { # "family-secure": {
# ( 8000, # port int # ( 8000, # port int
# 'https', # proto str
# True, # ssl_required bool # True, # ssl_required bool
# None, # timeout (in seconds) int | None # None, # timeout (in seconds) int | None
# [ # backends # [ # backends
...@@ -58,7 +56,6 @@ ...@@ -58,7 +56,6 @@
# }, # },
# "family-default": { # "family-default": {
# ( 8002, # port int # ( 8002, # port int
# 'https', # proto str
# False, # ssl_required bool # False, # ssl_required bool
# None, # timeout (in seconds) int | None # None, # timeout (in seconds) int | None
# [ # backends # [ # backends
...@@ -151,7 +148,7 @@ defaults ...@@ -151,7 +148,7 @@ defaults
{% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %} {% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
{% set path_routing_list = parameter_dict['path-routing-list'] %} {% set path_routing_list = parameter_dict['path-routing-list'] %}
{% for name, (port, _, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%} {% for name, (port, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
listen family_{{ name }} listen family_{{ name }}
{%- if parameter_dict.get('ca-cert') -%} {%- if parameter_dict.get('ca-cert') -%}
{%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional crt-ignore-err all' ) ~ ' crl-file ' ~ parameter_dict['crl'] %} {%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional crt-ignore-err all' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
......
...@@ -208,70 +208,8 @@ command = ${caucase-updater-housekeeper:output} ...@@ -208,70 +208,8 @@ command = ${caucase-updater-housekeeper:output}
update-command = ${:command} update-command = ${:command}
{% endif -%} {% endif -%}
{% set haproxy_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port'])) -%}
{% for family_name, parameter_id_list in sorted(
six.iteritems(slapparameter_dict['zope-family-dict'])) -%}
{% set zope_family_address_list = [] -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name] -%}
{% set has_webdav = [] -%}
{% for parameter_id in parameter_id_list -%}
{% set zope_address_list = slapparameter_dict[parameter_id] -%}
{% for zope_address, maxconn, webdav in zope_address_list -%}
{% if webdav -%}
{% do has_webdav.append(None) %}
{% endif -%}
{% set zope_effective_address = zope_address -%}
{% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
{% endfor -%}
{# # Generate entries with rewrite rule for test runnners #}
{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
{% if test_runner_address_list -%}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_balancer_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_balancer_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
{% endif -%}
{% endfor -%}
{# Make rendering fail artificially if any family has no known backend.
# This is useful as haproxy's hot-reconfiguration mechanism is
# supervisord-incompatible.
# As jinja2 postpones KeyError until place-holder value is actually used,
# do a no-op getitem.
-#}
{% do zope_family_address_list[0][0] -%}
{#
# We use to have haproxy then apache, now haproxy is playing apache's role
# To keep port stable, we consume one port so that haproxy use the same port
# that apache was using before.
-#}
{% set _ = next_port() -%}
{% set haproxy_port = next_port() -%}
{% if has_webdav -%}
{% set external_scheme = 'webdavs' -%}
{% else %}
{% set external_scheme = 'https' -%}
{% endif -%}
{% do haproxy_dict.__setitem__(family_name, (haproxy_port, external_scheme, slapparameter_dict['ssl-authentication-dict'][family_name], slapparameter_dict['timeout-dict'][family_name], zope_family_address_list)) -%}
{% endfor -%}
[haproxy-cfg-parameter-dict] [haproxy-cfg-parameter-dict]
recipe = slapos.recipe.build
ipv4 = {{ ipv4 }} ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }} ipv6 = {{ ipv6 }}
cert = ${haproxy-conf-ssl:certificate} cert = ${haproxy-conf-ssl:certificate}
...@@ -285,9 +223,90 @@ family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict ...@@ -285,9 +223,90 @@ family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict
pidfile = ${directory:run}/haproxy.pid pidfile = ${directory:run}/haproxy.pid
log-socket = ${rsyslogd-cfg-parameter-dict:log-socket} log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }} server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
backend-dict = {{ dumps(haproxy_dict) }} slapparameter-dict = {{ dumps(slapparameter_dict) }}
zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }} ports-state-file = ${buildout:directory}/.${:_buildout_section_name_}-ports.json
init =
import functools
import itertools
import json
import os
import shutil
import six
from zc.buildout import UserError
slapparameter_dict = options['slapparameter-dict']
ipv4 = options['ipv4']
ipv6 = options['ipv6']
# read port state file
port_dict = {}
previous_port_dict = None
if os.path.exists(options['ports-state-file']):
with open(options['ports-state-file']) as f:
port_dict = json.load(f)
previous_port_dict = dict(port_dict)
_next_port = functools.partial(next, itertools.count(slapparameter_dict['tcpv4-port']))
def get_port(name):
if name in port_dict:
return port_dict[name]
port = _next_port()
while port in port_dict.values():
port = _next_port()
port_dict[name] = port
return port
haproxy_dict = {}
zope_virtualhost_monster_backend_dict = {}
for family_name, parameter_id_list in sorted(
six.iteritems(slapparameter_dict['zope-family-dict'])):
zope_family_address_list = []
ssl_authentication = slapparameter_dict['ssl-authentication-dict'][family_name]
for parameter_id in parameter_id_list:
zope_family_address_list.extend(slapparameter_dict[parameter_id])
# Generate entries with rewrite rule for test runnners
test_runner_address_list = slapparameter_dict.get(parameter_id + '-test-runner-address-list', [])
if test_runner_address_list:
test_runner_backend_mapping = {}
test_runner_balancer_url_list = []
for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list):
test_runner_backend_mapping['unit_test_%s' % i] = \
'http://%s:%s' % (test_runner_internal_ip, test_runner_internal_port)
test_runner_balancer_url_list.append(
'https://%s:%s/unit_test_%s/' % (ipv4, get_port('test-runner-' + family_name), i))
zope_virtualhost_monster_backend_dict[(ipv4, get_port('test-runner-' + family_name))] =\
( ssl_authentication, test_runner_backend_mapping )
self.buildout['publish'][family_name + '-test-runner-url-list'] = test_runner_balancer_url_list
if not zope_family_address_list:
raise UserError('No zope defined for family %s (maybe not ready)' % family_name)
# consume a port for compatibility when were using apache + haproxy
get_port('apache-compatibility-' + family_name)
legacy_port = get_port('legacy-' + family_name)
# a port for monitoring promise (which port is not important, the promise checks
# that haproxy is healthy enough to listen on a port)
options['haproxy-promise-port'] = legacy_port
haproxy_dict[family_name] = (
legacy_port,
ssl_authentication,
slapparameter_dict['timeout-dict'][family_name],
zope_family_address_list,
)
external_scheme = 'webdavs' if any(a[2] for a in zope_family_address_list) else 'https'
self.buildout['publish'][family_name] = "{external_scheme}://{ipv4}:{legacy_port}".format(**locals())
self.buildout['publish'][family_name + "-v6"] = "{external_scheme}://[{ipv6}]:{legacy_port}".format(**locals())
options['backend-dict'] = haproxy_dict
options['zope-virtualhost-monster-backend-dict'] = zope_virtualhost_monster_backend_dict
if port_dict != previous_port_dict:
with open(options['ports-state-file'] + '.tmp', 'w') as f:
json.dump(port_dict, f, indent=True)
shutil.move(options['ports-state-file'] + '.tmp', options['ports-state-file'])
[haproxy-cfg] [haproxy-cfg]
< = jinja2-template-base < = jinja2-template-base
...@@ -371,17 +390,11 @@ certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output} ...@@ -371,17 +390,11 @@ certificate = ${haproxy-conf-ssl-certificate-and-key-from-parameters:output}
promise = check_socket_listening promise = check_socket_listening
name = haproxy.py name = haproxy.py
config-host = {{ ipv4 }} config-host = {{ ipv4 }}
config-port = {{ next(six.itervalues(haproxy_dict))[0] }} config-port = ${haproxy-cfg-parameter-dict:haproxy-promise-port}
[{{ section('publish') }}] [{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
{% for family_name, (port, scheme, _, _, _) in haproxy_dict.items() -%} # note: some values are pushed by haproxy-cfg-parameter-dict
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ port }}
{% endfor -%}
{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
{{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
{% endfor -%}
caucase-http-url = {{ caucase_url }} caucase-http-url = {{ caucase_url }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
...@@ -466,7 +479,7 @@ config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${mo ...@@ -466,7 +479,7 @@ config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${mo
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }} monitor-httpd-port = 2197
monitor-title = {{ slapparameter_dict['name'] }} monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }} password = {{ slapparameter_dict['monitor-passwd'] }}
......
...@@ -91,8 +91,9 @@ url = {{ template_balancer }} ...@@ -91,8 +91,9 @@ url = {{ template_balancer }}
filename = instance-balancer.cfg filename = instance-balancer.cfg
extra-context = extra-context =
section parameter_dict dynamic-template-balancer-parameters section parameter_dict dynamic-template-balancer-parameters
import itertools itertools
import hashlib hashlib import hashlib hashlib
# XXX: only used in software/slapos-master:
import itertools itertools
import functools functools import functools functools
import-list = import-list =
file caucase context:caucase-jinja2-library file caucase context:caucase-jinja2-library
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment