Commit bddfdad2 authored by Jérome Perrin's avatar Jérome Perrin

erp5: make testrunner access zserver through apache

Default behavior of runUnitTest was to listen on `127.0.0.1`/`localhost`, which is not
allowed for SlapOS instance and caused issues when the used DNS server does not resolve localhost.

Instead, we decided to run the zserver on the partition IP. Because this partition's IP is not seen as a secure origin (as defined in https://goo.gl/Y0ZkNV ) from firefox when running zelenium tests, we also add some entries in apache from balancer partition to act as an https proxy on this zserver and change the testrunners to access the https URL.

See also erp5!729 for the erp5 part.

/reviewed-on !374
parents 2c3e7dac 2a457867
...@@ -46,17 +46,42 @@ ...@@ -46,17 +46,42 @@
# #
# # The list of backends which apache should redirect to. # # The list of backends which apache should redirect to.
# "backend-list": [ # "backend-list": [
# # (port, unused, internal_scheme) # # (port, unused, internal_scheme, enable_authentication)
# (8000, _, "http://10.0.0.10:8001"), # (8000, _, "http://10.0.0.10:8001", True),
# (8002, _, "http://10.0.0.10:8003"), # (8002, _, "http://10.0.0.10:8003", False),
# ], # ],
#
# # The mapping of zope paths this apache should redirect to.
# # This is a Zope specific feature.
# # `enable_authentication` has same meaning as for `backend-list`.
# "zope-virtualhost-monster-backend-dict": {
# # {(ip, port): ( enable_authentication, {frontend_path: ( internal_scheme ) }, ) }
# ('[::1]', 8004): (
# True, {
# 'zope-1': 'http://10.0.0.10:8001',
# 'zope-2': 'http://10.0.0.10:8002',
# },
# ),
# },
# } # }
# #
# This sample of `parameter_dict` will make apache listening to : # This sample of `parameter_dict` will make apache listening to :
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 # From to `backend-list`:
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
# - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
# only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`.
# - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
# - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
# accepting requests from any client.
#
# From zope-virtualhost-monster-backend-dict`:
# - [::1]:8004 with some path based rewrite-rules redirecting to:
# * http://10.0.0.10/8001 when path matches /zope-1(.*)
# * http://10.0.0.10/8002 when path matches /zope-2(.*)
# with some VirtualHostMonster rewrite rules so zope writes URLs with
# [::1]:8004 as server name.
# For more details, refer to
# https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
-#} -#}
LoadModule unixd_module modules/mod_unixd.so LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so LoadModule access_compat_module modules/mod_access_compat.so
...@@ -153,3 +178,28 @@ Listen {{ ip }}:{{ port }} ...@@ -153,3 +178,28 @@ Listen {{ ip }}:{{ port }}
RewriteRule ^/(.*) {{ backend }}/$1 [L,P] RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
</VirtualHost> </VirtualHost>
{% endfor -%} {% endfor -%}
{% for (ip, port), (enable_authentication, path_mapping) in parameter_dict.get('zope-virtualhost-monster-backend-dict', {}).items() -%}
Listen {{ ip }}:{{ port }}
<VirtualHost {{ ip }}:{{ port }}>
SSLEngine on
{% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
SSLVerifyClient require
SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }}
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
# We would like to separate the the authentificated logs.
# XXX filename ? is it log-rotated ?
ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-error.log"
CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-access.log" combined
{% endif -%}
{% for path, backend in path_mapping.items() %}
RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path }}$1 [L,P]
{% endfor -%}
</VirtualHost>
{% endfor -%}
\ No newline at end of file
...@@ -16,6 +16,7 @@ extends = ...@@ -16,6 +16,7 @@ extends =
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../sqlite3/buildout.cfg ../sqlite3/buildout.cfg
../zlib/buildout.cfg ../zlib/buildout.cfg
./buildout.hash.cfg
[apr] [apr]
recipe = slapos.recipe.build:download-unpacked recipe = slapos.recipe.build:download-unpacked
...@@ -196,6 +197,5 @@ make-targets = ...@@ -196,6 +197,5 @@ make-targets =
[template-apache-backend-conf] [template-apache-backend-conf]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
shared = false shared = false
url = ${:_profile_base_location_}/apache-backend.conf.in url = ${:_profile_base_location_}/${:filename}
md5sum = 8f1f92ad308ab6bad973c790ee583428
mode = 640 mode = 640
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-apache-backend-conf]
filename = apache-backend.conf.in
md5sum = 3b430ca726a2707e1b6a2ae41a6c8e21
...@@ -3,7 +3,7 @@ Available ``software-type`` values ...@@ -3,7 +3,7 @@ Available ``software-type`` values
- ``default`` - ``default``
Recommended for developemnt and production use. Automatic creation of Recommended for development and production use. Automatic creation of
erp5-site. erp5-site.
Notes Notes
...@@ -100,7 +100,7 @@ Zope partitions should be assigned port ranges starting at 2200, incrementing ...@@ -100,7 +100,7 @@ Zope partitions should be assigned port ranges starting at 2200, incrementing
by some value which depends on how many zope process you want per partition by some value which depends on how many zope process you want per partition
(see the ``port-base`` parameter in ``zope-partition-dict``). (see the ``port-base`` parameter in ``zope-partition-dict``).
Notes to the Software Release developper: These ranges are not strictly Notes to the Software Release developer: These ranges are not strictly
defined. Not each port is actually used so one may reduce alread-assigned defined. Not each port is actually used so one may reduce alread-assigned
ranges if needed (ex: memcached partitions use actually fewer ports). There ranges if needed (ex: memcached partitions use actually fewer ports). There
should be enough room for evolution (as between smtp and mariadb types). It is should be enough room for evolution (as between smtp and mariadb types). It is
......
...@@ -345,5 +345,26 @@ ...@@ -345,5 +345,26 @@
}, },
"type": "object" "type": "object"
} }
},
"test-runner": {
"description": "Test runner parameters.",
"properties": {
"enabled": {
"description": "Generate helper scripts to run test suite.",
"default": true,
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
"extra-database-count": {
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
}
},
"type": "object"
} }
} }
...@@ -48,12 +48,6 @@ ...@@ -48,12 +48,6 @@
}, },
"type": "array" "type": "array"
}, },
"test-database-amount": {
"description": "The number of test databases to create, adding auto-generated entries to database-list",
"default": 1,
"minimum": 0,
"type": "integer"
},
"catalog-backup": { "catalog-backup": {
"description": "Backup control knobs", "description": "Backup control knobs",
"properties": { "properties": {
......
...@@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e ...@@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 83e0f627633ec3e394dc27d06c627b3b md5sum = 64fd49395de02e6c2107afd1dd9cbdc9
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35 ...@@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = e73678921067506e710ae11e41f0a9a8 md5sum = 408c8f358935d3167448ea3732adc994
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = 89872075cd2f31d824369447aaf0532f md5sum = dccdf2a8cbfa57e49fa04be9ba4ff595
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
......
...@@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }} ...@@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }}
{% endif -%} {% endif -%}
{% set haproxy_dict = {} -%} {% set haproxy_dict = {} -%}
{% set apache_dict = {} -%} {% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted( {% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%} slapparameter_dict['zope-family-dict'].iteritems()) -%}
{% set zope_family_address_list = [] -%} {% set zope_family_address_list = [] -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% set has_webdav = [] -%} {% set has_webdav = [] -%}
{% for parameter_id in parameter_id_list -%} {% for parameter_id in parameter_id_list -%}
{% set zope_address_list = slapparameter_dict[parameter_id] -%} {% set zope_address_list = slapparameter_dict[parameter_id] -%}
...@@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} ...@@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set zope_effective_address = zope_address -%} {% set zope_effective_address = zope_address -%}
{% endif -%} {% endif -%}
{% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%} {% do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
{# # Generate entries with rewrite rule for test runnners #}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in
enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
{% endfor -%} {% endfor -%}
{% endfor -%} {% endfor -%}
{# Make rendering fail artificially if any family has no known backend. {# Make rendering fail artificially if any family has no known backend.
# This is useful as haproxy's hot-reconfiguration mechanism is # This is useful as haproxy's hot-reconfiguration mechanism is
# supervisord-incompatible. # supervisord-incompatible.
...@@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }} ...@@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set internal_scheme = 'http' -%} {% set internal_scheme = 'http' -%}
{% set external_scheme = 'https' -%} {% set external_scheme = 'https' -%}
{% endif -%} {% endif -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%} {% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
{% endfor -%} {% endfor -%}
...@@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem ...@@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem
[apache-conf-parameter-dict] [apache-conf-parameter-dict]
backend-list = {{ dumps(apache_dict.values()) }} backend-list = {{ dumps(apache_dict.values()) }}
zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
ip-list = {{ dumps(apache_ip_list) }} ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid pid-file = ${directory:run}/apache.pid
log-dir = ${directory:log} log-dir = ${directory:log}
...@@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised ...@@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %} {{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }} {{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
{% endfor -%} {% endfor -%}
{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
{{ family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
{% endfor -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[apache-ssl] [apache-ssl]
......
...@@ -8,6 +8,16 @@ ...@@ -8,6 +8,16 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%} {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%} {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
{% set test_runner_total_database_count = test_runner_node_count * (1 + test_runner_extra_database_count) -%}
{# Backward compatibility for mariadb.test-database-amount #}
{% set mariadb_test_database_amount = slapparameter_dict.get('mariadb', {}).get('test-database-amount') -%}
{% if mariadb_test_database_amount is not none -%}
{% set test_runner_total_database_count = mariadb_test_database_amount %}
{% set test_runner_enabled = mariadb_test_database_amount > 0 %}
{% endif -%}
{% set monitor_base_url_dict = {} -%} {% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%} {% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %} {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
...@@ -44,7 +54,7 @@ config-name = {{ name }} ...@@ -44,7 +54,7 @@ config-name = {{ name }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%} {% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }} {{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %} {%- else %}
...@@ -93,44 +103,6 @@ connection-http-url = {{ caucase_url }} ...@@ -93,44 +103,6 @@ connection-http-url = {{ caucase_url }}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
{%- endif %}
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %}
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
[gen-deadlock-debugger-password]
<= gen-password
[gen-neo-cluster-base]
<= gen-password
[gen-neo-cluster]
name = neo-${gen-neo-cluster-base:passwd}
[gen-smtpd-sasl-password]
< = gen-password
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_address_list_id_dict = {} -%} {% set zope_address_list_id_dict = {} -%}
...@@ -142,6 +114,9 @@ return = ...@@ -142,6 +114,9 @@ return =
zope-address-list zope-address-list
hosts-dict hosts-dict
monitor-base-url monitor-base-url
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%} {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
...@@ -169,6 +144,8 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }} ...@@ -169,6 +144,8 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }} config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }} config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-zodb-dict = {{ dumps(zodb_dict) }} config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%} {% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%} {% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster} config-neo-cluster = ${publish-early:neo-cluster}
...@@ -183,6 +160,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} ...@@ -183,6 +160,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
software-type = zope software-type = zope
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%} {% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%} {% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%} {% set jupyter_zope_family_default = [] -%}
...@@ -190,6 +168,7 @@ software-type = zope ...@@ -190,6 +168,7 @@ software-type = zope
{% set partition_name = 'zope-' ~ custom_name -%} {% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%} {% set section_name = 'request-' ~ partition_name -%}
{% set zope_family = zope_parameter_dict.get('family', 'default') -%} {% set zope_family = zope_parameter_dict.get('family', 'default') -%}
{% do zope_family_name_list.append(zope_family) %}
{% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %} {% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
{# # default jupyter zope family is first zope family. -#} {# # default jupyter zope family is first zope family. -#}
{# # use list.append() to update it, because in jinja2 set changes only local scope. -#} {# # use list.append() to update it, because in jinja2 set changes only local scope. -#}
...@@ -214,6 +193,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques ...@@ -214,6 +193,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }} config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
{% endfor -%} {% endfor -%}
{# if not explicitly configured, connect jupyter to first zope family, which -#} {# if not explicitly configured, connect jupyter to first zope family, which -#}
...@@ -268,6 +248,9 @@ return = ...@@ -268,6 +248,9 @@ return =
{%- for family in zope_family_dict %} {%- for family in zope_family_dict %}
{{ family }} {{ family }}
{{ family }}-v6 {{ family }}-v6
{% if test_runner_enabled %}
{{ family }}-test-runner-url-list
{% endif %}
{% endfor -%} {% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
...@@ -275,6 +258,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} ...@@ -275,6 +258,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%} {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
{% if test_runner_enabled -%}
config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }}
{% endif -%}
{% endfor -%} {% endfor -%}
# XXX: should those really be same for all families ? # XXX: should those really be same for all families ?
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }} config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
...@@ -335,6 +321,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts- ...@@ -335,6 +321,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{% for name, value in publish_dict.items() -%} {% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }} {{ name }} = {{ value }}
{% endfor -%} {% endfor -%}
{% for zope_family_name in zope_family_name_list -%}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{% endfor -%}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
{%- endif %}
{% for zope_family_name in zope_family_name_list %}
{{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
{% endfor -%}
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %}
[default-balancer-test-runner-url-list]
recipe =
default = not-ready
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
[gen-deadlock-debugger-password]
<= gen-password
[gen-neo-cluster-base]
<= gen-password
[gen-neo-cluster]
name = neo-${gen-neo-cluster-base:passwd}
[gen-smtpd-sasl-password]
< = gen-password
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-port = 8386 monitor-httpd-port = 8386
......
...@@ -8,6 +8,9 @@ ...@@ -8,6 +8,9 @@
{% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%} {% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%}
{% set part_list = [] -%} {% set part_list = [] -%}
{% set publish_list = [] -%} {% set publish_list = [] -%}
{% set test_runner_address_list = [] -%}
{% set test_runner_enabled = slapparameter_dict['test-runner-enabled'] -%}
{% set test_runner_node_count = slapparameter_dict['test-runner-node-count'] -%}
{% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%} {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%} {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
...@@ -68,84 +71,6 @@ ca-certs = ${directory:test-ca-certs} ...@@ -68,84 +71,6 @@ ca-certs = ${directory:test-ca-certs}
ca-newcerts = ${directory:test-ca-newcerts} ca-newcerts = ${directory:test-ca-newcerts}
ca-crl = ${directory:test-ca-crl} ca-crl = ${directory:test-ca-crl}
{% if saucelabs_dict -%}
[test-zelenium-runner-parameter]
configuration = {{ dumps(saucelabs_dict) }}
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
[{{ section('test-zelenium-runner') }}]
<= jinja2-template-base
template = {{ parameter_dict['run-zelenium-template'] }}
rendered = ${directory:bin}/runTestSuite
mode = 755
context =
import json_module json
key configuration test-zelenium-runner-parameter:configuration
key user test-zelenium-runner-parameter:user
key password test-zelenium-runner-parameter:password
key bin_path test-zelenium-runner-parameter:bin-path
{% else -%}
{% if slapparameter_dict['mysql-test-url-list'] -%}
[{{ section('run-unit-test-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runUnitTest:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runUnitTest
[{{ section('run-test-suite-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runTestSuite:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runTestSuite
{% set connection_string_list = [] -%}
{% for url in slapparameter_dict['mysql-test-url-list'] -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% do connection_string_list.append(
'%s@%s:%s %s %s' % (
parsed_url.path.lstrip('/'),
parsed_url.hostname,
parsed_url.port,
parsed_url.username,
parsed_url.password,
),
) -%}
{% endfor -%}
[run-test-common]
< = run-common
environment-extra =
REAL_INSTANCE_HOME=${:instance-home}
OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
TEST_CA_PATH=${test-certificate-authority:ca-dir}
instance-home = ${directory:unit-test-path}
wrapper-path = ${directory:bin}/${:command-name}.real
command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent
--persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
[{{ section('runUnitTest') }}]
< = run-test-common
command-name = runUnitTest
command-line-extra =
--erp5_sql_connection_string '{{ connection_string_list[0] }}'
--extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
[{{ section('runTestSuite') }}]
< = run-test-common
command-name = runTestSuite
command-line-extra =
--db_list '{{ ','.join(connection_string_list) }}'
{%- endif %}
{%- endif %}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
...@@ -472,6 +397,111 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_} ...@@ -472,6 +397,111 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
[{{ section("wait_activities") }}] [{{ section("wait_activities") }}]
<= watch_activities <= watch_activities
{% if saucelabs_dict -%}
[test-zelenium-runner-parameter]
configuration = {{ dumps(saucelabs_dict) }}
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
[{{ section('test-zelenium-runner') }}]
<= jinja2-template-base
template = {{ parameter_dict['run-zelenium-template'] }}
rendered = ${directory:bin}/runTestSuite
mode = 755
context =
import json_module json
key configuration test-zelenium-runner-parameter:configuration
key user test-zelenium-runner-parameter:user
key password test-zelenium-runner-parameter:password
key bin_path test-zelenium-runner-parameter:bin-path
{% else -%}
{% if test_runner_enabled and test_runner_node_count -%}
{% for _ in range(test_runner_node_count) %}
{% do test_runner_address_list.append((ipv4, next_port())) %}
{% endfor %}
[{{ section('run-unit-test-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runUnitTest:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runUnitTest
[{{ section('run-test-suite-userhosts-wrapper') }}]
<= userhosts-wrapper-base
wrapped-command-line = ${runTestSuite:wrapper-path}
wrapper-path = ${buildout:bin-directory}/runTestSuite
{% set connection_string_list = [] -%}
{% for url in slapparameter_dict['mysql-test-url-list'] -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% do connection_string_list.append(
'%s@%s:%s %s %s' % (
parsed_url.path.lstrip('/'),
parsed_url.hostname,
parsed_url.port,
parsed_url.username,
parsed_url.password,
),
) -%}
{% endfor -%}
[run-test-common]
< = run-common
environment-extra =
REAL_INSTANCE_HOME=${:instance-home}
OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
TEST_CA_PATH=${test-certificate-authority:ca-dir}
instance-home = ${directory:unit-test-path}
wrapper-path = ${directory:bin}/${:command-name}.real
command-line =
'{{ parameter_dict['bin-directory'] }}/${:command-name}'
${:command-line-extra}
--conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
--conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
{#- BBB: We still have test suites that only accept the following 2 options. #}
--conversion_server_hostname=erp5-cloudooo
--conversion_server_port={{ port_dict['erp5-cloudooo'] }}
--volatile_memcached_server_hostname=erp5-memcached-volatile
--volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
--persistent_memcached_server_hostname=erp5-memcached-persistent
--persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
[{{ section('runUnitTest') }}]
< = run-test-common
command-name = runUnitTest
command-line-extra =
--erp5_sql_connection_string '{{ connection_string_list[0] }}'
--extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
--zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }}
--zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }}
[{{ section('runTestSuite') }}]
< = run-test-common
command-name = runTestSuite
command-line-extra =
--db_list '{{ ','.join(connection_string_list) }}'
environment-extra +=
{#- turn a list of (ip, port) in a list of 'ip:port' #}
{% set zserver_address_list = [] -%}
{% for ip, port in test_runner_address_list %}
{% do zserver_address_list.append(ip ~ ':' ~ port) %}
{% endfor -%}
zserver_address_list={{ ','.join(zserver_address_list) }}
zserver_frontend_url_list={{ ','.join(slapparameter_dict['test-runner-apache-url-list']) }}
[{{ section("promise-test-runner-apache-url") }}]
# promise to wait for apache partition to have returned the parameter
recipe = slapos.cookbook:check_parameter
value = {{ slapparameter_dict['test-runner-apache-url-list'] }}
expected-not-value = not-ready
path = ${directory:promises}/${:_buildout_section_name_}
expected-value =
{%- endif %}
{%- endif %}
[publish] [publish]
recipe = slapos.cookbook:publish.serialised recipe = slapos.cookbook:publish.serialised
zope-address-list = {{ dumps(publish_list) }} zope-address-list = {{ dumps(publish_list) }}
...@@ -484,6 +514,7 @@ hard to guess. ...@@ -484,6 +514,7 @@ hard to guess.
-#} -#}
hosts-dict = {{ dumps(hosts_dict) }} hosts-dict = {{ dumps(hosts_dict) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
test-runner-address-list = {{ dumps(test_runner_address_list) }}
[monitor-instance-parameter] [monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment