erp5: make testrunner access zserver through apache

Default behavior of runUnitTest was to listen on `127.0.0.1`/`localhost`, which is not
allowed for SlapOS instance and caused issues when the used DNS server does not resolve localhost.

Instead, we decided to run the zserver on the partition IP. Because this partition's IP is not seen as a secure origin (as defined in https://goo.gl/Y0ZkNV ) from firefox when running zelenium tests, we also add some entries in apache from balancer partition to act as an https proxy on this zserver and change the testrunners to access the https URL.

See also erp5!729 for the erp5 part.

/reviewed-on !374

component/apache/apache-backend.conf.in

@@ -46,17 +46,42 @@
  #
  #       #  The list of backends which apache should redirect to.
  #       "backend-list": [
- #         # (port, unused, internal_scheme)
- #         (8000, _, "http://10.0.0.10:8001"),
- #         (8002, _, "http://10.0.0.10:8003"),
+ #         # (port, unused, internal_scheme, enable_authentication)
+ #         (8000, _, "http://10.0.0.10:8001", True),
+ #         (8002, _, "http://10.0.0.10:8003", False),
  #       ],
+ #
+ #       # The mapping of zope paths this apache should redirect to.
+ #       # This is a Zope specific feature.
+ #       # `enable_authentication` has same meaning as for `backend-list`.
+ #       "zope-virtualhost-monster-backend-dict": {
+ #          # {(ip, port): ( enable_authentication, {frontend_path: ( internal_scheme ) }, ) }
+ #          ('[::1]', 8004): (
+ #            True, {
+ #              'zope-1': 'http://10.0.0.10:8001',
+ #              'zope-2': 'http://10.0.0.10:8002',
+ #            },
+ #          ),
+ #        },
  #     }
  #
  #  This sample of `parameter_dict` will make apache listening to :
- # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001
+ #  From to `backend-list`:
+ #   - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
  #   - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
+ #  only accepting requests from clients who provide a valid SSL certificate trusted in `ca-cert`.
  #   - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003
  #   - [::1]:8002 redirecting internaly to http://10.0.0.10:8003
+ #  accepting requests from any client.
+ #
+ # From zope-virtualhost-monster-backend-dict`:
+ #   - [::1]:8004 with some path based rewrite-rules redirecting to:
+ #     * http://10.0.0.10/8001 when path matches /zope-1(.*)
+ #     * http://10.0.0.10/8002 when path matches /zope-2(.*)
+ #   with some VirtualHostMonster rewrite rules so zope writes URLs with
+ #  [::1]:8004 as server name.
+ #  For more details, refer to
+ #  https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
 -#}
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so
@@ -153,3 +178,28 @@ Listen {{ ip }}:{{ port }}
   RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
 </VirtualHost>
 {% endfor -%}
+
+
+{% for (ip, port), (enable_authentication, path_mapping) in parameter_dict.get('zope-virtualhost-monster-backend-dict', {}).items() -%}
+Listen {{ ip }}:{{ port }}
+<VirtualHost {{ ip }}:{{ port }}>
+  SSLEngine on
+{%   if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
+  SSLVerifyClient require
+  SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+  SSLCARevocationCheck chain
+  SSLCARevocationFile {{ parameter_dict['crl'] }}
+
+  LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+
+  # We would like to separate the the authentificated logs.
+  # XXX filename ? is it log-rotated ?
+  ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-error.log"
+  CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-virtual-host-access.log" combined
+{%   endif -%}
+
+{%   for path, backend in path_mapping.items() %}
+  RewriteRule ^/{{path}}(.*) {{ backend }}/VirtualHostBase/https/{{ ip }}:{{ port }}/VirtualHostRoot/_vh_{{ path }}$1 [L,P]
+{%   endfor -%}
+</VirtualHost>
+{% endfor -%}
\ No newline at end of file

component/apache/buildout.cfg

@@ -16,6 +16,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../sqlite3/buildout.cfg
   ../zlib/buildout.cfg
+  ./buildout.hash.cfg
 
 [apr]
 recipe = slapos.recipe.build:download-unpacked
@@ -196,6 +197,5 @@ make-targets =
 [template-apache-backend-conf]
 recipe = slapos.recipe.build:download
 shared = false
-url = ${:_profile_base_location_}/apache-backend.conf.in
-md5sum = 8f1f92ad308ab6bad973c790ee583428
+url = ${:_profile_base_location_}/${:filename}
 mode = 640

component/apache/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[template-apache-backend-conf]
+filename = apache-backend.conf.in
+md5sum = 3b430ca726a2707e1b6a2ae41a6c8e21
+

software/erp5/README.rst

@@ -3,7 +3,7 @@ Available ``software-type`` values
 
 - ``default``
 
-  Recommended for developemnt and production use. Automatic creation of
+  Recommended for development and production use. Automatic creation of
   erp5-site.
 
 Notes
@@ -100,7 +100,7 @@ Zope partitions should be assigned port ranges starting at 2200, incrementing
 by some value which depends on how many zope process you want per partition
 (see the ``port-base`` parameter in ``zope-partition-dict``).
 
-Notes to the Software Release developper: These ranges are not strictly
+Notes to the Software Release developer: These ranges are not strictly
 defined. Not each port is actually used so one may reduce alread-assigned
 ranges if needed (ex: memcached partitions use actually fewer ports). There
 should be enough room for evolution (as between smtp and mariadb types). It is

software/erp5/instance-erp5-input-schema.json

@@ -345,5 +345,26 @@
       },
       "type": "object"
     }
+  },
+  "test-runner": {
+    "description": "Test runner parameters.",
+    "properties": {
+      "enabled": {
+        "description": "Generate helper scripts to run test suite.",
+        "default": true,
+        "type": "boolean"
+      },
+      "node-count": {
+        "description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
+        "default": 3,
+        "type": "integer"
+      },
+      "extra-database-count": {
+        "description": "Number of extra databases this instance tests will need.",
+        "default": 3,
+        "type": "integer"
+      }
+    },
+    "type": "object"
   }
 }

software/erp5/instance-mariadb-schema.json

@@ -48,12 +48,6 @@
       },
       "type": "array"
     },
-    "test-database-amount": {
-      "description": "The number of test databases to create, adding auto-generated entries to database-list",
-      "default": 1,
-      "minimum": 0,
-      "type": "integer"
-    },
     "catalog-backup": {
       "description": "Backup control knobs",
       "properties": {

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 83e0f627633ec3e394dc27d06c627b3b
+md5sum = 64fd49395de02e6c2107afd1dd9cbdc9
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -82,11 +82,11 @@ md5sum = 3e650915959ff31c9c13c84069bbcd35
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = e73678921067506e710ae11e41f0a9a8
+md5sum = 408c8f358935d3167448ea3732adc994
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 89872075cd2f31d824369447aaf0532f
+md5sum = dccdf2a8cbfa57e49fa04be9ba4ff595
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -97,10 +97,13 @@ ipv4 = {{ ipv4 }}
 {% endif -%}
 {% set haproxy_dict = {} -%}
 {% set apache_dict = {} -%}
+{% set zope_virtualhost_monster_backend_dict = {} %}
+{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
 {% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
 {% for family_name, parameter_id_list in sorted(
   slapparameter_dict['zope-family-dict'].iteritems()) -%}
 {%   set zope_family_address_list = [] -%}
+{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   set has_webdav = [] -%}
 {%   for parameter_id in parameter_id_list -%}
 {%     set zope_address_list = slapparameter_dict[parameter_id] -%}
@@ -121,8 +124,27 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%         set zope_effective_address = zope_address -%}
 {%       endif -%}
 {%       do zope_family_address_list.append((zope_effective_address, maxconn, webdav)) -%}
+
+{#       # Generate entries with rewrite rule for test runnners #}
+{%       set test_runner_backend_mapping = {} %}
+{%       set test_runner_apache_url_list = [] %}
+{%       set test_runner_external_port = next_port() %}
+{%       for i, (test_runner_internal_ip, test_runner_internal_port) in
+             enumerate(slapparameter_dict[parameter_id ~ '-test-runner-address-list']) %}
+{%         do test_runner_backend_mapping.__setitem__(
+                'unit_test_' ~ i,
+                'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
+{%         do test_runner_apache_url_list.append(
+                'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
+{%       endfor %}
+{%       do zope_virtualhost_monster_backend_dict.__setitem__(
+              (ipv4, test_runner_external_port),
+              ( ssl_authentication, test_runner_backend_mapping ) ) -%}
+{%       do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
+
 {%     endfor -%}
 {%   endfor -%}
+
 {# Make rendering fail artificially if any family has no known backend.
  # This is useful as haproxy's hot-reconfiguration mechanism is
  # supervisord-incompatible.
@@ -140,7 +162,6 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
 {%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {% endfor -%}
 
@@ -173,6 +194,7 @@ crl = ${directory:apache-conf}/crl.pem
 
 [apache-conf-parameter-dict]
 backend-list = {{ dumps(apache_dict.values()) }}
+zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
 log-dir = ${directory:log}
@@ -225,6 +247,10 @@ recipe = slapos.cookbook:publish.serialised
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
 {{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
 {% endfor -%}
+{% for family_name, test_runner_url_list in test_runner_url_dict.items() -%}
+{{    family_name ~ '-test-runner-url-list' }} = {{ dumps(test_runner_url_list) }}
+{% endfor -%}
+
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [apache-ssl]

stack/erp5/instance-erp5.cfg.in

@@ -8,6 +8,16 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
+{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
+{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
+{% set test_runner_total_database_count = test_runner_node_count * (1 + test_runner_extra_database_count) -%}
+{# Backward compatibility for mariadb.test-database-amount #}
+{% set mariadb_test_database_amount = slapparameter_dict.get('mariadb', {}).get('test-database-amount') -%}
+{% if mariadb_test_database_amount is not none -%}
+{%   set test_runner_total_database_count = mariadb_test_database_amount %}
+{%   set test_runner_enabled = mariadb_test_database_amount > 0 %}
+{% endif -%}
 {% set monitor_base_url_dict = {} -%}
 {% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
 {% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
@@ -44,7 +54,7 @@ config-name = {{ name }}
 
 {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
-{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', ''), 'test-database-amount': test_runner_total_database_count}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
 {%- else %}
@@ -93,44 +103,6 @@ connection-http-url = {{ caucase_url }}
 {%   endif -%}
 {% endfor -%}
 
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-  inituser-password gen-password:passwd
-  deadlock-debugger-password gen-deadlock-debugger-password:passwd
-{%- if has_posftix %}
-  smtpd-sasl-password gen-smtpd-sasl-password:passwd
-{%- endif %}
-{%- if neo %}
-  neo-cluster gen-neo-cluster:name
-{%-  if neo[0] %}
-neo-cluster = {{ dumps(neo[0]) }}
-{%-  endif %}
-{%- endif %}
-{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
-{%- if inituser_password %}
-inituser-password = {{ dumps(inituser_password) }}
-{%- endif %}
-{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
-{%- if deadlock_debugger_password %}
-deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
-{%- endif %}
-
-[gen-password]
-recipe = slapos.cookbook:generate.password
-storage-path =
-
-[gen-deadlock-debugger-password]
-<= gen-password
-
-[gen-neo-cluster-base]
-<= gen-password
-
-[gen-neo-cluster]
-name = neo-${gen-neo-cluster-base:passwd}
-
-[gen-smtpd-sasl-password]
-< = gen-password
 
 {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
 {% set zope_address_list_id_dict = {} -%}
@@ -142,6 +114,9 @@ return =
   zope-address-list
   hosts-dict
   monitor-base-url
+{%- if test_runner_enabled %}
+  test-runner-address-list
+{% endif %}
 {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
@@ -169,6 +144,8 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-zodb-dict = {{ dumps(zodb_dict) }}
+config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
+config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
 {% for server_type, server_dict in storage_dict.iteritems() -%}
 {%   if server_type == 'neo' -%}
 config-neo-cluster = ${publish-early:neo-cluster}
@@ -183,6 +160,7 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
+{% set zope_family_name_list = [] -%}
 {% set zope_backend_path_dict = {} -%}
 {% set ssl_authentication_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
@@ -190,6 +168,7 @@ software-type = zope
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{%   do zope_family_name_list.append(zope_family) %}
 {%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {#   # default jupyter zope family is first zope family. -#}
 {#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
@@ -214,6 +193,7 @@ config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longreques
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
 config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
+config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
 {% endfor -%}
 
 {# if not explicitly configured, connect jupyter to first zope family, which  -#}
@@ -268,6 +248,9 @@ return =
 {%- for family in zope_family_dict %}
   {{ family }}
   {{ family }}-v6
+  {% if test_runner_enabled %}
+  {{ family }}-test-runner-url-list
+  {% endif %}
 {% endfor -%}
 {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
 
@@ -275,6 +258,9 @@ config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
 config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
+{%   if test_runner_enabled -%}
+config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }}
+{%   endif -%}
 {% endfor -%}
 # XXX: should those really be same for all families ?
 config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
@@ -335,6 +321,57 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}
+{% for zope_family_name in zope_family_name_list -%}
+{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
+{% endfor -%}
+
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+  inituser-password gen-password:passwd
+  deadlock-debugger-password gen-deadlock-debugger-password:passwd
+{%- if has_posftix %}
+  smtpd-sasl-password gen-smtpd-sasl-password:passwd
+{%- endif %}
+{% for zope_family_name in zope_family_name_list %}
+  {{ zope_family_name }}-test-runner-url-list default-balancer-test-runner-url-list:default
+{% endfor -%}
+{%- if neo %}
+  neo-cluster gen-neo-cluster:name
+{%-  if neo[0] %}
+neo-cluster = {{ dumps(neo[0]) }}
+{%-  endif %}
+{%- endif %}
+{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
+{%- if inituser_password %}
+inituser-password = {{ dumps(inituser_password) }}
+{%- endif %}
+{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
+{%- if deadlock_debugger_password %}
+deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
+{%- endif %}
+
+
+[default-balancer-test-runner-url-list]
+recipe =
+default = not-ready
+
+[gen-password]
+recipe = slapos.cookbook:generate.password
+storage-path =
+
+[gen-deadlock-debugger-password]
+<= gen-password
+
+[gen-neo-cluster-base]
+<= gen-password
+
+[gen-neo-cluster]
+name = neo-${gen-neo-cluster-base:passwd}
+
+[gen-smtpd-sasl-password]
+< = gen-password
 
 [monitor-instance-parameter]
 monitor-httpd-port = 8386

stack/erp5/instance-zope.cfg.in

@@ -8,6 +8,9 @@
 {% set node_id_index_format = '-%%0%ii' % (len(str(instance_index_list[-1])), ) -%}
 {% set part_list = [] -%}
 {% set publish_list = [] -%}
+{% set test_runner_address_list = [] -%}
+{% set test_runner_enabled = slapparameter_dict['test-runner-enabled'] -%}
+{% set test_runner_node_count = slapparameter_dict['test-runner-node-count'] -%}
 {% set longrequest_logger_base_path = buildout_directory ~ '/var/log/longrequest_logger_' -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
@@ -68,84 +71,6 @@ ca-certs = ${directory:test-ca-certs}
 ca-newcerts = ${directory:test-ca-newcerts}
 ca-crl = ${directory:test-ca-crl}
 
-{% if saucelabs_dict -%}
-[test-zelenium-runner-parameter]
-configuration = {{ dumps(saucelabs_dict) }}
-user = {{ dumps(slapparameter_dict['inituser-login']) }}
-password = {{ dumps(slapparameter_dict['inituser-password']) }}
-bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
-
-[{{ section('test-zelenium-runner') }}]
-<= jinja2-template-base
-template = {{ parameter_dict['run-zelenium-template'] }}
-rendered = ${directory:bin}/runTestSuite
-mode = 755
-context =
-    import json_module json
-    key configuration test-zelenium-runner-parameter:configuration
-    key user test-zelenium-runner-parameter:user
-    key password test-zelenium-runner-parameter:password
-    key bin_path test-zelenium-runner-parameter:bin-path
-{% else -%}
-{% if slapparameter_dict['mysql-test-url-list'] -%}
-[{{ section('run-unit-test-userhosts-wrapper') }}]
-<= userhosts-wrapper-base
-wrapped-command-line = ${runUnitTest:wrapper-path}
-wrapper-path = ${buildout:bin-directory}/runUnitTest
-
-[{{ section('run-test-suite-userhosts-wrapper') }}]
-<= userhosts-wrapper-base
-wrapped-command-line = ${runTestSuite:wrapper-path}
-wrapper-path = ${buildout:bin-directory}/runTestSuite
-
-{% set connection_string_list = [] -%}
-{% for url in slapparameter_dict['mysql-test-url-list'] -%}
-{%   set parsed_url = urlparse.urlparse(url) -%}
-{%   do connection_string_list.append(
-       '%s@%s:%s %s %s' % (
-         parsed_url.path.lstrip('/'),
-         parsed_url.hostname,
-         parsed_url.port,
-         parsed_url.username,
-         parsed_url.password,
-       ),
-     ) -%}
-{% endfor -%}
-[run-test-common]
-< = run-common
-environment-extra =
-  REAL_INSTANCE_HOME=${:instance-home}
-  OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
-  TEST_CA_PATH=${test-certificate-authority:ca-dir}
-instance-home = ${directory:unit-test-path}
-wrapper-path = ${directory:bin}/${:command-name}.real
-command-line =
-  '{{ parameter_dict['bin-directory'] }}/${:command-name}'
-  ${:command-line-extra}
-  --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
-  --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
-{#- BBB: We still have test suites that only accept the following 2 options. #}
-  --conversion_server_hostname=erp5-cloudooo
-  --conversion_server_port={{ port_dict['erp5-cloudooo'] }}
-  --volatile_memcached_server_hostname=erp5-memcached-volatile
-  --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
-  --persistent_memcached_server_hostname=erp5-memcached-persistent
-  --persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
-
-[{{ section('runUnitTest') }}]
-< = run-test-common
-command-name = runUnitTest
-command-line-extra =
-  --erp5_sql_connection_string '{{ connection_string_list[0] }}'
-  --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
-
-[{{ section('runTestSuite') }}]
-< = run-test-common
-command-name = runTestSuite
-command-line-extra =
-  --db_list '{{ ','.join(connection_string_list) }}'
-{%- endif %}
-{%- endif %}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -472,6 +397,111 @@ wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
 [{{ section("wait_activities") }}]
 <= watch_activities
 
+
+{% if saucelabs_dict -%}
+[test-zelenium-runner-parameter]
+configuration = {{ dumps(saucelabs_dict) }}
+user = {{ dumps(slapparameter_dict['inituser-login']) }}
+password = {{ dumps(slapparameter_dict['inituser-password']) }}
+bin-path = {{ bin_directory }}/{{ parameter_dict['egg-interpreter'] }}
+
+[{{ section('test-zelenium-runner') }}]
+<= jinja2-template-base
+template = {{ parameter_dict['run-zelenium-template'] }}
+rendered = ${directory:bin}/runTestSuite
+mode = 755
+context =
+    import json_module json
+    key configuration test-zelenium-runner-parameter:configuration
+    key user test-zelenium-runner-parameter:user
+    key password test-zelenium-runner-parameter:password
+    key bin_path test-zelenium-runner-parameter:bin-path
+{% else -%}
+{%   if test_runner_enabled and test_runner_node_count -%}
+{%   for _ in range(test_runner_node_count) %}
+{%     do test_runner_address_list.append((ipv4, next_port())) %}
+{%   endfor %}
+
+[{{ section('run-unit-test-userhosts-wrapper') }}]
+<= userhosts-wrapper-base
+wrapped-command-line = ${runUnitTest:wrapper-path}
+wrapper-path = ${buildout:bin-directory}/runUnitTest
+
+[{{ section('run-test-suite-userhosts-wrapper') }}]
+<= userhosts-wrapper-base
+wrapped-command-line = ${runTestSuite:wrapper-path}
+wrapper-path = ${buildout:bin-directory}/runTestSuite
+
+{%   set connection_string_list = [] -%}
+{%   for url in slapparameter_dict['mysql-test-url-list'] -%}
+{%     set parsed_url = urlparse.urlparse(url) -%}
+{%     do connection_string_list.append(
+         '%s@%s:%s %s %s' % (
+           parsed_url.path.lstrip('/'),
+           parsed_url.hostname,
+           parsed_url.port,
+           parsed_url.username,
+           parsed_url.password,
+         ),
+       ) -%}
+{%   endfor -%}
+[run-test-common]
+< = run-common
+environment-extra =
+  REAL_INSTANCE_HOME=${:instance-home}
+  OPENSSL_BINARY=${test-certificate-authority:openssl-binary}
+  TEST_CA_PATH=${test-certificate-authority:ca-dir}
+instance-home = ${directory:unit-test-path}
+wrapper-path = ${directory:bin}/${:command-name}.real
+command-line =
+  '{{ parameter_dict['bin-directory'] }}/${:command-name}'
+  ${:command-line-extra}
+  --conversion_server_url={{ slapparameter_dict['cloudooo-url'] }}
+  --conversion_server_retry_count={{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
+{#- BBB: We still have test suites that only accept the following 2 options. #}
+  --conversion_server_hostname=erp5-cloudooo
+  --conversion_server_port={{ port_dict['erp5-cloudooo'] }}
+  --volatile_memcached_server_hostname=erp5-memcached-volatile
+  --volatile_memcached_server_port={{ port_dict['erp5-memcached-volatile'] }}
+  --persistent_memcached_server_hostname=erp5-memcached-persistent
+  --persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
+
+[{{ section('runUnitTest') }}]
+< = run-test-common
+command-name = runUnitTest
+command-line-extra =
+  --erp5_sql_connection_string '{{ connection_string_list[0] }}'
+  --extra_sql_connection_string_list '{{ ','.join(connection_string_list[1:]) }}'
+  --zserver {{ test_runner_address_list[0][0] ~ ':' ~ test_runner_address_list[0][1] }}
+  --zserver_frontend_url {{ slapparameter_dict['test-runner-apache-url-list'][0] }}
+
+[{{ section('runTestSuite') }}]
+< = run-test-common
+command-name = runTestSuite
+command-line-extra =
+  --db_list '{{ ','.join(connection_string_list) }}'
+environment-extra +=
+  {#- turn a list of (ip, port) in a list of 'ip:port' #}
+  {% set zserver_address_list = [] -%}
+  {% for ip, port in test_runner_address_list %}
+  {%   do zserver_address_list.append(ip ~ ':' ~ port) %}
+  {% endfor -%}
+  zserver_address_list={{ ','.join(zserver_address_list) }}
+  zserver_frontend_url_list={{ ','.join(slapparameter_dict['test-runner-apache-url-list']) }}
+
+[{{ section("promise-test-runner-apache-url") }}]
+# promise to wait for apache partition to have returned the parameter
+recipe = slapos.cookbook:check_parameter
+value = {{ slapparameter_dict['test-runner-apache-url-list'] }}
+expected-not-value = not-ready
+path = ${directory:promises}/${:_buildout_section_name_}
+expected-value =
+
+{%-   endif %}
+{%- endif %}
+
+
+
 [publish]
 recipe = slapos.cookbook:publish.serialised
 zope-address-list = {{ dumps(publish_list) }}
@@ -484,6 +514,7 @@ hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+test-runner-address-list = {{ dumps(test_runner_address_list) }}
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}