Update Release Candidate

component/boost-lib/buildout.cfg

@@ -27,5 +27,5 @@ environment =
   LZMA_LIBRARY_PATH=${xz-utils:location}/lib
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/fix-ftbfs-python-3.3.patch#c85fb479d51354deafd1cc7af78f25d2
+  https://sources.debian.org/data/main/b/boost1.67/1.67.0-17/debian/patches/fix-ftbfs-python-3.3.patch#c85fb479d51354deafd1cc7af78f25d2
 patch-binary = ${patch:location}/bin/patch

component/boost-lib/fix-ftbfs-python-3.3.patch

-Description: python3.3 has an extra multiarch include location
-Author: Dmitrijs Ledkovs <dmitrij.ledkov@ubuntu.com>
-Last-Update: 2012-10-26
-Forwarded: no
-
---- boost1.65.1-1.65.1+dfsg.orig/tools/build/src/tools/python.jam
-+++ boost1.65.1-1.65.1+dfsg/tools/build/src/tools/python.jam
-@@ -544,7 +544,9 @@
-     }
-     else
-     {
--        includes ?= $(prefix)/include/python$(version) ;
-+        python_includes = [ shell-cmd "printf `python$(version) -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_inc());'`" ] ;
-+        python_platincludes = [ shell-cmd "printf `python$(version) -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_inc(plat_specific=1));'`" ] ;
-+        includes ?= $(python_includes) $(python_platincludes) ;
- 
-         local lib = $(exec-prefix)/lib ;
-         libraries ?= $(lib)/python$(version)/config $(lib) ;

component/cloudooo/buildout.cfg

@@ -14,16 +14,13 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/cloudooo.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = 67e233f25845335aaf191e80abae53733d1f2579
+revision = bf99e5dea3ecf45c59083085540316c48cfa5488
 
 [cloudooo]
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   PasteScript
-  python-magic
-  psutil
-  WSGIUtils
   cloudooo
 entry-points =
   main=cloudooo.paster_application:application

component/firefox/buildout.cfg

@@ -40,6 +40,9 @@ install =
   cd {}
   export LD_LIBRARY_PATH=$PWD:{}
   export PATH={}:$PATH
+  # BBB use a default fonts.conf for compatibility, but it's software instance
+  # responsability to build a fonts.conf with the fonts they want.
+  [ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${firefox-default-fonts-conf:rendered}
   exec ./firefox "$@"
   """.format(
     firefox,
@@ -68,6 +71,21 @@ part = ${firefox-52:location}
 wrapper-name = firefox-51
 part = ${firefox-51:location}
 
+[firefox-default-fonts-conf]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fonts-conf:output}
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/fonts.conf
+context =
+  key cachedir :cache-dir
+  key fonts :fonts
+  key includes :includes
+fonts =
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+includes =
+  ${fontconfig:location}/etc/fonts/conf.d
+cache-dir =
+  ~/.fontconfig-firefox/
 
 [firefox]
 # The default installed firefox version when installing firefox-wrapper.

component/glibmm/buildout.cfg

@@ -4,6 +4,7 @@ extends =
   ../glib/buildout.cfg
   ../libsigc/buildout.cfg
   ../perl/buildout.cfg
+  ../patch/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
@@ -17,8 +18,10 @@ md5sum = e7416beff6cba1f38c2bccd0dc8c3278
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
 configure-options =
   --disable-documentation
+patches = ${:_profile_base_location_}/glibmm-fix-threads-gobject.patch#75ab0e9e5d1df55e8f6177b9934359ae
+patch-options = -p1
 environment =
-  PATH=${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:${glib:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:${glib:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${gettext:location}/include
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib

component/glibmm/glibmm-fix-threads-gobject.patch

+From 37d57ae9572b7d74aa385a30313eceae7f2d3fce Mon Sep 17 00:00:00 2001
+From: Kjell Ahlstedt <kjellahlstedt@gmail.com>
+Date: Wed, 20 Dec 2017 20:00:32 +0100
+Subject: [PATCH] Glib::Threads::Private: Fix gobj()
+
+Bug 791711
+---
+ glib/src/threads.hg | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/glib/src/threads.hg b/glib/src/threads.hg
+index 86d7a17b..c82a6130 100644
+--- a/glib/src/threads.hg
++++ b/glib/src/threads.hg
+@@ -628,7 +628,7 @@ public:
+    */
+   inline void replace(T* data);
+ 
+-  GPrivate* gobj() { return gobject_; }
++  GPrivate* gobj() { return &gobject_; }
+ 
+ private:
+   GPrivate gobject_;
+--- a/glib/glibmm/threads.h	2017-09-04 15:27:31.000000000 +0200
++++ b/glib/glibmm/threads.h	2018-05-05 10:53:44.339288554 +0200
+@@ -657,7 +657,7 @@
+    */
+   inline void replace(T* data);
+ 
+-  GPrivate* gobj() { return gobject_; }
++  GPrivate* gobj() { return &gobject_; }
+ 
+ private:
+   GPrivate gobject_;

component/kumo/buildout.cfg

@@ -8,6 +8,10 @@ extends =
 
 parts = kumo
 
+[gcc]
+# KumoFS fails to build with GCC 6.
+min_version = 7
+
 [kumo]
 recipe = slapos.recipe.cmmi
 shared = true

component/mariadb/buildout.cfg

@@ -30,13 +30,14 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.4.12
-md5sum = 97d7c0f508c04a31c138fdb24e95dbc4
+version = 10.4.13
+md5sum = 89e832d6d89dc9bd4c98657a954e0f2c
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
   grep -q "$@"
   sed -i "/$1/d" "$2"
+  mv cmake/Findzstd.cmake cmake/FindZSTD.cmake
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=${:location}
@@ -80,6 +81,10 @@ patch-options = -p1
 patches =
   https://sources.debian.org/data/main/m/mariadb-10.3/1:10.3.22-0+deb10u1/debian/patches/0024-Revert-to-using-system-pcre-library.patch#1c6a0f2634f5a56122299674b77b1131
 post-install = 
+  ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
+  for x in ${lz4:location} ${snappy:location} ${zstd:location}
+  do echo "$ldd" |grep -qF " $x/lib/"
+  done
   set -- wsrep-lib/wsrep-API/*/wsrep_api.h
   install -DpT $1 ${:location}/$1
   cp -a wsrep-lib/include ${:location}/wsrep-lib

component/re6stnet/buildout.cfg

@@ -9,6 +9,9 @@ extends =
 parts =
   re6stnet
 
+[gcc]
+min_version = 0
+
 [re6stnet]
 recipe = zc.recipe.egg
 eggs =

component/userhosts/buildout.cfg

-[buildout]
-extends =
-  ../git/buildout.cfg
-
 [userhosts]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = cd ${userhosts-get:location} && make
-# For convenience (one section to build & know result path)
-location = ${userhosts-get:location}/userhosts
-
-[userhosts-get]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/userhosts.git
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
 revision = 1d3b463e7856db6e674a06258c0840206e6a7b72
-git-executable = ${git:location}/bin/git
-location = ${buildout:parts-directory}/userhosts
+configure-command = true
+make-options = PREFIX=@@LOCATION@@
+make-targets = check install

software/build-rina/software.cfg

@@ -8,6 +8,9 @@ parts =
   template
 download-cache = ${:directory}/download-cache
 
+[gcc]
+min_version = 0
+
 [template]
 recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe

software/buildout-testing/runTestSuite.in

@@ -6,6 +6,11 @@ from erp5.util import taskdistribution
 from erp5.util.testsuite import SubprocessError, TestSuite
 from zc.buildout.buildout import Buildout
 
+if str is bytes:
+  str2bytes = lambda s: s
+else:
+  str2bytes = lambda s: s.encode()
+
 slapos_buildout = {{repr(slapos_buildout)}}
 test_dict = {
   'zc.buildout': slapos_buildout,
@@ -90,7 +95,7 @@ def main():
 
   fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666)
   try:
-    os.write(fd, """\
+    os.write(fd, str2bytes("""\
 [buildout]
 extends = %s
 develop =%s
@@ -112,7 +117,7 @@ scripts =
   zope-testrunner
 """ % (os.path.join(slapos_buildout, 'buildout.cfg'),
        ''.join('\n  ' + x for x in test_dict.itervalues()),
-       '\n'.join(x + ' =' for x in test_dict)))
+       '\n'.join(x + ' =' for x in test_dict))))
   finally:
     os.close(fd)
   Buildout('buildout.cfg', {}).install(['bootstrap'])

software/buildout-testing/software-py3.cfg

+[buildout]
+extends =
+  software.cfg
+
+[python]
+part = python3

software/caddy-frontend/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 087bd9404cd120bd7602a9fbfcddc064
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 29a61267959cc9ba7cdcd96fef41641a
+md5sum = d96fea7dd4d7f0a157c86d25a263d8e1
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -42,7 +42,7 @@ md5sum = 7e3ee70c447f8203273d78f66ab519c3
 
 [template-caddy-frontend-configuration]
 filename = templates/Caddyfile.in
-md5sum = 908b859ff76469381024947f5c98c891
+md5sum = f0faf6d2e6c187df7e25bf717676f9df
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in

software/caddy-frontend/templates/Caddyfile.in

@@ -4,7 +4,8 @@ import {{frontend_configuration.get('log-access-configuration')}}
 import {{ slave_configuration_directory }}/*.conf
 import {{ slave_with_cache_configuration_directory }}/*.conf
 
-:{{ https_port }} {
+{% for port in [https_port] %}
+:{{ port }} {
   tls {{ master_certificate }} {{ master_certificate }}
   bind {{ local_ipv4 }}
   status 404 /
@@ -16,8 +17,10 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
     * {{ not_found_file }}
   }
 }
+{%- endfor %}
 
-:{{ http_port }} {
+{% for port in [http_port, cached_port, ssl_cached_port] %}
+:{{ port }} {
   bind {{ local_ipv4 }}
   status 404 /
   log / {{ access_log }} "{remote} - {>REMOTE_USER} [{when}] \"{method} {uri} {proto}\" {status} {size} \"{>Referer}\" \"{>User-Agent}\" {latency_ms}" {
@@ -28,6 +31,7 @@ import {{ slave_with_cache_configuration_directory }}/*.conf
     * {{ not_found_file }}
   }
 }
+{%- endfor %}
 
 # Access to server-status Caddy-style
 https://[{{ global_ipv6 }}]:{{ https_port }}/server-status, https://{{ local_ipv4 }}:{{ https_port }}/server-status {

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -42,6 +42,10 @@ create = true
 {%   set slave_kedifa_information = {} %}
 {% endif %}
 
+# empty sections if no slaves are available
+[slave-log-directory-dict]
+[slave-password]
+
 # empty section if no cached slaves are available
 [slave-log-cache-direct-directory-dict]
 

software/caddy-frontend/test/test.py

@@ -500,193 +500,19 @@ class TestHandler(BaseHTTPRequestHandler):
     self.wfile.write(response)
 
 
-class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
+class HttpFrontendTestCase(SlapOSInstanceTestCase):
   # show full diffs, as it is required for proper analysis of problems
   maxDiff = None
 
   # minimise partition path
   __partition_reference__ = 'T-'
 
-  @classmethod
-  def callSupervisorMethod(cls, method, *args, **kwargs):
-    with cls.slap.instance_supervisor_rpc as instance_supervisor:
-      return getattr(instance_supervisor, method)(*args, **kwargs)
-
-  @classmethod
-  def requestSlaves(cls):
-    software_url = cls.getSoftwareURL()
-    for slave_reference, partition_parameter_kw in cls\
-            .getSlaveParameterDictDict().items():
-      cls.logger.debug(
-        'requesting slave "%s" software:%s parameters:%s',
-        slave_reference, software_url, partition_parameter_kw)
-      cls.slap.request(
-        software_release=software_url,
-        partition_reference=slave_reference,
-        partition_parameter_kw=partition_parameter_kw,
-        shared=True
-      )
-
   @classmethod
   def getInstanceSoftwareType(cls):
     # Because of unknown problem yet, the root instance software type changes
     # from RootSoftwareInstance to '', so always request it with given type
     return "RootSoftwareInstance"
 
-  @classmethod
-  def requestDefaultInstance(cls, state='started'):
-    default_instance = super(
-      SlaveHttpFrontendTestCase, cls).requestDefaultInstance(state=state)
-    if state != 'destroyed':
-      cls.requestSlaves()
-    return default_instance
-
-  @classmethod
-  def setUpClass(cls):
-    try:
-      cls.createWildcardExampleComCertificate()
-      cls.startServerProcess()
-    except BaseException:
-      cls.logger.exception("Error during setUpClass")
-      cls._cleanup("{}.{}.setUpClass".format(cls.__module__, cls.__name__))
-      cls.setUp = lambda self: self.fail('Setup Class failed.')
-      raise
-
-    super(SlaveHttpFrontendTestCase, cls).setUpClass()
-
-    try:
-      # expose instance directory
-      cls.instance_path = cls.slap.instance_directory
-      # expose software directory, extract from found computer partition
-      cls.software_path = os.path.realpath(os.path.join(
-          cls.computer_partition_root_path, 'software_release'))
-      # do working directory
-      cls.working_directory = os.path.join(os.path.realpath(
-          os.environ.get(
-              'SLAPOS_TEST_WORKING_DIR',
-              os.path.join(os.getcwd(), '.slapos'))),
-          'caddy-frontend-test')
-      if not os.path.isdir(cls.working_directory):
-        os.mkdir(cls.working_directory)
-      cls.setUpMaster()
-      cls.setUpSlaves()
-      cls.waitForCaddy()
-    except BaseException:
-      cls.logger.exception("Error during setUpClass")
-      # "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used
-      # by SlapOSInstanceTestCase.setUpClass so we use another name for
-      # snapshot, to make sure we don't store another snapshot in same
-      # directory.
-      cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(
-        cls.__module__, cls.__name__))
-      cls.setUp = lambda self: self.fail('Setup Class failed.')
-      raise
-
-  def assertLogAccessUrlWithPop(self, parameter_dict):
-    log_access_url = parameter_dict.pop('log-access-url')
-
-    self.assertTrue(len(log_access_url) >= 1)
-    # check only the first one, as second frontend will be stopped
-    log_access = log_access_url[0]
-    entry = log_access.split(': ')
-    if len(entry) != 2:
-      self.fail('Cannot parse %r' % (log_access,))
-    frontend, url = entry
-    result = requests.get(url, verify=False)
-    self.assertEqual(
-      httplib.OK,
-      result.status_code,
-      'While accessing %r of %r the status code was %r' % (
-        url, frontend, result.status_code))
-
-  def assertKedifaKeysWithPop(self, parameter_dict, prefix=''):
-    generate_auth_url = parameter_dict.pop('%skey-generate-auth-url' % (
-      prefix,))
-    upload_url = parameter_dict.pop('%skey-upload-url' % (prefix,))
-    kedifa_ipv6_base = 'https://[%s]:%s' % (self._ipv6_address, KEDIFA_PORT)
-    base = '^' + kedifa_ipv6_base.replace(
-      '[', r'\[').replace(']', r'\]') + '/.{32}'
-    self.assertRegexpMatches(
-      generate_auth_url,
-      base + r'\/generateauth$'
-    )
-    self.assertRegexpMatches(
-      upload_url,
-      base + r'\?auth=$'
-    )
-
-    kedifa_caucase_url = parameter_dict.pop('kedifa-caucase-url')
-    self.assertEqual(
-      kedifa_caucase_url,
-      'http://[%s]:%s' % (self._ipv6_address, CAUCASE_PORT),
-    )
-
-    return generate_auth_url, upload_url
-
-  def assertKeyWithPop(self, key, d):
-    self.assertTrue(key in d, 'Key %r is missing in %r' % (key, d))
-    d.pop(key)
-
-  def assertEqualResultJson(self, result, key, value):
-    try:
-      j = result.json()
-    except Exception:
-      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
-    self.assertTrue(key in j, 'No key %r in %s' % (key, j))
-    self.assertEqual(value, j[key])
-
-  def parseParameterDict(self, parameter_dict):
-    parsed_parameter_dict = {}
-    for key, value in parameter_dict.items():
-      if key in [
-        'rejected-slave-dict',
-        'warning-slave-dict',
-        'warning-list',
-        'request-error-list',
-        'log-access-url']:
-        value = json.loads(value)
-      parsed_parameter_dict[key] = value
-    return parsed_parameter_dict
-
-  def parseConnectionParameterDict(self):
-    return self.parseParameterDict(
-      self.requestDefaultInstance().getConnectionParameterDict()
-    )
-
-  @classmethod
-  def runComputerPartitionUntil(cls, until):
-    max_try = 10
-    try_num = 1
-    while True:
-      if until():
-        break
-      if try_num > max_try:
-        raise ValueError('Failed to run computer partition with %r' % (until,))
-      try:
-        cls.slap.waitForInstance()
-      except Exception:
-        cls.logger.exception("Error during until run")
-      try_num += 1
-
-  @classmethod
-  def untilNotReadyYetNotInMasterKeyGenerateAuthUrl(cls):
-    parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
-    key = 'master-key-generate-auth-url'
-    if key not in parameter_dict:
-      return False
-    if 'NotReadyYet' in parameter_dict[key]:
-      return False
-    return True
-
-  @classmethod
-  def getSlaveParameterDictDict(cls):
-    return {
-      'waitforcaddyslave': {
-        'url': cls.backend_url,
-        'enable_cache': True,
-      }
-    }
-
   @classmethod
   def startServerProcess(cls):
     server = HTTPServer(
@@ -789,50 +615,6 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
       # block until caddy is refreshed
       time.sleep(2)
 
-  @classmethod
-  def untilSlavePartitionReady(cls):
-    # all on-watch services shall not be exited
-    for process in cls.callSupervisorMethod('getAllProcessInfo'):
-      if process['name'].endswith('-on-watch') and \
-        process['statename'] == 'EXITED':
-        if process['name'].startswith('monitor-http'):
-          continue
-        return False
-    for slave_reference, partition_parameter_kw in cls\
-            .getSlaveParameterDictDict().items():
-      parameter_dict = cls.slap.request(
-        software_release=cls.getSoftwareURL(),
-        partition_reference=slave_reference,
-        partition_parameter_kw=partition_parameter_kw,
-        shared=True
-      ).getConnectionParameterDict()
-
-      log_access_ready = 'log-access-url' in parameter_dict
-      key = 'key-generate-auth-url'
-      key_generate_auth_ready = key in parameter_dict \
-          and 'NotReadyYet' not in parameter_dict[key]
-      if log_access_ready and key_generate_auth_ready:
-        return True
-    return False
-
-  @classmethod
-  def setUpSlaves(cls):
-    cls.slave_connection_parameter_dict_dict = {}
-    # run partition for slaves to be setup
-    cls.runComputerPartitionUntil(
-      cls.untilSlavePartitionReady)
-    request = cls.slap.request
-    for slave_reference, partition_parameter_kw in cls\
-            .getSlaveParameterDictDict().items():
-      slave_instance = request(
-        software_release=cls.getSoftwareURL(),
-        partition_reference=slave_reference,
-        partition_parameter_kw=partition_parameter_kw,
-        shared=True
-      )
-      cls.slave_connection_parameter_dict_dict[slave_reference] = \
-          slave_instance.getConnectionParameterDict()
-
   @classmethod
   def createWildcardExampleComCertificate(cls):
     _, cls.key_pem, _, cls.certificate_pem = createSelfSignedCertificate(
@@ -842,39 +624,88 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
         '*.alias1.example.com',
       ])
 
-  check_slave_id = 'waitforcaddyslave'
   @classmethod
-  def waitForCaddy(cls):
-    # awaits until caddy is ready to serve slaves
-    parameter_dict = cls.slave_connection_parameter_dict_dict[
-      cls.check_slave_id
-    ]
-    wait_time = 600
-    begin = time.time()
-    try_num = 0
-    cls.logger.debug('waitForCaddy for %is' % (wait_time,))
+  def runComputerPartitionUntil(cls, until):
+    max_try = 10
+    try_num = 1
     while True:
+      if until():
+        break
+      if try_num > max_try:
+        raise ValueError('Failed to run computer partition with %r' % (until,))
       try:
-        try_num += 1
-        fakeHTTPSResult(
-          parameter_dict['domain'], parameter_dict['public-ipv4'],
-          '/',
-        )
+        cls.slap.waitForInstance()
       except Exception:
-        if time.time() - begin > wait_time:
-          cls.logger.exception(
-            "Error during waitForCaddy after %.2fs" % ((time.time() - begin),))
-          raise
-        else:
-          time.sleep(0.5)
-      else:
-        cls.logger.info("waitForCaddy took %.2fs" % ((time.time() - begin),))
-        break
+        cls.logger.exception("Error during until run")
+      try_num += 1
 
   @classmethod
-  def _cleanup(cls, snapshot_name):
-    cls.stopServerProcess()
-    super(SlaveHttpFrontendTestCase, cls)._cleanup(snapshot_name)
+  def untilNotReadyYetNotInMasterKeyGenerateAuthUrl(cls):
+    parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
+    key = 'master-key-generate-auth-url'
+    if key not in parameter_dict:
+      return False
+    if 'NotReadyYet' in parameter_dict[key]:
+      return False
+    return True
+
+  @classmethod
+  def callSupervisorMethod(cls, method, *args, **kwargs):
+    with cls.slap.instance_supervisor_rpc as instance_supervisor:
+      return getattr(instance_supervisor, method)(*args, **kwargs)
+
+  def assertLogAccessUrlWithPop(self, parameter_dict):
+    log_access_url = parameter_dict.pop('log-access-url')
+
+    self.assertTrue(len(log_access_url) >= 1)
+    # check only the first one, as second frontend will be stopped
+    log_access = log_access_url[0]
+    entry = log_access.split(': ')
+    if len(entry) != 2:
+      self.fail('Cannot parse %r' % (log_access,))
+    frontend, url = entry
+    result = requests.get(url, verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code,
+      'While accessing %r of %r the status code was %r' % (
+        url, frontend, result.status_code))
+
+  def assertKedifaKeysWithPop(self, parameter_dict, prefix=''):
+    generate_auth_url = parameter_dict.pop('%skey-generate-auth-url' % (
+      prefix,))
+    upload_url = parameter_dict.pop('%skey-upload-url' % (prefix,))
+    kedifa_ipv6_base = 'https://[%s]:%s' % (self._ipv6_address, KEDIFA_PORT)
+    base = '^' + kedifa_ipv6_base.replace(
+      '[', r'\[').replace(']', r'\]') + '/.{32}'
+    self.assertRegexpMatches(
+      generate_auth_url,
+      base + r'\/generateauth$'
+    )
+    self.assertRegexpMatches(
+      upload_url,
+      base + r'\?auth=$'
+    )
+
+    kedifa_caucase_url = parameter_dict.pop('kedifa-caucase-url')
+    self.assertEqual(
+      kedifa_caucase_url,
+      'http://[%s]:%s' % (self._ipv6_address, CAUCASE_PORT),
+    )
+
+    return generate_auth_url, upload_url
+
+  def assertKeyWithPop(self, key, d):
+    self.assertTrue(key in d, 'Key %r is missing in %r' % (key, d))
+    d.pop(key)
+
+  def assertEqualResultJson(self, result, key, value):
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertTrue(key in j, 'No key %r in %s' % (key, j))
+    self.assertEqual(value, j[key])
 
   def patchRequests(self):
     HTTPResponse = requests.packages.urllib3.response.HTTPResponse
@@ -917,7 +748,216 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
 
   def tearDown(self):
     self.unpatchRequests()
-    super(SlaveHttpFrontendTestCase, self).tearDown()
+    super(HttpFrontendTestCase, self).tearDown()
+
+  def parseParameterDict(self, parameter_dict):
+    parsed_parameter_dict = {}
+    for key, value in parameter_dict.items():
+      if key in [
+        'rejected-slave-dict',
+        'warning-slave-dict',
+        'warning-list',
+        'request-error-list',
+        'log-access-url']:
+        value = json.loads(value)
+      parsed_parameter_dict[key] = value
+    return parsed_parameter_dict
+
+  def getMasterPartitionPath(self):
+    return '/' + os.path.join(
+      *glob.glob(
+        os.path.join(
+          self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave'
+        )
+      )[0].split('/')[:-2])
+
+  def parseConnectionParameterDict(self):
+    return self.parseParameterDict(
+      self.requestDefaultInstance().getConnectionParameterDict()
+    )
+
+  @classmethod
+  def waitForMethod(cls, name, method):
+    wait_time = 600
+    begin = time.time()
+    try_num = 0
+    cls.logger.debug('%s for %is' % (name, wait_time,))
+    while True:
+      try:
+        try_num += 1
+        method()
+      except Exception:
+        if time.time() - begin > wait_time:
+          cls.logger.exception(
+            "Error during %s after %.2fs" % (name, (time.time() - begin),))
+          raise
+        else:
+          time.sleep(0.5)
+      else:
+        cls.logger.info("%s took %.2fs" % (name, (time.time() - begin),))
+        break
+
+  @classmethod
+  def waitForCaddy(cls):
+    def method():
+      fakeHTTPSResult(
+        cls._ipv4_address, cls._ipv4_address,
+        '/',
+      )
+    cls.waitForMethod('waitForCaddy', method)
+
+  @classmethod
+  def _cleanup(cls, snapshot_name):
+    cls.stopServerProcess()
+    super(HttpFrontendTestCase, cls)._cleanup(snapshot_name)
+
+  @classmethod
+  def setUpClass(cls):
+    try:
+      cls.createWildcardExampleComCertificate()
+      cls.startServerProcess()
+    except BaseException:
+      cls.logger.exception("Error during setUpClass")
+      cls._cleanup("{}.{}.setUpClass".format(cls.__module__, cls.__name__))
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+    super(HttpFrontendTestCase, cls).setUpClass()
+
+    try:
+      # expose instance directory
+      cls.instance_path = cls.slap.instance_directory
+      # expose software directory, extract from found computer partition
+      cls.software_path = os.path.realpath(os.path.join(
+          cls.computer_partition_root_path, 'software_release'))
+      # do working directory
+      cls.working_directory = os.path.join(os.path.realpath(
+          os.environ.get(
+              'SLAPOS_TEST_WORKING_DIR',
+              os.path.join(os.getcwd(), '.slapos'))),
+          'caddy-frontend-test')
+      if not os.path.isdir(cls.working_directory):
+        os.mkdir(cls.working_directory)
+      cls.setUpMaster()
+      cls.waitForCaddy()
+    except BaseException:
+      cls.logger.exception("Error during setUpClass")
+      # "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used
+      # by SlapOSInstanceTestCase.setUpClass so we use another name for
+      # snapshot, to make sure we don't store another snapshot in same
+      # directory.
+      cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(
+        cls.__module__, cls.__name__))
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+
+class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
+  @classmethod
+  def requestDefaultInstance(cls, state='started'):
+    default_instance = super(
+      SlaveHttpFrontendTestCase, cls).requestDefaultInstance(state=state)
+    if state != 'destroyed':
+      cls.requestSlaves()
+    return default_instance
+
+  @classmethod
+  def requestSlaves(cls):
+    software_url = cls.getSoftwareURL()
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      cls.logger.debug(
+        'requesting slave "%s" software:%s parameters:%s',
+        slave_reference, software_url, partition_parameter_kw)
+      cls.slap.request(
+        software_release=software_url,
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      )
+
+  @classmethod
+  def setUpClass(cls):
+    super(SlaveHttpFrontendTestCase, cls).setUpClass()
+
+    try:
+      cls.setUpSlaves()
+      cls.waitForSlave()
+    except BaseException:
+      cls.logger.exception("Error during setUpClass")
+      # "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used
+      # by SlapOSInstanceTestCase.setUpClass so we use another name for
+      # snapshot, to make sure we don't store another snapshot in same
+      # directory.
+      cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(
+        cls.__module__, cls.__name__))
+      cls.setUp = lambda self: self.fail('Setup Class failed.')
+      raise
+
+  @classmethod
+  def waitForSlave(cls):
+    def method():
+      for parameter_dict in cls.getSlaveConnectionParameterDictList():
+        if 'domain' in parameter_dict and 'public-ipv4' in parameter_dict:
+          try:
+            fakeHTTPSResult(
+              parameter_dict['domain'], parameter_dict['public-ipv4'], '/')
+          except requests.exceptions.InvalidURL:
+            # ignore slaves to which connection is impossible by default
+            continue
+    cls.waitForMethod('waitForSlave', method)
+
+  @classmethod
+  def getSlaveConnectionParameterDictList(cls):
+    parameter_dict_list = []
+
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      parameter_dict_list.append(cls.slap.request(
+        software_release=cls.getSoftwareURL(),
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      ).getConnectionParameterDict())
+    return parameter_dict_list
+
+  @classmethod
+  def untilSlavePartitionReady(cls):
+    # all on-watch services shall not be exited
+    for process in cls.callSupervisorMethod('getAllProcessInfo'):
+      if process['name'].endswith('-on-watch') and \
+        process['statename'] == 'EXITED':
+        if process['name'].startswith('monitor-http'):
+          continue
+        return False
+
+    for parameter_dict in cls.getSlaveConnectionParameterDictList():
+      log_access_ready = 'log-access-url' in parameter_dict
+      key = 'key-generate-auth-url'
+      key_generate_auth_ready = key in parameter_dict \
+          and 'NotReadyYet' not in parameter_dict[key]
+      if not(log_access_ready and key_generate_auth_ready):
+        return False
+
+    return True
+
+  @classmethod
+  def setUpSlaves(cls):
+    cls.slave_connection_parameter_dict_dict = {}
+    # run partition for slaves to be setup
+    cls.runComputerPartitionUntil(
+      cls.untilSlavePartitionReady)
+    request = cls.slap.request
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      slave_instance = request(
+        software_release=cls.getSoftwareURL(),
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      )
+      cls.slave_connection_parameter_dict_dict[slave_reference] = \
+          slave_instance.getConnectionParameterDict()
 
   def parseSlaveParameterDict(self, key):
     return self.parseParameterDict(
@@ -945,16 +985,8 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
 
     return parameter_dict
 
-  def getMasterPartitionPath(self):
-    return '/' + os.path.join(
-      *glob.glob(
-        os.path.join(
-          self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave'
-        )
-      )[0].split('/')[:-2])
-
 
-class TestMasterRequestDomain(SlaveHttpFrontendTestCase, TestDataMixin):
+class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
@@ -965,14 +997,7 @@ class TestMasterRequestDomain(SlaveHttpFrontendTestCase, TestDataMixin):
       'caucase_port': CAUCASE_PORT,
     }
 
-  @classmethod
-  def waitForCaddy(cls):
-    pass
-
   def test(self):
-    # run partition until AIKC finishes
-    self.runComputerPartitionUntil(
-      self.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
@@ -982,16 +1007,16 @@ class TestMasterRequestDomain(SlaveHttpFrontendTestCase, TestDataMixin):
       {
         'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
         'domain': 'example.com',
-        'accepted-slave-amount': '1',
+        'accepted-slave-amount': '0',
         'rejected-slave-amount': '0',
-        'slave-amount': '1',
+        'slave-amount': '0',
         'rejected-slave-dict': {}
       },
       parameter_dict
     )
 
 
-class TestMasterRequest(SlaveHttpFrontendTestCase, TestDataMixin):
+class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
@@ -1001,14 +1026,7 @@ class TestMasterRequest(SlaveHttpFrontendTestCase, TestDataMixin):
       'caucase_port': CAUCASE_PORT,
     }
 
-  @classmethod
-  def waitForCaddy(cls):
-    pass
-
   def test(self):
-    # run partition until AIKC finishes
-    self.runComputerPartitionUntil(
-      self.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
     parameter_dict = self.parseConnectionParameterDict()
     self.assertKeyWithPop('monitor-setup-url', parameter_dict)
     self.assertKedifaKeysWithPop(parameter_dict, 'master-')
@@ -1017,9 +1035,9 @@ class TestMasterRequest(SlaveHttpFrontendTestCase, TestDataMixin):
       {
         'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
         'domain': 'None',
-        'accepted-slave-amount': '1',
+        'accepted-slave-amount': '0',
         'rejected-slave-amount': '0',
-        'slave-amount': '1',
+        'slave-amount': '0',
         'rejected-slave-dict': {}},
       parameter_dict
     )
@@ -1101,8 +1119,6 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'request-timeout': '12',
     }
 
-  check_slave_id = 'Url'
-
   @classmethod
   def startServerProcess(cls):
     cls.ca = CertificateAuthority('TestSlave')
@@ -1621,7 +1637,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
                  r'"python-requests.*" \d+'
 
     self.assertRegexpMatches(
-      open(log_file, 'r').read(),
+      open(log_file, 'r').readlines()[-1],
       log_regexp)
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
@@ -4270,7 +4286,6 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       'caucase_port': CAUCASE_PORT,
     }
 
-  check_slave_id = 'replicate'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4338,7 +4353,6 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
       'caucase_port': CAUCASE_PORT,
     }
 
-  check_slave_id = 'empty'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4377,7 +4391,6 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
       'caucase_port': CAUCASE_PORT,
     }
 
-  check_slave_id = 'dummy-cached'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4469,7 +4482,6 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
       'caucase_port': CAUCASE_PORT,
     }
 
-  check_slave_id = 'dummy-cached'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4559,10 +4571,6 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
       'caucase_port': CAUCASE_PORT,
     }
 
-  @classmethod
-  def waitForCaddy(cls):
-    pass
-
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4572,6 +4580,11 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
       },
     }
 
+  @classmethod
+  def waitForSlave(cls):
+    # no need to wait for slave availability here
+    return True
+
   def test_default(self):
     parameter_dict = self.parseSlaveParameterDict('default')
     self.assertLogAccessUrlWithPop(parameter_dict)
@@ -4618,10 +4631,6 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
       'caucase_port': CAUCASE_PORT,
     }
 
-  @classmethod
-  def waitForCaddy(cls):
-    pass
-
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4777,7 +4786,6 @@ class TestDefaultMonitorHttpdPort(SlaveHttpFrontendTestCase, TestDataMixin):
   def runKedifaUpdater(cls):
     return
 
-  check_slave_id = 'test'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -4828,7 +4836,6 @@ class TestQuicEnabled(SlaveHttpFrontendTestCase, TestDataMixin):
       'caucase_port': CAUCASE_PORT,
     }
 
-  check_slave_id = 'url'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -5591,7 +5598,6 @@ class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
       'mpm-graceful-shutdown-timeout': 2,
     }
 
-  check_slave_id = 'ssl_from_master_kedifa_overrides_master_certificate'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -5730,7 +5736,6 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       'mpm-graceful-shutdown-timeout': 2,
     }
 
-  check_slave_id = 'ssl_from_master'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -6489,7 +6494,6 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
     cls.instance_parameter_dict['public-ipv4'] = cls._ipv4_address
     return cls.instance_parameter_dict
 
-  check_slave_id = 'ssl_from_master'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {
@@ -6591,7 +6595,6 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
       'ciphers': 'ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384'
     }
 
-  check_slave_id = 'default_ciphers'
   @classmethod
   def getSlaveParameterDictDict(cls):
     return {

software/caddy-frontend/test/test_data/test.TestMasterRequest.test_file_list_log-CADDY.txt

@@ -7,11 +7,7 @@ T-1/var/log/monitor-httpd-access.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
-T-2/var/log/httpd-cache-direct/_waitforcaddyslave_access_log
-T-2/var/log/httpd-cache-direct/_waitforcaddyslave_error_log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
-T-2/var/log/httpd/_waitforcaddyslave_access_log
-T-2/var/log/httpd/_waitforcaddyslave_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
 T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestMasterRequestDomain.test_file_list_log-CADDY.txt

@@ -7,11 +7,7 @@ T-1/var/log/monitor-httpd-access.log
 T-1/var/log/monitor-httpd-error.log
 T-2/var/log/frontend-access.log
 T-2/var/log/frontend-error.log
-T-2/var/log/httpd-cache-direct/_waitforcaddyslave_access_log
-T-2/var/log/httpd-cache-direct/_waitforcaddyslave_error_log
 T-2/var/log/httpd-csr_id/expose-csr_id.log
-T-2/var/log/httpd/_waitforcaddyslave_access_log
-T-2/var/log/httpd/_waitforcaddyslave_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log
 T-2/var/log/trafficserver/manager.log

software/cloudooo/software-common.cfg

@@ -7,11 +7,6 @@ extends =
 parts =
   ${cloudooo-buildout:parts}
 
-[gcc]
-# For old version of glibmm.
-part = gcc-5.5
-max_version = 6
-
 [cloudooo-buildout]
 parts =
   ${stack-cloudooo-buildout:parts}

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 filename = instance-default.cfg
-md5sum = cf68a9c7e03ee0a3a527c4edfb6ab5d1
+md5sum = 40364ff26e9284cea97a58f3cd8c75e3

software/erp5testnode/instance-default.cfg

@@ -240,5 +240,5 @@ config-port = $${shellinabox-frontend:port}
 node-quantity = 1
 test-suite-master-url =
 instance-dict =
-software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.152/software/seleniumrunner/software.cfg"]
+software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.154/software/seleniumrunner/software.cfg"]
 keep-log-days = 15

software/jstestnode/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = ab08e75c26d06d95674042d4a2934716
+md5sum = cd8648ca8a4a098e1a6faf7246f0d395
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in
@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 5cac160fd6f14cd69cc8d63f87cc9726
+md5sum = bb3f053b6cdb0a8888e9d32e63085ed5

software/jstestnode/instance.cfg.in

@@ -28,6 +28,21 @@ www = $${:srv}/www
 home = $${:etc}/home
 ssl = $${:etc}/ssl
 framebuffer = $${:srv}/framebuffer
+fontconfig-cache = $${buildout:directory}/.fontconfig
+
+[fontconfig-conf]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fonts-conf:output}
+rendered = $${directory:etc}/fonts.conf
+context =
+  key cachedir directory:fontconfig-cache
+  key fonts :fonts
+  key includes :includes
+fonts =
+  ${ipaex-fonts:location}
+  ${liberation-fonts:location}
+includes =
+  ${fontconfig:location}/etc/fonts/conf.d
 
 #################################
 # Test runner

software/jstestnode/runTestSuite.in

@@ -20,6 +20,7 @@ import json
 
 os.environ['XORG_LOCK_DIR'] = '$${xvfb-instance:lock-dir}'
 os.environ['DISPLAY'] = '$${xvfb-instance:display}'
+os.environ['FONTCONFIG_FILE'] = '$${fontconfig-conf:rendered}'
 
 BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
 ETC_DIRECTORY = '$${directory:etc}'

software/jstestnode/software.cfg

@@ -5,6 +5,8 @@ extends =
   ../../component/coreutils/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/xorg/buildout.cfg
+  ../../component/fontconfig/buildout.cfg
+  ../../component/fonts/buildout.cfg
   ../../component/firefox/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/nginx/buildout.cfg

software/kvm/software.cfg

@@ -51,6 +51,9 @@ eggs =
   websockify
   slapos.cookbook
   erp5.util
+# BBB: eggs used as recipe should be kept otherwise sections depending
+# on it can't be uninstalled
+  collective.recipe.shelloutput
 scripts =
   websockify
 

software/kvm/test/test.py

@@ -365,14 +365,6 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
     )
     self.assertIn('<title>noVNC</title>', result.text)
 
-    result = requests.get(
-      connection_parameter_dict['KVM1-url'], verify=False)
-    self.assertEqual(
-      httplib.OK,
-      result.status_code
-    )
-    self.assertIn('<title>noVNC</title>', result.text)
-
 @skipUnlessKvm
 class TestInstanceResilient(InstanceTestCase):
   __partition_reference__ = 'ir'

software/metabase/README.md

+# Metabae
+
+https://www.metabase.com/
+
+## TODO:
+ * export backups for resilience
+ * security (proper passwords, verifiable certificate, study metabase encryption option)

software/metabase/buildout.hash.cfg

+[instance-profile]
+filename = instance.cfg.in
+md5sum = 8e48fa7c66a59b3d5faf0216922a574f

software/metabase/instance.cfg.in

+[buildout]
+parts =
+  publish-connection-parameter
+
+extends = ${monitor2-template:rendered}
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[metabase-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:service}/$${:_buildout_section_name_}
+command-line = sh -c "cd $${directory:srv-metabase}; ${java-re-8:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar"
+# https://www.metabase.com/docs/latest/operations-guide/customizing-jetty-webserver.html
+# note that we set org.quartz.scheduler.instanceId through $JAVA_ARGS as a workaround for machines
+# which cannot resolve their hostnames. See also https://github.com/metabase/metabase/issues/8373
+environment =
+  MB_EMOJI_IN_LOGS=false
+  MB_JETTY_HOST=$${:ip}
+  MB_JETTY_PORT=$${:-http-port}
+  MB_JETTY_SSL_PORT=$${:port}
+  MB_JETTY_SSL=true
+  MB_JETTY_SSL_KEYSTORE=$${metabase-keystore:file}
+  MB_JETTY_SSL_KEYSTORE_PASSWORD=$${metabase-keystore:password}
+  MB_DB_TYPE=postgres
+  MB_DB_DBNAME=$${postgresql:dbname}
+  MB_DB_PORT=$${postgresql:port}
+  MB_DB_USER=$${postgresql:superuser}
+  MB_DB_PASS=$${postgresql:password}
+  MB_DB_HOST=$${postgresql:ipv4}
+  JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id}
+hash-existing-files =
+  $${buildout:directory}/software_release/buildout.cfg
+
+ip = $${instance-parameter:ipv6-random}
+port = 8443
+# XXX It does not seem we can prevent metabase to also listen on http, so we
+# give it an http port, but don't use it.
+-http-port = 18080
+hostname = [$${:ip}]
+scheme = https
+url = $${:scheme}://$${:hostname}:$${:port}
+promises =
+  $${metabase-promise:name}
+
+[metabase-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url= $${metabase-instance:url}/api/session/properties
+
+[metabase-keystore]
+recipe = plone.recipe.command
+command =
+  ${java-re-8-output:keytool} \
+    -genkeypair \
+    -alias "metabase" \
+    -keyalg RSA \
+    -keypass "$${:password}" \
+    -dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \
+    -keystore "$${:file}" \
+    -storepass "$${:password}"
+file = $${directory:etc}/.metabase_keystore
+password = insecure
+
+[postgresql]
+recipe = slapos.cookbook:postgres
+bin = ${postgresql10:location}/bin/
+services = $${directory:service}
+dbname  = metabase_db
+superuser = metabase-psql
+password = insecure
+pgdata-directory = $${directory:srv}/postgresql
+
+ipv4 = $${instance-parameter:ipv4-random}
+# disable listening on ipv6
+ipv6 =
+port = 5432
+
+promises = $${postgresql-promise:name}
+
+
+[postgresql-psql]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+command-line =
+    $${postgresql:bin}/psql
+        -h $${postgresql:pgdata-directory}
+        -U $${postgresql:superuser}
+        -d $${postgresql:dbname}
+
+[postgresql-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = promise-postgresql.py
+config-command = $${postgresql-psql:wrapper-path} -c '\q'
+
+[postgresql-backup-crontab-entry]
+recipe = slapos.cookbook:cron.d
+name = $${:_buildout_section_name_}
+cron-entries = $${cron:cron-entries}
+time = daily
+command = $${postgresql-backup:wrapper-path}
+
+[postgresql-backup]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+# XXX there's a recipe for backup in slapos cookbook, but it does not create
+# the backup file in an atomic way, which is not acceptable here, because we
+# don't want to risk pulling a partial file. To prevent this, we create a
+# temp file and move it when finished.
+command-line =
+    sh -c "$${postgresql:bin}/pg_dump \
+      -h $${postgresql:pgdata-directory} \
+      -U $${postgresql:superuser} \
+      --format=custom \
+      -f $${:backup-file}.tmp \
+      $${postgresql:dbname} \
+    && mv $${:backup-file}.tmp $${:backup-file}"
+backup-file = $${directory:srv-backup}/backup.pg_dump
+
+[postgresql-restore-backup]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+command-line =
+  sh -e -c "\
+    echo 'This will replace current database with latest backup. Hit Ctrl+C to cancel';
+    sleep 5;
+    $${postgresql:bin}/pg_restore \
+        --exit-on-error \
+        -h $${postgresql:pgdata-directory} \
+        -U $${postgresql:superuser} \
+        -d $${postgresql:dbname} \
+        $${postgresql-backup:backup-file}"
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron-output:crond}
+cron-entries = $${directory:etc-cron.d}
+crontabs = $${directory:var-crontabs}
+cronstamps = $${directory:var-cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:bin}/crond
+
+[cron-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${cron:binary}
+wrapper-path = $${directory:services}/crond
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/cron.log
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+etc-cron.d = $${:etc}/cron.d
+var = $${buildout:directory}/var
+var-crontabs = $${:var}/crontabs
+var-cronstamps = $${:var}/cronstamps
+var-cron-entries = $${:var}/cron-entries
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+service = $${:etc}/service
+srv-metabase = $${:srv}/metabase
+srv-backup = $${:srv}/backup
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = $${metabase-instance:url}
+backup-crontab = $${postgresql-backup-crontab-entry:name}
+restore-backup-script = $${postgresql-restore-backup:wrapper-path}

software/metabase/software.cfg

+[buildout]
+extends =
+    ../../component/defaults.cfg
+    ../../component/java/buildout.cfg
+    ../../component/postgresql/buildout.cfg
+    ../../component/dcron/buildout.cfg
+    ../../stack/slapos.cfg
+    buildout.hash.cfg
+    ../../stack/monitor/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance-profile
+
+[python]
+part = python3
+
+[metabase.jar]
+recipe = slapos.recipe.build:download
+url = https://downloads.metabase.com/v0.35.3/metabase.jar
+md5sum = 73c98cdf5cecde80463ef868e77d3b0e
+
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+
+[versions]
+slapos.recipe.template = 4.4

software/metabase/test/README.md

+Tests for Metabase software release

software/metabase/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.metabase'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' metabase",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.cookbook',
+        'slapos.libnetworkcache',
+        'supervisor',
+        'six',
+        'requests'
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/metabase/test/test.py

+##############################################################################
+# coding: utf-8
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import json
+from six.moves.urllib import parse
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, MetabaseTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestMetabaseSetup(MetabaseTestCase):
+  __partition_reference__ = 'S'  # postgresql use a socket in data dir
+  # instance can take time, /api/session/properties timeout at the beginning.
+  instance_max_retry = 30
+
+  def test_setup(self):
+    url = self.computer_partition.getConnectionParameterDict()['url']
+    resp = requests.get(parse.urljoin(url, '/setup'), verify=False)
+    self.assertTrue(resp.text)
+
+    # get a setup token as described in https://github.com/metabase/metabase/issues/4240#issuecomment-290717451
+    properties = requests.get(
+        parse.urljoin(url, '/api/session/properties'), verify=False,
+        timeout=10).json()
+
+    email = "youlooknicetoday@email.com"
+    password = "password123456"
+
+    request_json = {
+        'token': properties['setup-token'],
+        'prefs': {
+            'allow_tracking': 'false',
+            'site_name': 'Org'
+        },
+        'user': {
+            'email': email,
+            'password': password,
+            'first_name': 'Johnny',
+            'last_name': 'Appleseed',
+            'site_name': 'Org',
+        },
+        'database': None
+    }
+    resp = requests.post(
+        parse.urljoin(url, '/api/setup'),
+        json=request_json,
+        verify=False,
+        timeout=5)
+    self.assertTrue(resp.ok)
+
+    resp = requests.post(
+        parse.urljoin(url, '/api/session'),
+        verify=False,
+        json={
+            "username": email,
+            "password": "wrong"
+        })
+    self.assertEqual(requests.codes.bad_request, resp.status_code)
+
+    session = requests.post(
+        parse.urljoin(url, '/api/session'),
+        verify=False,
+        json={
+            "username": email,
+            "password": password
+        }).json()
+    self.assertTrue(session.get('id'))

software/nextcloud/software.cfg

@@ -4,11 +4,6 @@ extends =
   ../../component/redis/buildout.cfg
   ../../stack/lamp/buildout.cfg
 
-[gcc]
-# For old version of glibmm.
-part = gcc-5.5
-max_version = 6
-
 [nc-download-base]
 recipe = hexagonit.recipe.download
 ignore-existing = true

software/slapos-sr-testing/software.cfg

@@ -98,6 +98,11 @@ setup = ${slapos-repository:location}/software/seleniumserver/test/
 egg = slapos.test.slaprunner
 setup = ${slapos-repository:location}/software/slaprunner/test/
 
+[slapos.test.metabase-setup]
+<= setup-develop-egg
+egg = slapos.test.metabase
+setup = ${slapos-repository:location}/software/metabase/test/
+
 [slapos.test.helloworld-setup]
 <= setup-develop-egg
 egg = slapos.test.helloworld
@@ -187,6 +192,7 @@ eggs =
   ${slapos.test.theia-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
   ${slapos.test.dream-setup:egg}
+  ${slapos.test.metabase-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -229,6 +235,7 @@ context =
 tests =
   ${slapos.test.kvm-setup:setup}
   ${slapos.test.slaprunner-setup:setup}
+  ${slapos.test.metabase-setup:setup}
   ${:extra}
 extra =
   ${slapos.cookbook-setup:setup}

stack/cloudooo.cfg

@@ -72,14 +72,15 @@ recipe = plone.recipe.command
 command = true
 
 [versions]
-Paste = 2.0.2
-PasteScript = 2.0.2
-WSGIUtils = 0.7
-python-magic = 0.4.6
+Paste = 3.4.0
+PasteScript = 3.2.0
+WSGIUtils = 0.7.2
+WSGIserver = 1.3
+python-magic = 0.4.18
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 4.4
 
 # Required by:
-# PasteScript==2.0
-# cloudooo==1.2.5.dev0
-PasteDeploy = 1.5.2
+# PasteScript== 3.2.0
+# cloudooo==1.2.6.dev0
+PasteDeploy = 2.1.0

stack/erp5/buildout.cfg

@@ -84,10 +84,6 @@ parts +=
 # jupyter
   jupyter-notebook-initialized-scripts
 
-[gcc]
-# KumoFS is known not to build with gcc>6.
-# It may work with slightly more recent but not tested.
-part = gcc-5.5
 
 # override instance-jupyter-notebook not to render into default template.cfg
 [instance-jupyter-notebook]

stack/erp5/buildout.hash.cfg

@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = e19aaec1878f40bf4ddb4c45a4470b21
+md5sum = f0f3b18f9963b137e366752886591fc3
 
 [monitor-template-dummy]
 filename = dummy.cfg

stack/erp5/instance.cfg.in

@@ -102,7 +102,7 @@ link-binary = {{ dumps(zope_link_binary) }}
 fonts = {{ dumps(zope_fonts) }}
 fontconfig-includes = {{ dumps(zope_fontconfig_includes) }}
 template-fonts-conf = {{ dumps(template_fonts_conf) }}
-userhosts = {{ userhosts_location }}
+userhosts = {{ userhosts_location }}/bin/userhosts
 site-zcml = {{ site_zcml }}
 extra-path-list = {{ dumps(extra_path_list) }}
 matplotlibrc = {{ matplotlibrc_location }}

stack/slapos.cfg

@@ -118,7 +118,7 @@ eggs =
 [versions]
 setuptools = 44.0.0
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos007
+zc.buildout = 2.7.1+slapos008
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -212,7 +212,7 @@ enum34 = 1.1.6
 
 # Required by:
 # slapos.toolbox==0.94
-erp5.util = 0.4.67
+erp5.util = 0.4.68
 
 # Required by:
 # slapos.toolbox==0.94