Update Release Candidate

component/firefox/buildout.cfg

@@ -71,11 +71,6 @@ part = ${firefox-60:location}
 wrapper-name = firefox-52
 part = ${firefox-52:location}
 
-[firefox-wrapper-51]
-<= firefox-wrapper
-wrapper-name = firefox-51
-part = ${firefox-51:location}
-
 [firefox-default-fonts-conf]
 recipe = slapos.recipe.template:jinja2
 template = ${template-fonts-conf:output}
@@ -124,13 +119,6 @@ version = 52.9.0esr
 i686-md5sum = 9aa18888b7812670208490609d75c9bc
 x86_64-md5sum = 9336d70f45070c743d08e5473b783a7a
 
-[firefox-51]
-<= firefox-download
-version = 51.0.1
-i686-md5sum = 9a5b67e9d759a1e4df004294a24b2b43
-x86_64-md5sum = bd93f2652d1d90d59ae462439a93c85f
-
-
 [firefox-download]
 recipe = slapos.recipe.build
 slapos_promise =
@@ -207,36 +195,12 @@ version = 0.22.0
 i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
 x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
 
-[geckodriver-0.21.0]
-<= geckodriver-base
-version = 0.21.0
-i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
-x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
-
-[geckodriver-0.19.0]
-<= geckodriver-base
-version = 0.19.0
-i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
-x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
-
-[geckodriver-0.17.0]
-<= geckodriver-base
-version = 0.17.0
-i686-md5sum = 79b1a158f96d29942a111c0905f1c807
-x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
-
 [geckodriver-0.16.1]
 <= geckodriver-base
 version = 0.16.1
 i686-md5sum = not not on github
 x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
 
-[geckodriver-0.14.0]
-<= geckodriver-base
-version = 0.14.0
-i686-md5sum = b5836f5a944fe9f3ed1a67c7b342c6a7
-x86_64-md5sum = 4a185d3179862a35104603b9274452e7
-
 
 [geckodriver-base]
 # Installs geckodriver ${version}

component/headless-chromium/buildout.cfg

@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # called "src".
 name = src
 
-# There's nothing special about version 92.0.4515.107. It just happened
-# to be the current Chromium stable version at the time of writing.
-version = 92.0.4515.107
+# 96.0.4664.129 version is the latest stable version in December 2021.
+# Note that we need a version compiling without python2
+version = 96.0.4664.129
 
 
 [headless-chromium]

component/libnsl/buildout.cfg

+[buildout]
+extends =
+  ../xz-utils/buildout.cfg
+  ../libtirpc/buildout.cfg
+
+[libnsl]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/thkukuk/libnsl/releases/download/v1.3.0/libnsl-1.3.0.tar.xz
+md5sum = 9214f674bd0c2bcfdd6c1da0cadb061f
+make-options = 
+  CFLAGS=-I${libtirpc:location}/include/tirpc
+  LDFLAGS='-L${libtirpc:location}/lib -Wl,-rpath=${libtirpc:location}/lib -ltirpc'
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/libtirpc/buildout.cfg

+[buildout]
+parts = libtirpc
+
+[libtirpc]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://downloads.sourceforge.net/libtirpc/libtirpc-1.3.2.tar.bz2
+md5sum = cf4ca51f3fc401bea61c702c69171ab0
+configure-options = --disable-gssapi

component/nginx/buildout.cfg

@@ -61,8 +61,8 @@ mime = ${nginx:location}/conf/mime.types
 [nginx-push-stream-module]
 recipe = slapos.recipe.build:download-unpacked
 shared = true
-url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
-md5sum = d9cba621b8739e13bdb5e02b9425f205
+url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.5.5.tar.gz
+md5sum = 9bb5237a93c957130aaf7033ed3c0989
 
 [nginx-push-stream]
 <= nginx-common

component/p7zip/buildout.cfg

 [buildout]
+extends =
+  ../patch/buildout.cfg
 parts =
   p7zip
 
 [p7zip]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
-version = 9.38.1
-md5sum = 6cba8402ccab2370d3b70c5e28b3d651
+url = https://downloads.sourceforge.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
+version = 16.02
+md5sum = a0128d661cfe7cc8c121e73519c54fbf
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patch-base = https://sources.debian.org/data/main/p/p7zip/16.02%2Bdfsg-8/debian/patches/
+patches =
+  ${:patch-base}/12-CVE-2016-9296.patch#f0f5dc36d97d9a71a6da3adba717fbf2
+  ${:patch-base}/13-CVE-2017-17969.patch#8399bdc913cf8cf8194ecdb5487f48d9
+  ${:patch-base}/14-Fix-g%2B%2B-warning.patch#9db8fbfbc8d624669c095240621460bd
+  ${:patch-base}/15-Fix-FTBFS-gcc10.patch#50ebfd79c715d8b5d649e57a1da2ce34
 configure-command = rm -r DOC
-make-binary =
-make-options =
-make-targets = make -j1 7z install
-  DEST_HOME=@@LOCATION@@
+make-targets = 7z
+post-install = make install DEST_HOME=%(location)s

component/postfix/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../cyrus-sasl/buildout.cfg
+  ../libnsl/buildout.cfg
   ../m4/buildout.cfg
 
 [postfix]
@@ -20,7 +21,9 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
 configure-command = make
-configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
+configure-options = makefiles CCARGS=${:configure-options-CCARGS} AUXLIBS=${:configure-options-AUXLIBS}
+configure-options-CCARGS = '-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl -I${libnsl:location}/include'
+configure-options-AUXLIBS = '-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -L${libtirpc:location}/lib -L${libnsl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib  -Wl,-rpath=${libnsl:location}/lib' 
 make-targets = non-interactive-package install_root=${:location}
 environment =
   PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s

software/erp5/test/test/__init__.py

@@ -49,17 +49,20 @@ def setUpModule():
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """
-  def getRootPartitionConnectionParameterDict(self):
+  @classmethod
+  def getRootPartitionConnectionParameterDict(cls):
     """Return the output paramters from the root partition"""
     return json.loads(
-        self.computer_partition.getConnectionParameterDict()['_'])
+        cls.computer_partition.getConnectionParameterDict()['_'])
 
-  def getComputerPartition(self, partition_reference):
-    for computer_partition in self.slap.computer.getComputerPartitionList():
+  @classmethod
+  def getComputerPartition(cls, partition_reference):
+    for computer_partition in cls.slap.computer.getComputerPartitionList():
       if partition_reference == computer_partition.getInstanceParameter(
           'instance_title'):
         return computer_partition
 
-  def getComputerPartitionPath(self, partition_reference):
-    partition_id = self.getComputerPartition(partition_reference).getId()
-    return os.path.join(self.slap._instance_root, partition_id)
+  @classmethod
+  def getComputerPartitionPath(cls, partition_reference):
+    partition_id = cls.getComputerPartition(partition_reference).getId()
+    return os.path.join(cls.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -26,21 +26,30 @@
 ##############################################################################
 
 from __future__ import absolute_import
-import os
-import json
+
+import contextlib
 import glob
-import six.moves.urllib.parse
+import json
+import os
+import shutil
 import socket
+import ssl
+import subprocess
+import sys
+import tempfile
 import time
+import unittest
 
 import psutil
 import requests
-
-from . import ERP5InstanceTestCase
-from . import setUpModule
 import six
-from six.moves import map
-from six.moves import range
+import six.moves.urllib.parse
+import six.moves.xmlrpc_client
+import urllib3
+from slapos.testing.utils import CrontabMixin
+
+from . import ERP5InstanceTestCase, setUpModule
+
 setUpModule # pyflakes
 
 
@@ -68,8 +77,8 @@ class TestPublishedURLIsReachableMixin(object):
       session.mount(
           base_url,
           requests.adapters.HTTPAdapter(
-              max_retries=requests.packages.urllib3.util.retry.Retry(
-                  total=60,
+              max_retries=urllib3.util.retry.Retry(
+                  total=20,
                   backoff_factor=.5,
                   status_forcelist=(404, 500, 503))))
 
@@ -389,3 +398,443 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
         }, {
           "cache-size": None,
         })
+
+
+class TestWatchActivities(ERP5InstanceTestCase):
+  """Tests for bin/watch_activities scripts in zope partitions.
+  """
+  __partition_reference__ = 'wa'
+
+  def test(self):
+    # "watch_activites" scripts use watch command. We'll fake a watch command
+    # that executes the actual command only once to check the output.
+    tmpdir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, tmpdir)
+    with open(os.path.join(tmpdir, 'watch'), 'w') as f:
+      f.write("""#!/bin/sh
+
+      if [ "$1" != "-n" ] || [ "$2" != "5" ]
+      then
+        echo unexpected arguments: "$1" "$2"
+        exit 1
+      fi
+      shift
+      shift
+
+      exec bash -c "$@"
+      """)
+      os.fchmod(f.fileno(), 0o700)
+    try:
+      output = subprocess.check_output(
+          [
+              os.path.join(
+                  self.getComputerPartitionPath('zope-1'),
+                  'bin',
+                  'watch_activities',
+              )
+          ],
+          env=dict(os.environ,
+                   PATH=os.pathsep.join([tmpdir, os.environ['PATH']])),
+          stderr=subprocess.STDOUT,
+          universal_newlines=True,
+      )
+    except subprocess.CalledProcessError as e:
+      self.fail(e.output)
+    self.assertIn(' dict ', output)
+
+
+class ZopeTestMixin(CrontabMixin):
+  """Mixin class for zope features.
+  """
+  wsgi = NotImplemented # type: bool
+
+
+  __partition_reference__ = 'z'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            "zope-partition-dict": {
+                "default": {
+                    "longrequest-logger-interval": 1,
+                    "longrequest-logger-timeout": 1,
+                },
+            },
+            "wsgi": cls.wsgi,
+        })
+    }
+
+  @classmethod
+  def _setUpClass(cls):
+    super(ZopeTestMixin, cls)._setUpClass()
+
+    param_dict = cls.getRootPartitionConnectionParameterDict()
+    # rebuild an url with user and password
+    parsed = six.moves.urllib.parse.urlparse(param_dict['family-default'])
+    cls.zope_base_url = parsed._replace(
+        netloc='{}:{}@{}:{}'.format(
+            param_dict['inituser-login'],
+            param_dict['inituser-password'],
+            parsed.hostname,
+            parsed.port,
+        ),
+        path=param_dict['site-id'] + '/',
+    ).geturl()
+
+    cls.zope_deadlock_debugger_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        '/manage_debug_threads?{deadlock-debugger-password}'.format(
+            **param_dict),
+    )
+
+
+    @contextlib.contextmanager
+    def getXMLRPCClient():
+      # don't verify certificate
+      ssl_context = ssl.create_default_context()
+      ssl_context.check_hostname = False
+      ssl_context.verify_mode = ssl.CERT_NONE
+      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+          cls.zope_base_url,
+          context=ssl_context,
+      )
+      # BBB use as a context manager only on python3
+      if sys.version_info < (3, ):
+        yield erp5_xmlrpc_client
+      else:
+        with erp5_xmlrpc_client:
+          yield erp5_xmlrpc_client
+
+    with getXMLRPCClient() as erp5_xmlrpc_client:
+      # wait for ERP5 to be ready (TODO: this should probably be a promise)
+      for _ in range(120):
+        time.sleep(1)
+        try:
+          erp5_xmlrpc_client.getTitle()
+        except (six.moves.xmlrpc_client.ProtocolError,
+                six.moves.xmlrpc_client.Fault):
+          pass
+        else:
+          break
+
+    def addPythonScript(script_id, params, body):
+      with getXMLRPCClient() as erp5_xmlrpc_client:
+        custom = erp5_xmlrpc_client.portal_skins.custom
+        try:
+          custom.manage_addProduct.PythonScripts.manage_addPythonScript(
+              script_id)
+        except six.moves.xmlrpc_client.ProtocolError as e:
+          if e.errcode != 302:
+            raise
+        getattr(custom, script_id).ZPythonScriptHTML_editAction(
+            '',
+            '',
+            params,
+            body,
+        )
+
+    # a python script to verify activity processing
+    addPythonScript(
+        script_id='ERP5Site_verifyActivityProcessing',
+        params='mode',
+        body='''if 1:
+          import json
+          portal = context.getPortalObject()
+          if mode == "count":
+            return json.dumps(dict(count=len(portal.portal_activities.getMessageList())))
+          if mode == "activate":
+            for _ in range(10):
+              portal.portal_templates.activate(activity="SQLQueue").getTitle()
+            return "activated"
+          raise ValueError("Unknown mode: %s" % mode)
+        ''',
+    )
+    cls.zope_verify_activity_processing_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_verifyActivityProcessing',
+    )
+    # a python script logging to event log
+    addPythonScript(
+        script_id='ERP5Site_logMessage',
+        params='name',
+        body='''if 1:
+            from erp5.component.module.Log import log
+            return log("hello %s" % name)
+        ''',
+    )
+    cls.zope_log_message_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_logMessage',
+    )
+    # a python script issuing a long request
+    addPythonScript(
+        script_id='ERP5Site_executeLongRequest',
+        params='',
+        body='''if 1:
+          import time
+          for _ in range(5):
+            time.sleep(1)
+          return "done"    
+        ''',
+    )
+    cls.zope_long_request_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_executeLongRequest',
+    )
+
+  def setUp(self):
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+  def tearDown(self):
+    # reset logrotate status
+    logrotate_status = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'logrotate.status',
+    )
+    if os.path.exists(logrotate_status):
+      os.unlink(logrotate_status)
+    for logfile in glob.glob(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'srv',
+            'backup',
+            'logrotate',
+            '*',
+        )):
+      os.unlink(logfile)
+    for logfile in glob.glob(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'srv',
+            'monitor',
+            'private',
+            'documents',
+            '*',
+        )):
+      os.unlink(logfile)
+
+  def _getCrontabCommand(self, crontab_name):
+    # type: (str) -> str
+    """Read a crontab and return the command that is executed.
+
+    overloaded to use crontab from zope partition
+    """
+    with open(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'etc',
+            'cron.d',
+            crontab_name,
+        )) as f:
+      crontab_spec, = f.readlines()
+    self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
+    return crontab_spec.split(None, 5)[-1]
+
+  def test_event_log_rotation(self):
+    requests.get(
+        self.zope_log_message_url,
+        params={
+            "name": "world"
+        },
+        verify=False,
+    ).raise_for_status()
+
+    zope_event_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-event.log',
+    )
+    with open(zope_event_log_path) as f:
+      self.assertIn('hello world', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'zope-0-event.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('hello world', f.read())
+
+    requests.get(
+        self.zope_log_message_url,
+        params={
+            "name": "le monde"
+        },
+        verify=False,
+    ).raise_for_status()
+    with open(zope_event_log_path) as f:
+      self.assertNotIn('hello world', f.read())
+    with open(zope_event_log_path) as f:
+      self.assertIn('hello le monde', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_access_log_rotation(self):
+    requests.get(
+        self.zope_base_url,
+        verify=False,
+        headers={
+            'User-Agent': 'before rotation'
+        },
+    ).raise_for_status()
+
+    zope_access_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-Z2.log',
+    )
+    with open(zope_access_log_path) as f:
+      self.assertIn('before rotation', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'zope-0-Z2.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('before rotation', f.read())
+
+    requests.get(
+        self.zope_base_url,
+        verify=False,
+        headers={
+            'User-Agent': 'after rotation'
+        },
+    ).raise_for_status()
+    with open(zope_access_log_path) as f:
+      self.assertNotIn('before rotation', f.read())
+    with open(zope_access_log_path) as f:
+      self.assertIn('after rotation', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_long_request_log_rotation(self):
+    requests.get(self.zope_long_request_url,
+                 verify=False,
+                 params={
+                     'when': 'before rotation'
+                 }).raise_for_status()
+
+    zope_long_request_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'longrequest_logger_zope-0.log',
+    )
+    with open(zope_long_request_log_path) as f:
+      self.assertIn('before rotation', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'longrequest_logger_zope-0.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('before rotation', f.read())
+
+    requests.get(
+        self.zope_long_request_url,
+        verify=False,
+        params={
+            'when': 'after rotation'
+        },
+    ).raise_for_status()
+    with open(zope_long_request_log_path) as f:
+      self.assertNotIn('before rotation', f.read())
+    with open(zope_long_request_log_path) as f:
+      self.assertIn('after rotation', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_basic_authentication_user_in_access_log(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    requests.get(self.zope_base_url,
+                 verify=False,
+                 auth=requests.auth.HTTPBasicAuth(
+                     param_dict['inituser-login'],
+                     param_dict['inituser-password'],
+                 )).raise_for_status()
+
+    zope_access_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-Z2.log',
+    )
+    with open(zope_access_log_path) as f:
+      self.assertIn(param_dict['inituser-login'], f.read())
+
+  def test_deadlock_debugger(self):
+    dump_response = requests.get(
+        self.zope_deadlock_debugger_url,
+        verify=False,
+    )
+    dump_response.raise_for_status()
+    self.assertIn('Thread ', dump_response.text)
+
+  def test_activity_processing(self):
+    requests.get(
+        self.zope_verify_activity_processing_url,
+        params={
+            'mode': 'activate'
+        },
+        verify=False,
+    ).raise_for_status()
+
+    for retry in range(60):
+      time.sleep(10)
+      count = requests.get(
+          self.zope_verify_activity_processing_url,
+          params={
+              'mode': 'count',
+              'retry': retry,
+          },
+          verify=False,
+      ).json()['count']
+      if not count:
+        break
+    else:
+      self.assertEqual(count, 0)
+
+
+class TestZopeMedusa(ZopeTestMixin, ERP5InstanceTestCase):
+  wsgi = False
+
+
+class TestZopeWSGI(ZopeTestMixin, ERP5InstanceTestCase):
+  wsgi = True
+
+  @unittest.expectedFailure
+  def test_long_request_log_rotation(self):
+    super(TestZopeWSGI, self).test_long_request_log_rotation(self)
+
+  @unittest.expectedFailure
+  def test_basic_authentication_user_in_access_log(self):
+    super(TestZopeWSGI, self).test_basic_authentication_user_in_access_log(self)

software/ors-amarisoft/buildout.hash.cfg

@@ -16,27 +16,27 @@
 
 [template]
 filename = instance.cfg
-md5sum = 96a76300b2f714b6c47157920fe79a53
+md5sum = b65347ea7c1936c2926ef11258cea602
 
 [template-lte-enb-epc]
 _update_hash_filename_ = instance-enb-epc.jinja2.cfg
-md5sum = 762d55291e75e8b61e35f9f28d29915a
+md5sum = 0883225f978255ecd343416759c1c5a4
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = 83c37b43d7b70584bf71aed9289ea13c
+md5sum = 4dc82af266a63b7045279fbf4f1e25e1
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
-md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
+md5sum = 5f8a4d85b26a7181fc8b16e588b88e1e
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
+md5sum = d04961535968b1b3e59d18663ec45eae
 
 [template-lte-epc]
 _update_hash_filename_ = instance-epc.jinja2.cfg
-md5sum = 0585b4f8fd42538595e6abb61d07fd93
+md5sum = 106d1079275d169a9c07854125aa4fa1
 
 [ue_db.jinja2.cfg]
 filename = config/ue_db.jinja2.cfg
@@ -61,3 +61,11 @@ md5sum = 518c71ce57204304b703b977c665a164
 [ims.jinja2.cfg]
 filename = config/ims.jinja2.cfg
 md5sum = e561ec26a70943c61557def1781cf65f
+
+[sdr-busy-promise]
+_update_hash_filename_ = promise/check_sdr_busy.py
+md5sum = 9e867282d7dd80c4255e87a82a47a19d
+
+[interface-up-promise]
+_update_hash_filename_ = promise/check_interface_up.py
+md5sum = 505efcbe04e717088924f2267b10c2b9

software/ors-amarisoft/instance-enb-epc.jinja2.cfg

@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-enb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -60,17 +60,26 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  {{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.cfg 2>> ${directory:log}/enb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-enb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -97,3 +106,10 @@ monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
 [monitor-instance-parameter]
 monitor-title = {{ slapparameter_dict['name'] | string }}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance-epc.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-mme-config
   lte-mme-service
-# Temporarily extend monitor-base until promises are added
+  tun-up-promise
   monitor-base
   publish-connection-information
 
@@ -62,7 +62,7 @@ context =
 [lte-ims-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
-command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
+command-line = {{ mme }}/lteims ${directory:etc}/ims.cfg
 wrapper-path = ${directory:service}/lte-ims
 mode = 0775
 pidfile = ${directory:run}/ims.pid
@@ -71,17 +71,29 @@ hash-files =
   {{ ue_db_path }}
 environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
 
+[lte-mme-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  rm -f ${directory:var}/lte_ue.db;
+  {{ mme }}/ltemme ${directory:etc}/mme.cfg >> ${directory:log}/mme-output.cfg 2>> ${directory:log}/mme-output.cfg
+
 ### MME
 [lte-mme-service]
 recipe = slapos.cookbook:wrapper
+# When the machine shutdowns abruptly, lte_ue is not cleaned up which causes
+# amarisoft ltemme to fail. TODO: find a cleaner way to handle this
 init = ${ltelogs:rendered} ${directory:log}/mme.log
-command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
+command-line = ${lte-mme-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-mme
 mode = 0775
 pidfile = ${directory:run}/mme.pid
 hash-files =
   ${lte-mme-config:rendered}
   {{ ue_db_path }}
+  ${lte-mme-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -121,15 +133,16 @@ context =
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish.serialised
-
-{% if slapparameter_dict.get("monitor-password", None) %}
 monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
-{% else %}
-monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
-{% endif %}
 
-{% if slapparameter_dict.get("monitor-password", None) %}
 [monitor-instance-parameter]
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-password'] }}
-{% endif %}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[tun-up-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ interface_up_promise }}
+output = ${directory:plugins}/check-tun-up.py
+config-ifname = ${slap-configuration:tun-name}

software/ors-amarisoft/instance-gnb-epc.jinja2.cfg

@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-gnb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -60,17 +60,26 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  {{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.cfg 2>> ${directory:log}/gnb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-gnb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -97,3 +106,10 @@ monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
 [monitor-instance-parameter]
 monitor-title = {{ slapparameter_dict['name'] | string }}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance.cfg

@@ -62,6 +62,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw enb_template   ${enb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_earfcn ${enb:default-dl-earfcn}
     raw default_lte_dl_freq ${enb:default-lte-dl-freq}
@@ -82,6 +83,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw gnb_template   ${gnb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
     raw default_nr_band ${enb:default-nr-band}
@@ -99,6 +101,7 @@ filename = instance-lte-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    raw interface_up_promise   ${interface-up-promise:target}
     raw mme   ${mme:destination}
     raw mme_template   ${mme.jinja2.cfg:target}
     raw ims_template   ${ims.jinja2.cfg:target}

software/ors-amarisoft/promise/check_interface_up.py

+import socket
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    ifname = self.getConfig('ifname')
+
+    f = open('/sys/class/net/%s/operstate' % ifname, 'r')
+    if f.read() == 'up\n':
+      self.logger.info("%s is up", ifname)
+    else:
+      self.logger.error("%s is down", ifname)
+    f.close()
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/promise/check_sdr_busy.py

+import os
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    sdr_dev = '/dev/sdr0'
+    try:
+      open(sdr_dev, 'w').close()
+      self.logger.error("eNB is not using %s", sdr_dev)
+    except IOError as e:
+      if e.errno == errno.EBUSY:
+        self.logger.info("eNB is using %s", sdr_dev)
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/software.cfg

@@ -49,6 +49,12 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 [template-lte-epc]
 <= download-base
 
+[sdr-busy-promise]
+<= download-base
+
+[interface-up-promise]
+<= download-base
+
 [amarisoft]
 recipe = slapos.recipe.build
 path = /opt/amarisoft/lte

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
+md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -46,7 +46,7 @@ md5sum = 1de449e8c0c4a85c5ce2b447785b7654
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
-md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
+md5sum = f928b9dc99f7f970caadfe7dd6f95d34
 
 [template-postfix]
 filename = instance-postfix.cfg.in

stack/erp5/instance-mariadb.cfg.in

 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
+{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
 {% set test_database_list = [] %}
 {% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
 {%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}

stack/erp5/mariadb_initial_setup.sql.in

@@ -4,16 +4,19 @@ USE mysql;
 SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
 {% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
-#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
+{% macro database(name, user, password, with_process_privilege) -%}
 CREATE DATABASE IF NOT EXISTS `{{ name }}`;
 {% if user -%}
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%   if with_process_privilege %}
+GRANT PROCESS ON *.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT PROCESS ON *.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%-   endif %}
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('with-process-privilege')) }}
 {% endfor -%}