Update Release Candidate

component/egg-patch/Zope/PythonScript-2.13.patch

---- Zope2-2.13.30/src/Shared/DC/Scripts/Signature.py	2022-04-25 08:05:09.312966168 +0000
-+++ Zope2-2.13.30/src/Shared/DC/Scripts/Signature.py	2022-04-25 08:06:20.120743425 +0000
-@@ -35,7 +35,7 @@ def _setFuncSignature(self, defaults=None, varnames=(), argcount=-1):
-         argcount = len(varnames)
-     # Generate a change only if we have to.
-     if self.func_defaults != defaults:
--        self.func_defaults = defaults
-+        self.func_defaults = self.__defaults__ = defaults
-     code = FuncCode(varnames, argcount)
-     if self.func_code != code:
--        self.func_code = code
-+        self.func_code = self.__code__ = code

component/readline/buildout.cfg

@@ -31,8 +31,7 @@ md5sum = e39331f32ad14009b9ff49cc10c5e751
 configure-options =
   --enable-multibyte
   --disable-static
-environment =
-  PATH=${patch:location}/bin:%(PATH)s
+patch-binary = ${patch:location}/bin/patch
 
 [readline]
 recipe = slapos.recipe.cmmi
@@ -47,3 +46,4 @@ configure-options =
 environment =
   CPPFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
+patch-binary = ${patch:location}/bin/patch

component/slapos/obs.cfg

@@ -29,8 +29,8 @@ download-cache = download-cache
 init +=
   buildout = self.buildout['buildout']
   assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
-    "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
-    buildout['directory'], buildout['destdir'] + buildout['rootdir'])
+    "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)" %
+    (buildout['directory'], buildout['destdir'] + buildout['rootdir']))
 
 [python3-common]
 configure-options +=

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.244'
+version = '1.0.246'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/librecipe/execute.py

@@ -67,7 +67,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     else:
       # With chained shebangs, several paths may be inserted at the beginning.
       n = len(args)
-      for i in xrange(1+len(running)-n):
+      for i in six.moves.xrange(1+len(running)-n):
         if args == running[i:n+i]:
           sys.exit("Already running with pid %s." % pid)
     with open(pidfile, 'w') as f:
@@ -91,16 +91,19 @@ def generic_exec(args, extra_environ=None, wait_list=None,
     uid = os.getuid()
     gid = os.getgid()
     unshare(CLONE_NEWUSER |CLONE_NEWNS)
-    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
-    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
-    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    with open('/proc/self/setgroups', 'w') as f:
+      f.write('deny')
+    with open('/proc/self/uid_map', 'w') as f:
+      f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map', 'w') as f:
+      f.write('%s %s 1' % (gid, gid))
     for size, path in private_tmpfs:
       try:
         os.mkdir(path)
       except OSError as e:
         if e.errno != errno.EEXIST:
           raise
-      mount('tmpfs', path, 'tmpfs', 0, 'size=' + size)
+      mount(b'tmpfs', path.encode(), b'tmpfs', 0, ('size=' + size).encode())
 
   if extra_environ:
     env = os.environ.copy()

slapos/test/recipe/test_re6stnet.py

@@ -52,13 +52,15 @@ class Re6stnetTest(unittest.TestCase):
       return makeRecipe(
             re6stnet.Recipe,
             options=self.options,
-            slap_connection={
+            buildout={
+              'slap-connection': {
                    'computer-id': 'comp-test',
                    'partition-id': 'slappart0',
                    'server-url': 'http://server.com',
                    'software-release-url': 'http://software.com',
                    'key-file': '/path/to/key',
                    'cert-file': '/path/to/cert'
+              }
             },
             name='re6stnet')
 

slapos/test/recipe/test_wrapper.py

+import errno
+import functools
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+
+from slapos.recipe import wrapper
+from slapos.test.utils import makeRecipe
+
+
+class WrapperTestCase(unittest.TestCase):
+  def getOptions(self):
+    raise NotImplementedError()
+
+  def setUp(self):
+    self.buildout_directory = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.buildout_directory)
+    self.getTempPath = functools.partial(os.path.join, self.buildout_directory)
+
+    self.wrapper_path = self.getTempPath('wrapper')
+
+    self.recipe = makeRecipe(
+      wrapper.Recipe,
+      options=self.getOptions(),
+      name="wrapper",
+      buildout={'buildout': {
+        'directory': self.buildout_directory,
+      }})
+
+  def terminate_process(self, process):
+    try:
+      process.terminate()
+    except OSError as e:
+      if e.errno != errno.ESRCH:
+        raise
+    process.wait()
+
+
+class TestSimpleCommandLineWrapper(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'echo hello world',
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      'hello world\n')
+
+
+class TestEscapeCommandLine(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': "echo esca $PE",
+      'wrapper-path': self.wrapper_path,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "esca $PE\n")
+
+
+class TestEnvironment(WrapperTestCase):
+  def getOptions(self):
+    return {
+      'command-line': 'sh -c "echo $FOO"',
+      'wrapper-path': self.wrapper_path,
+      'environment': 'FOO=bar',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed, universal_newlines=True, env={'FOO': 'foo'})
+    self.assertEqual(output, 'bar\n')
+
+
+class TestHashFiles(WrapperTestCase):
+  def getOptions(self):
+    hashed_file = self.getTempPath('hashed_file')
+    with open(hashed_file, 'w') as f:
+      f.write('hello world')
+    return {
+      'command-line': "cat " + hashed_file,
+      'wrapper-path': self.wrapper_path,
+      'hash-files': hashed_file
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    # 83af3240d992b2165abbd245a3e43368 is hashlib.md5(b'11\nhello world').hexdigest()
+    self.assertEqual(
+      installed, self.wrapper_path + '-83af3240d992b2165abbd245a3e43368')
+    self.assertEqual(
+      subprocess.check_output(installed, universal_newlines=True),
+      "hello world")
+
+
+class TestPidFile(WrapperTestCase):
+  def getOptions(self):
+    self.pidfile = self.getTempPath('hello.pid')
+    return {
+      'command-line': "/bin/sleep 10",
+      'wrapper-path': self.wrapper_path,
+      'pidfile': self.pidfile
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      if os.path.exists(self.pidfile):
+        break
+
+    with open(self.pidfile) as f:
+      pid = int(f.read())
+    self.assertEqual(process.pid, pid)
+
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      subprocess.check_output(
+        installed, stderr=subprocess.STDOUT, universal_newlines=True)
+    self.assertEqual(
+      ctx.exception.output, 'Already running with pid %s.\n' % pid)
+
+  def test_stale_pidfile_is_ignored(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+    with open(self.pidfile, 'w') as f:
+      f.write('1234')
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    for _ in range(20):
+      time.sleep(0.1)
+      with open(self.pidfile) as f:
+        pid = int(f.read())
+      if process.pid == pid:
+        break
+    else:
+      self.fail('pidfile not updated', process.stdout.read())
+
+
+class TestWaitForFiles(WrapperTestCase):
+  def getOptions(self):
+    self.waitfile = self.getTempPath('wait')
+    return {
+      'command-line': "/bin/echo done",
+      'wrapper-path': self.wrapper_path,
+      'wait-for-files': self.waitfile,
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    process = subprocess.Popen(
+      installed,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      universal_newlines=True,
+    )
+    self.addCleanup(self.terminate_process, process)
+    if process.poll():
+      self.fail(process.stdout.read())
+
+    # nothing happens when file is not there
+    for _ in range(10):
+      time.sleep(0.1)
+      if process.poll():
+        self.fail(process.stdout.read())
+
+    open(self.waitfile, 'w').close()
+    for _ in range(20):
+      time.sleep(0.1)
+      if process.poll() is not None:
+        self.assertEqual(process.stdout.read(), 'done\n')
+        self.assertEqual(process.returncode, 0)
+        break
+    else:
+      self.fail('process did not start after file was created')
+
+
+class TestPrivateTmpFS(WrapperTestCase):
+  def getOptions(self):
+    self.tmpdir = self.getTempPath('tmpdir')
+    self.tmpfile = self.getTempPath('tmpdir', 'file')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({tmpfile!r}, 'w') as f:
+            f.write('ok')
+          with open({tmpfile!r}, 'r') as f:
+            print(f.read())
+        ''').format(sys_executable=sys.executable, tmpfile=self.tmpfile))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'private-tmpfs': '1000 ' + self.tmpdir
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+    )
+    self.assertEqual(output, 'ok\n')
+    self.assertFalse(os.path.exists(self.tmpfile))
+
+
+class TestReserveCPU(WrapperTestCase):
+  def getOptions(self):
+    self.slapos_cpu_exclusive = self.getTempPath('.slapos-cpu-exclusive')
+    self.program = self.getTempPath('program')
+    with open(self.program, 'w') as f:
+      f.write(
+        textwrap.dedent(
+          '''\
+          #!{sys_executable}
+          import os
+          with open({slapos_cpu_exclusive!r}, 'r') as f:
+            print('ok' if int(f.read()) == os.getpid() else 'error')
+        ''').format(
+            sys_executable=sys.executable,
+            slapos_cpu_exclusive=self.slapos_cpu_exclusive,
+          ))
+    os.chmod(self.program, 0o700)
+    return {
+      'command-line': self.program,
+      'wrapper-path': self.wrapper_path,
+      'reserve-cpu': 'true',
+    }
+
+  def test_install_and_execute(self):
+    installed = self.recipe.install()
+    self.assertEqual(installed, self.wrapper_path)
+
+    output = subprocess.check_output(
+      installed,
+      universal_newlines=True,
+      env={'HOME': self.buildout_directory})
+    self.assertEqual(output, 'ok\n')

slapos/test/utils.py

@@ -2,18 +2,18 @@
 """
 import os
 import sys
+import six
 
 
-def makeRecipe(recipe_class, options, name='test', slap_connection=None):
-  """Instanciate a recipe of `recipe_class` with `options` with a buildout
-  mapping containing a python and an empty `slapos-connection` mapping, unless
-  provided as `slap_connection`.
+def makeRecipe(recipe_class, options, name='test', buildout=None):
+  """Instantiate a recipe of `recipe_class` with `options` with a `buildout`
+  mapping containing by default a python and an empty slap-connection.
 
   This function expects the test suite to have set SLAPOS_TEST_EGGS_DIRECTORY
   and SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY environment variables, so that the
   test recipe does not need to install eggs again when using working set.
   """
-  buildout = {
+  _buildout = {
     'buildout': {
       'bin-directory': '',
       'find-links': '',
@@ -32,15 +32,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
        'software-release-url': '',
      }
   }
-  if slap_connection is not None:
-    buildout['slap-connection'] = slap_connection
 
-  buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
-  buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+  _buildout['buildout']['eggs-directory'] = os.environ['SLAPOS_TEST_EGGS_DIRECTORY']
+  _buildout['buildout']['develop-eggs-directory'] = os.environ['SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY']
+
+  if buildout:
+    for section, _options in six.iteritems(buildout):
+      _buildout.setdefault(section, {}).update(**_options)
 
   # Prevent test from accidentally writing to the buildout's eggs
-  buildout['buildout']['newest'] = False
-  buildout['buildout']['offline'] = True
+  _buildout['buildout']['newest'] = False
+  _buildout['buildout']['offline'] = True
 
-  return recipe_class(buildout=buildout, name=name, options=options)
+  return recipe_class(buildout=_buildout, name=name, options=options)
 

software/erp5/upgrade_test/test.py

@@ -25,35 +25,43 @@
 #
 ##############################################################################
 
+import contextlib
 import glob
 import json
 import os
+import ssl
+import sys
 import tempfile
 import time
 
 import requests
-import urlparse
+import six.moves.urllib as urllib
+import six.moves.xmlrpc_client
+import urllib3
 
-from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.testcase import installSoftwareUrlList
-from slapos.testing.testcase import SlapOSNodeCommandError
 from slapos.grid.utils import md5digest
+from slapos.testing.testcase import (
+  SlapOSNodeCommandError,
+  installSoftwareUrlList,
+  makeModuleSetUpAndTestCaseClass,
+)
 
 old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
+  os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
 
 _, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    old_software_release_url,
-    software_id="upgrade_erp5",
-    skip_software_check=True,
+  old_software_release_url,
+  software_id="upgrade_erp5",
+  skip_software_check=True,
 )
 
+
 def setUpModule():
   installSoftwareUrlList(
-      SlapOSInstanceTestCase,
-      [old_software_release_url, new_software_release_url],
-      debug=SlapOSInstanceTestCase._debug,
+    SlapOSInstanceTestCase,
+    [old_software_release_url, new_software_release_url],
+    debug=SlapOSInstanceTestCase._debug,
   )
 
 
@@ -87,7 +95,7 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
 
     # wait for slapos node instance
     snapshot_name = "{}.{}.setUpClass new instance".format(
-        cls.__module__, cls.__name__)
+      cls.__module__, cls.__name__)
     try:
       if cls._debug and cls.instance_max_retry:
         try:
@@ -95,8 +103,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
         except SlapOSNodeCommandError:
           cls.slap.waitForInstance(debug=True)
       else:
-        cls.slap.waitForInstance(max_retry=cls.instance_max_retry,
-                                 debug=cls._debug)
+        cls.slap.waitForInstance(
+          max_retry=cls.instance_max_retry, debug=cls._debug)
       cls.logger.debug("instance on new software done")
     except BaseException:
       cls.logger.exception("Error during instance on new software")
@@ -113,20 +121,149 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
 class TestERP5Upgrade(ERP5UpgradeTestCase):
   @classmethod
   def setUpOldInstance(cls):
-    cls._default_instance_old_parameter_dict = json.loads(
-        cls.computer_partition.getConnectionParameterDict()['_'])
+    cls._default_instance_old_parameter_dict = param_dict = json.loads(
+      cls.computer_partition.getConnectionParameterDict()['_'])
+
+    # use a session to retry on failures, when ERP5 is not ready.
+    # (see also TestPublishedURLIsReachableMixin)
+    cls.session = requests.Session()
+    cls.session.mount(
+      param_dict['family-default-v6'],
+      requests.adapters.HTTPAdapter(
+        max_retries=urllib3.util.retry.Retry(
+          total=20,
+          backoff_factor=.1,
+          status_forcelist=(404, 500, 503),
+        )))
+
+    # rebuild an url with user and password
+    parsed = urllib.parse.urlparse(param_dict['family-default'])
+    cls.authenticated_zope_base_url = parsed._replace(
+      netloc='{}:{}@{}:{}'.format(
+        param_dict['inituser-login'],
+        param_dict['inituser-password'],
+        parsed.hostname,
+        parsed.port,
+      ),
+      path=param_dict['site-id'] + '/',
+    ).geturl()
+
+    cls.zope_base_url = '{family_default_v6}/{site_id}'.format(
+      family_default_v6=param_dict['family-default-v6'],
+      site_id=param_dict['site-id'],
+    )
+
+    # wait for old site creation
+    cls.session.get(
+      '{zope_base_url}/person_module'.format(zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    # Create scripts to create test data and search catalog for test data.
+    @contextlib.contextmanager
+    def getXMLRPCClient():
+      # don't verify certificate
+      ssl_context = ssl.create_default_context()
+      ssl_context.check_hostname = False
+      ssl_context.verify_mode = ssl.CERT_NONE
+      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+        cls.authenticated_zope_base_url,
+        context=ssl_context,
+      )
+      # BBB use as a context manager only on python3
+      if sys.version_info < (3, ):
+        yield erp5_xmlrpc_client
+      else:
+        with erp5_xmlrpc_client:
+          yield erp5_xmlrpc_client
+
+    def addPythonScript(script_id, params, body):
+      with getXMLRPCClient() as erp5_xmlrpc_client:
+        custom = erp5_xmlrpc_client.portal_skins.custom
+        try:
+          custom.manage_addProduct.PythonScripts.manage_addPythonScript(
+            script_id)
+        except six.moves.xmlrpc_client.ProtocolError as e:
+          if e.errcode != 302:
+            raise
+        getattr(custom, script_id).ZPythonScriptHTML_editAction(
+          '',
+          '',
+          params,
+          body,
+        )
+
+    # a python script to create a person with a name
+    addPythonScript(
+      script_id='ERP5Site_createTestPerson',
+      params='name',
+      body='''if 1:
+          portal = context.getPortalObject()
+          portal.person_module.newContent(
+            first_name=name,
+          )
+          return 'Done.'
+        ''',
+    )
+    # a python script to search for persons by name
+    addPythonScript(
+      script_id='ERP5Site_searchTestPerson',
+      params='name',
+      body='''if 1:
+          import json
+          portal = context.getPortalObject()
+          result = [brain.getObject().getTitle() for brain in portal.portal_catalog(
+              portal_type='Person',
+              title=name,)]
+          assert result # raise so that we retry until indexed
+          return json.dumps(result)
+        ''',
+    )
+
+    cls.session.post(
+      '{zope_base_url}/ERP5Site_createTestPerson'.format(
+        zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      data={
+        'name': 'before upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    assert cls.session.get(
+      '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+        zope_base_url=cls.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      params={
+        'name': 'before upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).json() == ['before upgrade']
 
   def test_published_url_is_same(self):
     default_instance_new_parameter_dict = json.loads(
-        self.computer_partition.getConnectionParameterDict()['_'])
+      self.computer_partition.getConnectionParameterDict()['_'])
     self.assertEqual(
-        default_instance_new_parameter_dict['family-default-v6'],
-        self._default_instance_old_parameter_dict['family-default-v6'],
+      default_instance_new_parameter_dict['family-default-v6'],
+      self._default_instance_old_parameter_dict['family-default-v6'],
     )
 
   def test_published_url_is_reachable(self):
     default_instance_new_parameter_dict = json.loads(
-        self.computer_partition.getConnectionParameterDict()['_'])
+      self.computer_partition.getConnectionParameterDict()['_'])
 
     # get certificate from caucase
     with tempfile.NamedTemporaryFile(
@@ -135,44 +272,84 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
         delete=False,
     ) as ca_cert:
       ca_cert.write(
-          requests.get(
-              urlparse.urljoin(
-                  default_instance_new_parameter_dict['caucase-http-url'],
-                  '/cas/crt/ca.crt.pem',
-              )).text)
+        requests.get(
+          urllib.parse.urljoin(
+            default_instance_new_parameter_dict['caucase-http-url'],
+            '/cas/crt/ca.crt.pem',
+          )).text)
       ca_cert.flush()
 
-      # use a session to retry on failures, when ERP5 is not ready.
-      # (see also TestPublishedURLIsReachableMixin)
-      session = requests.Session()
-      session.mount(
-        default_instance_new_parameter_dict['family-default-v6'],
-        requests.adapters.HTTPAdapter(
-            max_retries=requests.packages.urllib3.util.retry.Retry(
-                total=60,
-                backoff_factor=.5,
-                status_forcelist=(404, 500, 503))))
-
-      session.get(
-            '{}/{}/login_form'.format(
-                default_instance_new_parameter_dict['family-default-v6'],
-                default_instance_new_parameter_dict['site-id'],
-            ),
-            verify=False,
-            # TODO: we don't use caucase yet here.
-            # verify=ca_cert.name,
-        ).raise_for_status()
-
+      self.session.get(
+        '{}/{}/login_form'.format(
+          default_instance_new_parameter_dict['family-default-v6'],
+          default_instance_new_parameter_dict['site-id'],
+        ),
+        verify=False,
+        # TODO: we don't use caucase yet here.
+        # verify=ca_cert.name,
+      ).raise_for_status()
 
   def test_all_instances_use_new_software_release(self):
     self.assertEqual(
-        {
-            os.path.basename(os.readlink(sr))
-            for sr in glob.glob(
-                os.path.join(
-                    self.slap.instance_directory,
-                    '*',
-                    'software_release',
-                ))
+      {
+        os.path.basename(os.readlink(sr))
+        for sr in glob.glob(
+          os.path.join(
+            self.slap.instance_directory,
+            '*',
+            'software_release',
+          ))
+      },
+      {md5digest(self.getSoftwareURL())},
+    )
+
+  def test_catalog_available(self):
+    param_dict = json.loads(
+      self.computer_partition.getConnectionParameterDict()['_'])
+
+    # data created before upgrade is available
+    self.assertEqual(
+      self.session.get(
+        '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+          zope_base_url=self.zope_base_url),
+        auth=requests.auth.HTTPBasicAuth(
+          username=param_dict['inituser-login'],
+          password=param_dict['inituser-password'],
+        ),
+        params={
+          'name': 'before upgrade'
+        },
+        verify=False,
+        allow_redirects=False,
+      ).json(), ['before upgrade'])
+
+    # create data after upgrade
+    self.session.post(
+      '{zope_base_url}/ERP5Site_createTestPerson'.format(
+        zope_base_url=self.zope_base_url),
+      auth=requests.auth.HTTPBasicAuth(
+        username=param_dict['inituser-login'],
+        password=param_dict['inituser-password'],
+      ),
+      data={
+        'name': 'after upgrade'
+      },
+      verify=False,
+      allow_redirects=False,
+    ).raise_for_status()
+
+    # new data can also be found
+    self.assertEqual(
+      self.session.get(
+        '{zope_base_url}/ERP5Site_searchTestPerson'.format(
+          zope_base_url=self.zope_base_url),
+        auth=requests.auth.HTTPBasicAuth(
+          username=param_dict['inituser-login'],
+          password=param_dict['inituser-password'],
+        ),
+        params={
+          'name': 'after upgrade'
         },
-        {md5digest(self.getSoftwareURL())},)
+        verify=False,
+        allow_redirects=False,
+      ).json(), ['after upgrade'])

software/erp5testnode/testsuite/deploy-test/software.cfg

@@ -12,6 +12,7 @@ parts =
 recipe = zc.recipe.egg
 eggs =
   erp5.util
+  requests
 interpreter = ${:_buildout_section_name_}
 
 [python2.7-with-eggs]

software/kvm/buildout.hash.cfg

@@ -55,7 +55,7 @@ md5sum = a8cf453d20f01c707f02c4b4014580d8
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 395ee373ccda3382d257fde1ff4222b0
+md5sum = 6c100eec00de5e53f64b075dd69a9865
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in
@@ -79,11 +79,11 @@ md5sum = 438192aab9f11e40dc521b46a4854dcf
 
 [image-download-controller]
 filename = template/image-download-controller.py
-md5sum = 4d48b3da5bc611fc6533335b5953c840
+md5sum = 3cc10323fd4d2db4cfbac536b66eae7c
 
 [image-download-config-creator]
 filename = template/image-download-config-creator.py
-md5sum = 8fbe05c4175a7f31b6bffced9ad4e91d
+md5sum = 22ed19d9b8f7b983c97c52caa686bcd7
 
 [whitelist-firewall-download-controller]
 filename = template/whitelist-firewall-download-controller.py

software/kvm/instance-kvm-input-schema.json

@@ -204,7 +204,7 @@
       "title": "Size of additional disk to create for virtual machine, in Gigabytes",
       "description": "Specify the size of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
       "type": "integer",
-      "minimum": 10,
+      "minimum": 1,
       "default": 20
     },
     "external-disk-format": {

software/kvm/software-py3.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python3

software/kvm/software.cfg

@@ -40,6 +40,9 @@ parts = ${:common-parts}
 
 #XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
 
+[python]
+part = python3
+
 [python-with-eggs]
 recipe = zc.recipe.egg
 interpreter = ${:_buildout_section_name_}

software/kvm/template/image-download-config-creator.py

@@ -49,6 +49,10 @@ if __name__ == "__main__":
         image_list.append({
           'md5sum': md5sum,
           'url': url,
+          # Note: The destination here it's the waneted md5sum on purpose, as
+          #       it allows to assume, that correctly downloaded and hashed
+          #       image stored at this filename matches the md5sum, so it does
+          #       not have to be hashed on each download run.
           'destination': md5sum,
           'destination-tmp': md5sum + '_tmp',
           'image-number': '%03i' % (image_number,),

software/kvm/template/image-download-controller.py

@@ -68,13 +68,17 @@ if __name__ == "__main__":
     destination = os.path.join(
       config['destination-directory'], image['destination'])
     if os.path.exists(destination):
-      if md5Checksum(destination) == image['md5sum']:
-        print('INF: %s : already downloaded' % (image['url'],))
-        continue
-      else:
-        print('INF: %s : Removed, as expected checksum does not match %s' % (
-          image['url'], image['md5sum']))
-        os.remove(destination)
+      # Note: There is no need to recheck md5sum here
+      #       The image name is its md5sum, so if it exists, it means it has
+      #       correct md5sum
+      #       Calculating md5sum of big images takes more time than processing
+      #       of the partition and running promises and this leads to endless
+      #       loop of never ending promise failures
+      #       Of course, someone nasty can come to the partition and damage
+      #       this image, but it's another story, and shall not be fixed
+      #       during download phase.
+      print('INF: %s : already downloaded' % (image['url'],))
+      continue
     # key is str, as the dict is dumped to JSON which does not accept tuples
     md5sum_state_key = '%s#%s' % (image['url'], image['md5sum'])
     md5sum_state_amount = md5sum_state_dict.get(md5sum_state_key, 0)

software/kvm/template/template-kvm-run.in

@@ -130,7 +130,7 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
       # ID are writen in one line: data1 data3 data2 ...
       content = mf.readline()
       for id in content.split(' '):
-        if disk_storage_dict.has_key(id):
+        if id in disk_storage_dict:
           id_list.append(id)
         else:
           # Mean that this disk path has been removed (disk unmounted)

software/kvm/test/test.py

@@ -60,8 +60,7 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
 if has_kvm:
   setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-      os.path.join(os.path.dirname(__file__), '..',
-                   'software%s.cfg' % ("-py3" if six.PY3 else ""))))
+      os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
   # XXX Keep using slapos node instance --all, because of missing promises
   InstanceTestCase.slap._force_slapos_node_instance_all = True
 else:
@@ -99,13 +98,12 @@ bootstrap_machine_param_dict = {
     "ram-size": 4096,
     "cpu-count": 2,
     "disk-size": 50,
-    # Debian 10 image
     "virtual-hard-drive-url":
-    "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183d"
-    "f6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f0"
-    "30573b904d5ab",
-    "virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
-    "virtual-hard-drive-gzipped": False,
+    "http://shacache.org/shacache/a869d906fcd0af5091d5104451a2b86736485ae38e5"
+    "c4388657bb957c25593b98378ed125f593683e7fda7e0dd485a376a0ce29dcbaa8d60766"
+    "e1f67a7ef7b96",
+    "virtual-hard-drive-md5sum": "9ffd690a5fcb4fa56702f2b99183e493",
+    "virtual-hard-drive-gzipped": True,
     "hard-drive-url-check-certificate": False,
     "use-tap": True,
     "use-nat": True,
@@ -424,8 +422,6 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
 class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
   __partition_reference__ = 'adb'
   expected_partition_with_monitor_base_url_count = 1
-  # as few gigabytes are being downloaded, wait a bit longer
-  instance_max_retry = 100
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -433,8 +429,34 @@ class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
       bootstrap_common_param_dict, **bootstrap_machine_param_dict))}
 
   def test(self):
-    connection_parameter_dict = self.computer_partition\
-      .getConnectionParameterDict()
+    # START: mock .slapos-resource with tap.ipv4_addr
+    # needed for netconfig.sh
+    test_partition_slapos_resource_file = os.path.join(
+      self.computer_partition_root_path, '.slapos-resource')
+    path = os.path.realpath(os.curdir)
+    while path != '/':
+      root_slapos_resource_file = os.path.join(path, '.slapos-resource')
+      if os.path.exists(root_slapos_resource_file):
+        break
+      path = os.path.realpath(os.path.join(path, '..'))
+    else:
+      raise ValueError('No .slapos-resource found to base the mock on')
+    with open(root_slapos_resource_file) as fh:
+      root_slapos_resource = json.load(fh)
+    if root_slapos_resource['tap']['ipv4_addr'] == '':
+      root_slapos_resource['tap'].update({
+        "ipv4_addr": "10.0.0.2",
+        "ipv4_gateway": "10.0.0.1",
+        "ipv4_netmask": "255.255.0.0",
+        "ipv4_network": "10.0.0.0"
+      })
+    with open(test_partition_slapos_resource_file, 'w') as fh:
+      json.dump(root_slapos_resource, fh, indent=4)
+    self.slap.waitForInstance(max_retry=10)
+    # END: mock .slapos-resource with tap.ipv4_addr
+
+    cp = self.computer_partition
+    connection_parameter_dict = cp.getConnectionParameterDict()
 
     result = requests.get(connection_parameter_dict['url'], verify=False)
     self.assertEqual(
@@ -526,8 +548,6 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
 class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
   __partition_reference__ = 'akcb'
   expected_partition_with_monitor_base_url_count = 3
-  # as few gigabytes are being downloaded, wait a bit longer
-  instance_max_retry = 100
 
   @classmethod
   def getInstanceSoftwareType(cls):
@@ -539,12 +559,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
       "kvm-partition-dict": {
           "test-machine1": bootstrap_machine_param_dict,
           "test-machine2": dict(bootstrap_machine_param_dict, **{
-              # Debian 9 image
               "virtual-hard-drive-url":
-              "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
-              "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
-              "a508930046a78cffea6e0e29d03345333cc",
-              "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
+              "http://shacache.org/shacache/5bdc95ea3f8ca40ff4fb8d086776e393"
+              "87a68e91f76b1a5f883dfc33fa13cf1ee71c7d218a4e9401f56519a352791"
+              "272ada4a5c334b3ca38a32c0bcacb6838e2",
+              "virtual-hard-drive-md5sum": "deaf751a31dd6aec320d67c75c88c2e1",
               "virtual-hard-drive-gzipped": True,
           })
       }
@@ -608,13 +627,20 @@ class TestInstanceResilient(InstanceTestCase, KvmMixin):
       if k in connection_parameter_dict:
         present_key_list.append(k)
         connection_parameter_dict.pop(k)
+    self.assertIn('feed-url-kvm-1-pull', connection_parameter_dict)
+    feed_pull = connection_parameter_dict.pop('feed-url-kvm-1-pull')
+    self.assertRegexpMatches(
+      feed_pull,
+      'http://\\[%s\\]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-pull' % (
+        self._ipv6_address,))
+    feed_push = connection_parameter_dict.pop('feed-url-kvm-1-push')
+    self.assertRegexpMatches(
+      feed_push,
+      'http://\\[%s\\]:[0-9][0-9][0-9][0-9]/get/local-ir0-kvm-1-push' % (
+        self._ipv6_address,))
     self.assertEqual(
       connection_parameter_dict,
       {
-        'feed-url-kvm-1-pull': 'http://[%s]:8088/get/local-ir0-kvm-1-pull' % (
-          self._ipv6_address,),
-        'feed-url-kvm-1-push': 'http://[%s]:8088/get/local-ir0-kvm-1-push' % (
-          self._ipv6_address,),
         'ipv6': self._ipv6_address,
         'monitor-base-url': 'https://[%s]:8160' % (self._ipv6_address,),
         'monitor-user': 'admin',
@@ -851,11 +877,11 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
   @classmethod
   def setUpClass(cls):
     cls.startImageHttpServer()
-    super(InstanceTestCase, cls).setUpClass()
+    super(TestBootImageUrlList, cls).setUpClass()
 
   @classmethod
   def tearDownClass(cls):
-    super(InstanceTestCase, cls).tearDownClass()
+    super(TestBootImageUrlList, cls).tearDownClass()
     cls.stopImageHttpServer()
 
   def tearDown(self):
@@ -866,15 +892,16 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     # 2nd ...move instance to "default" state
     self.rerequestInstance({})
     self.slap.waitForInstance(max_retry=10)
-    super(InstanceTestCase, self).tearDown()
+    super(TestBootImageUrlList, self).tearDown()
 
-  def getRunningImageList(self, kvm_instance_partition,
+  def getRunningImageList(
+      self, kvm_instance_partition,
       _match_cdrom=re.compile('file=(.+),media=cdrom$').match,
       _sub_iso=re.compile(r'(/debian)(-[^-/]+)(-[^/]+-netinst\.iso)$').sub,
-      ):
+    ):
     with self.slap.instance_supervisor_rpc as instance_supervisor:
       kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
-                       if 'kvm-' in q['name'])['pid']
+                     if 'kvm-' in q['name'])['pid']
     sub_shared = re.compile(r'^%s/[^/]+/[0-9a-f]{32}/'
                             % re.escape(self.slap.shared_directory)).sub
     image_list = []
@@ -883,10 +910,12 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
       if m:
         path = m.group(1)
         image_list.append(
-          _sub_iso(r'\1-${ver}\3',
-          sub_shared(r'${shared}/',
-          path.replace(kvm_instance_partition, '${inst}')
-        )))
+          _sub_iso(
+            r'\1-${ver}\3',
+            sub_shared(
+              r'${shared}/',
+              path.replace(kvm_instance_partition, '${inst}')
+            )))
     return image_list
 
   def test(self):
@@ -1020,6 +1049,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
 @skipUnlessKvm
 class TestBootImageUrlListResilient(TestBootImageUrlList):
   kvm_instance_partition_reference = 'biul2'
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'kvm-resilient'
@@ -1130,6 +1160,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
 @skipUnlessKvm
 class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
   kvm_instance_partition_reference = 'bius2'
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'kvm-resilient'
@@ -1148,12 +1179,12 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
   config_file_name = 'boot-image-url-list.conf'
 
   def setUp(self):
-    super(InstanceTestCase, self).setUp()
+    super(TestBootImageUrlListKvmCluster, self).setUp()
     self.startImageHttpServer()
 
   def tearDown(self):
     self.stopImageHttpServer()
-    super(InstanceTestCase, self).tearDown()
+    super(TestBootImageUrlListKvmCluster, self).tearDown()
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -1245,6 +1276,7 @@ class TestNatRulesKvmCluster(InstanceTestCase):
   __partition_reference__ = 'nrkc'
 
   nat_rules = ["100", "200", "300"]
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'kvm-cluster'
@@ -1321,6 +1353,7 @@ class TestWhitelistFirewall(InstanceTestCase):
 @skipUnlessKvm
 class TestWhitelistFirewallRequest(TestWhitelistFirewall):
   whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
+
   @classmethod
   def getInstanceParameterDict(cls):
     return {
@@ -1448,7 +1481,7 @@ class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
   def tearDown(self):
     self.stopImageHttpServer()
     shutil.rmtree(self.working_directory)
-    super(InstanceTestCase, self).tearDown()
+    super(TestImageDownloadController, self).tearDown()
 
   def callImageDownloadController(self, *args):
     call_list = [sys.executable, self.image_download_controller] + list(args)
@@ -1636,7 +1669,7 @@ class TestParameterDefault(InstanceTestCase, KvmMixin):
   def _test(self, parameter_dict, expected):
     self.rerequestInstance(self.mangleParameterDict(parameter_dict))
     self.slap.waitForInstance(max_retry=10)
-    
+
     kvm_raw = glob.glob(os.path.join(
       self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
     self.assertEqual(len(kvm_raw), 1)
@@ -1685,6 +1718,7 @@ class TestParameterDefault(InstanceTestCase, KvmMixin):
 @skipUnlessKvm
 class TestParameterResilient(TestParameterDefault):
   __partition_reference__ = 'pr'
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'kvm-resilient'
@@ -1718,3 +1752,136 @@ class TestParameterCluster(TestParameterDefault):
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'kvm-cluster'
+
+
+@skipUnlessKvm
+class TestExternalDisk(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'ed'
+  kvm_instance_partition_reference = 'ed0'
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'external-disk-number': 2,
+      'external-disk-size': 1
+    }
+
+  @classmethod
+  def _prepareExternalStorageList(cls):
+    external_storage_path = os.path.join(cls.working_directory, 'STORAGE')
+    os.mkdir(external_storage_path)
+    # reuse .slapos-resource infomration of the containing partition
+    # it's similar to slapos/recipe/slapconfiguration.py
+    _resource_home = cls.slap.instance_directory
+    parent_slapos_resource = None
+    while not os.path.exists(os.path.join(_resource_home, '.slapos-resource')):
+      _resource_home = os.path.normpath(os.path.join(_resource_home, '..'))
+      if _resource_home == "/":
+        break
+    else:
+      with open(os.path.join(_resource_home, '.slapos-resource')) as fh:
+        parent_slapos_resource = json.load(fh)
+    assert parent_slapos_resource is not None
+
+    for partition in os.listdir(cls.slap.instance_directory):
+      if not partition.startswith(cls.__partition_reference__):
+        continue
+      partition_store_list = []
+      for number in range(10):
+        storage = os.path.join(external_storage_path, 'data%s' % (number,))
+        if not os.path.exists(storage):
+          os.mkdir(storage)
+        partition_store = os.path.join(storage, partition)
+        os.mkdir(partition_store)
+        partition_store_list.append(partition_store)
+      slapos_resource = parent_slapos_resource.copy()
+      slapos_resource['external_storage_list'] = partition_store_list
+      with open(
+        os.path.join(
+          cls.slap.instance_directory, partition, '.slapos-resource'),
+        'w') as fh:
+        json.dump(slapos_resource, fh, indent=2)
+    # above is not enough: the presence of parameter is required in slapos.cfg
+    slapos_config = []
+    with open(cls.slap._slapos_config) as fh:
+      for line in fh.readlines():
+        if line.strip() == '[slapos]':
+          slapos_config.append('[slapos]\n')
+          slapos_config.append(
+            'instance_storage_home = %s\n' % (external_storage_path,))
+        else:
+          slapos_config.append(line)
+    with open(cls.slap._slapos_config, 'w') as fh:
+      fh.write(''.join(slapos_config))
+
+  @classmethod
+  def _dropExternalStorageList(cls):
+    slapos_config = []
+    with open(cls.slap._slapos_config) as fh:
+      for line in fh.readlines():
+        if line.startswith("instance_storage_home ="):
+          continue
+        slapos_config.append(line)
+    with open(cls.slap._slapos_config, 'w') as fh:
+      fh.write(''.join(slapos_config))
+
+  @classmethod
+  def _setUpClass(cls):
+    super(TestExternalDisk, cls)._setUpClass()
+    cls.working_directory = tempfile.mkdtemp()
+    # setup the external_storage_list, to mimic part of slapformat
+    cls._prepareExternalStorageList()
+    # re-run the instance, as information has been updated
+    cls.waitForInstance()
+
+  @classmethod
+  def tearDownClass(cls):
+    cls._dropExternalStorageList()
+    super(TestExternalDisk, cls).tearDownClass()
+    shutil.rmtree(cls.working_directory)
+
+  def getRunningDriveList(self, kvm_instance_partition):
+    _match_drive = re.compile('file=(.+),if=virtio').match
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
+                     if 'kvm-' in q['name'])['pid']
+    dirve_list = []
+    for entry in psutil.Process(kvm_pid).cmdline():
+      m = _match_drive(entry)
+      if m:
+        path = m.group(1)
+        dirve_list.append(
+          path.replace(kvm_instance_partition, '${partition}')
+        )
+    return dirve_list
+
+  def test(self):
+    kvm_instance_partition = os.path.join(
+      self.slap.instance_directory, self.kvm_instance_partition_reference)
+    drive_list = self.getRunningDriveList(kvm_instance_partition)
+
+    # Note: Do to unknown set of drives it's impossible to directly check
+    #       drive paths, thus the count is important
+    self.assertEqual(
+      1 + 2,  # 1 the default drive, 2 additional ones
+      len(drive_list)
+    )
+
+    # restart the VM
+    self.requestDefaultInstance(state='stopped')
+    self.waitForInstance()
+    self.requestDefaultInstance(state='started')
+    self.waitForInstance()
+    restarted_drive_list = self.getRunningDriveList(kvm_instance_partition)
+    self.assertEqual(drive_list, restarted_drive_list)
+    # prove that even on resetting parameters, drives are still there
+    self.rerequestInstance({}, state='stopped')
+    self.waitForInstance()
+    self.rerequestInstance({})
+    self.waitForInstance()
+    dropped_drive_list = self.getRunningDriveList(kvm_instance_partition)
+    self.assertEqual(drive_list, dropped_drive_list)

software/matomo/software.cfg

@@ -34,6 +34,9 @@ md5sum = 8d592676bc2c0d51363ad7b2caf171fe
 [custom-application-deployment]
 path = ${template-matomo-instance:output}
 part-list = matomo-backup.sh matomo-backup-cron matomo-apache-httpd
+db-name = matomo
+db-user = matomo
+db-password = 12345678
 
 [template-matomo-instance]
 recipe = slapos.recipe.template:jinja2

software/metabase/software.cfg

@@ -22,8 +22,8 @@ part = python3
 
 [metabase.jar]
 recipe = slapos.recipe.build:download
-url = https://downloads.metabase.com/v0.41.4/metabase.jar
-md5sum = 9b81838e5c40302b552c66df5a767f8e
+url = https://downloads.metabase.com/v0.43.1/metabase.jar
+md5sum = 8033ba58825239e7dff29be8d4c885a7
 
 [instance-profile]
 recipe = slapos.recipe.template

software/nextcloud/software.cfg

@@ -12,7 +12,7 @@ url = ${:_profile_base_location_}/${:filename}
 url = https://download.nextcloud.com/server/releases/nextcloud-16.0.3.tar.bz2
 md5sum = d81902d2dec5d547779bec6336a438be
 
-[template-nextcloud-install.sh] 
+[template-nextcloud-install.sh]
 <= nc-download
 
 [template-apache-httpd]
@@ -26,7 +26,7 @@ recipe = slapos.recipe.template:jinja2
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/instance-nextcloud.cfg
 extensions = jinja2.ext.do
-context = 
+context =
           key gzip_location          gzip:location
           key python3_location       python3:location
           key news_updater_location  news-updater:location
@@ -41,6 +41,9 @@ context =
 [custom-application-deployment]
 path = ${template-nextcloud-instance:output}
 part-list = nextcloud-install.sh
+db-name = nextcloud
+db-user = nextcloud
+db-password = insecure
 
 [nc-download-unpacked]
 recipe = slapos.recipe.build:download-unpacked

software/slapos-sr-testing/software-py3.cfg

@@ -19,6 +19,7 @@ extra =
   helloworld ${slapos.test.helloworld-setup:setup}
   hugo ${slapos.test.hugo-setup:setup}
   jupyter ${slapos.test.jupyter-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
   matomo ${slapos.test.matomo-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
   nextcloud ${slapos.test.nextcloud-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -350,7 +350,6 @@ tests =
 # here, to check there's no promise issue when slapos node runs with Python 2.
   erp5 ${slapos.test.erp5-setup:setup}
   fluentd ${slapos.test.fluentd-setup:setup}
-  kvm ${slapos.test.kvm-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
 ###
   ${:extra}

stack/erp5/buildout.cfg

@@ -469,7 +469,6 @@ eggs = ${neoppod:eggs}
   pytz
   requests
   responses
-  threadframe
   urlnorm
   uuid
   xml_marshaller
@@ -524,9 +523,7 @@ eggs = ${neoppod:eggs}
   five.localsitemanager
 
 # Other products
-  Products.DCWorkflowGraph
   Products.MimetypesRegistry
-  Products.ExternalEditor
   Products.TIDStorage
   Products.LongRequestLogger
 
@@ -552,10 +549,8 @@ eggs = ${neoppod:eggs}
   ipykernel
 
 # Used by DiffTool
-  xmltodict
   deepdiff
   unidiff
-  jsonpickle
 
 # WSGI server
   zope.globalrequest
@@ -592,8 +587,6 @@ Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acqu
 Acquisition-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
-Zope2-patches = ${:_profile_base_location_}/../../component/egg-patch/Zope/PythonScript-2.13.patch#124c0d37394dd5020c6fd241ad75cc29
-Zope2-patch-options = -p1
 
 [eggs-all-scripts]
 recipe = zc.recipe.egg
@@ -634,7 +627,6 @@ pysvn = 1.9.15+SlapOSPatched001
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
 PyPDF2 = 1.26.0+SlapOSPatched001
-Zope2 = 2.13.30+SlapOSPatched001
 ## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
 pylint = 1.4.4+SlapOSPatched002
 # astroid 1.4.1 breaks testDynamicClassGeneration
@@ -668,9 +660,6 @@ zope.app.testing = 3.8.1
 APacheDEX = 1.8
 Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
-Products.DCWorkflowGraph = 0.4.1
-# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
-Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.6
 Products.LongRequestLogger = 2.1.0
 # Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
@@ -713,7 +702,6 @@ rsa = 3.4.2
 spyne = 2.12.14
 suds = 0.4
 facebook-sdk = 2.0.0
-threadframe = 0.2
 urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.1.0
@@ -732,8 +720,6 @@ zope.globalrequest = 1.5
 waitress = 1.4.4
 xlrd = 1.1.0
 
-# Re-add for as it is required to be there for uninstallation
-erp5.recipe.w3validator = 1.0.2
 Products.ZSQLMethods = 2.13.5
 fpconst = 0.7.2
 graphviz = 0.5.2
@@ -748,7 +734,6 @@ mpmath = 0.19
 openpyxl = 2.4.8
 sympy = 1.1.1
 jdcal = 1.3
-xmltodict = 0.11.0
 deepdiff = 3.3.0
 unidiff = 0.5.5
 jsonpickle = 0.9.6

stack/lamp/buildout.cfg

@@ -53,6 +53,10 @@ part = python3
 # See software/maarch/software.cfg for an example.
 path =
 part-list =
+# database information
+db-name = lamp
+db-user = lamp
+db-password = insecure
 
 #----------------
 #-- Instance-level buildout profiles.
@@ -100,6 +104,9 @@ context =
     key unixodbc_location unixodbc:location
     key openssl_location openssl:location
     key custom_application_template custom-application-deployment:path
+    key db_name custom-application-deployment:db-name
+    key db_user custom-application-deployment:db-user
+    key db_password custom-application-deployment:db-password
 
 [instance-apache-php]
 <= template-download-base

stack/lamp/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance]
 filename = instance.cfg.in
-md5sum = 29df0dc24386ecb97dc52c9fb59108c8
+md5sum = a5a630377bfb0421d6993c9c2c411a23
 
 [instance-apache-php]
 filename = instance-apache-php.cfg.in
@@ -22,7 +22,7 @@ md5sum = 0952ef9f6cb5e259ad5519d2975d2f37
 
 [instance-lamp]
 filename = instance-lamp.cfg.jinja2.in
-md5sum = e0e2e88b6deeb011b998b78e4e468555
+md5sum = b3d68a13d7a7ffcac774f51f02a68359
 
 [template-apache.conf]
 filename = apache.conf.in

stack/lamp/instance-lamp.cfg.jinja2.in

@@ -37,8 +37,8 @@ return =
 {% do publish_dict.__setitem__('backend-url', '${request-apache:connection-backend-url}') -%}
 {% do monitor_base_url_dict.__setitem__('apache', '${request-apache:connection-monitor-base-url}') -%}
 
+{% do mariadb_dict.__setitem__('database-list', [{'name': db_name, 'user': db_user, 'password': db_password}]) -%}
 
-{% do mariadb_dict.__setitem__('database-list', [{'name': 'nextcloud', 'user': 'nextcloud', 'password': 'insecure'}]) -%}
 {% do mariadb_dict.__setitem__('test-database-amount', 0) -%}
 {% do mariadb_dict.__setitem__('tcpv4-port', 2099) -%}
 {% do mariadb_dict.__setitem__('max-slowqueries-threshold', 1000) -%}
@@ -56,7 +56,7 @@ sla-computer_guid = {{ dumps(slapparameter_dict.get('mariadb-computer-guid', '')
 {% for key, value in mariadb_dict.items() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
-return = 
+return =
   database-list
   monitor-base-url
 
@@ -92,7 +92,7 @@ recipe = slapos.cookbook:publish-early
 
 [monitor-instance-parameter]
 monitor-httpd-port = 8060
-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} 
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
 username = admin
 password = ${publish-early:monitor-password}
 
@@ -120,7 +120,7 @@ parts =
 
 # Complete parts with sections
   {{ part_list | join('\n  ') }}
- 
+
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true

stack/lamp/instance.cfg.in

@@ -49,6 +49,9 @@ url = {{ template_lamp }}
 filename = template-lamp.cfg
 extra-context =
   section parameter_dict dynamic-template-lamp-parameters
+  raw db_name {{ db_name }}
+  raw db_user {{ db_user }}
+  raw db_password {{ db_password }}
 
 [dynamic-template-apache-php-parameters]
 application-location = {{ application_location }}

stack/resilient/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 44a3166048a81d0d76d69527b1934ef7
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = c4012ccc2c473ae5c7cad9dcac61e0f1
+md5sum = 2eea3b0227c3ae9e44cfc41df9930fa7
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/template-replicated.cfg.in

@@ -6,7 +6,7 @@
 {% set monitor_url_list = [] -%}
 # prepare sla-parameters
 {% if slapparameter_dict is defined -%}
-{%   for key in slapparameter_dict.keys() -%}
+{%   for key in list(slapparameter_dict.keys()) -%}
 {%     if key.startswith('-sla-') -%}
 {%       do sla_parameter_dict.__setitem__(key, slapparameter_dict.pop(key)) -%}
 {%     endif -%}
@@ -59,7 +59,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
@@ -111,7 +111,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
@@ -214,7 +214,7 @@ sla-mode = unique_by_network
 {%   for key in sla_parameter_dict.keys() -%}
 {%     if key.startswith(sla_key_main) -%}
 {%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
-{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%     elif key.startswith(sla_key_secondary) and key[sla_key_secondary_length:] not in sla_dict -%}
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}