software/erp5/test: Run tests with ZEO & NEO

Before this patch all ERP5 SlapOS Integration tests only run with ZEO
storage. We should also run them against NEO, because we are using
ERP5 with NEO in SlapOS.

In order to do so we implemented parameterized test classes for our ERP5
integration tests. Each test case can be configured via its __test_matrix__
attribute. A test matrix is a dict which maps the flavoured class name suffix to
a tuple of parameters. A parameter is a function which receives the
instance_parameter_dict and modifies it in place. You can use the
'matrix' helper function to construct a test matrix. If .__test_matrix__
is 'None' the test case is ignored.

/reviewed-by @kirr & @jerome
/reviewed-on nexedi/slapos!1306

software/erp5/test/test/__init__.py

 ##############################################################################
 #
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -25,8 +25,10 @@
 #
 ##############################################################################
 
+import itertools
 import json
 import os
+import sys
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -45,9 +47,138 @@ def setUpModule():
   setup_module_executed = True
 
 
-class ERP5InstanceTestCase(SlapOSInstanceTestCase):
+# Metaclass to parameterize our tests.
+# This is a rough adaption of the parameterized package:
+#   https://github.com/wolever/parameterized
+# Consult following note for rationale why we don't use parameterized:
+#   https://lab.nexedi.com/nexedi/slapos/merge_requests/1306
+class ERP5InstanceTestMeta(type):
+  """Adjust ERP5InstanceTestCase instances to be run in several flavours (e.g. NEO/ZEO)
+
+  Adjustements can be declared via setting the '__test_matrix__' attribute
+  of a test case.
+  A test matrix is a dict which maps the flavoured class name suffix to
+  a tuple of parameters.
+  A parameter is a function which receives the instance_parameter_dict
+  and modifies it in place (therefore no return value is needed).
+  You can use the 'matrix' helper function to construct a test matrix.
+  If .__test_matrix__ is 'None' the test case is ignored.
+  If the test case should be run without any adaptions, you can set
+  .__test_matrix__ to 'matrix((default,))'.
+  """
+
+  def __new__(cls, name, bases, attrs):
+    base_class = super().__new__(cls, name, bases, attrs)
+    if base_class._isParameterized():
+      cls._parameterize(base_class)
+    return base_class
+
+  # _isParameterized tells whether class is parameterized.
+  # All classes with 'metaclass=ERP5InstanceTestMeta' are parameterized
+  # except from a class which has been automatically instantiated from
+  # such user class. This exception prevents infinite recursion due to
+  # a parameterized class which tries to parameterize itself again.
+  def _isParameterized(self):
+    return not getattr(self, '.created_by_parametrize', False)
+
+  # Create multiple test classes from single definition.
+  @classmethod
+  def _parameterize(cls, base_class):
+    mod_dict = sys.modules[base_class.__module__].__dict__
+    for class_name_suffix, parameter_tuple in (base_class.__test_matrix__ or {}).items():
+      parameterized_cls_dict = dict(
+        base_class.__dict__,
+        **{
+          # Avoid infinite loop by a parameterized class which
+          # parameterize itself again and again and..
+          ".created_by_parametrize": True,
+          # Switch
+          #
+          #  .getInstanceParameterDict       to ._test_getInstanceParameterDict
+          #  ._base_getInstanceParameterDict to .getInstanceParameterDict
+          #
+          # so that we could inject base implementation to be called above
+          # user-defined getInstanceParameterDict.
+          "_test_getInstanceParameterDict": base_class.getInstanceParameterDict,
+          "getInstanceParameterDict": cls._getParameterizedInstanceParameterDict(parameter_tuple)
+        }
+      )
+      name = f"{base_class.__name__}_{class_name_suffix}"
+      mod_dict[name] = type(name, (base_class,), parameterized_cls_dict)
+
+  # _getParameterizedInstanceParameterDict returns a modified version of
+  # a test cases original 'getInstanceParameterDict'. The modified version
+  # applies parameters on the default instance parameters.
+  @staticmethod
+  def _getParameterizedInstanceParameterDict(parameter_tuple):
+    @classmethod
+    def getInstanceParameterDict(cls):
+      instance_parameter_dict = json.loads(
+        cls._test_getInstanceParameterDict().get("_", r"{}")
+      )
+      [p(instance_parameter_dict) for p in parameter_tuple]
+      return {"_": json.dumps(instance_parameter_dict)}
+    return getInstanceParameterDict
+
+  # Hide tests in unpatched base class: It doesn't make sense to run tests
+  # in original class, because parameters have not been assigned yet.
+  #
+  # We can't simply call 'delattr', because this wouldn't remove
+  # inherited tests. Overriding dir is sufficient, because this is
+  # the way how unittest discovers tests:
+  #   https://github.com/python/cpython/blob/3.11/Lib/unittest/loader.py#L237
+  def __dir__(self):
+    if self._isParameterized():
+      return [attr for attr in super().__dir__() if not attr.startswith('test')]
+    return super().__dir__()
+
+
+def matrix(*parameter_tuple):
+  """matrix creates a mapping of test_name -> parameter_tuple.
+
+  Each provided parameter_tuple won't be combined within itself,
+  but with any other provided parameter_tuple, for instance
+
+    >>> parameter_tuple0 = (param0, param1)
+    >>> parameter_tuple1 = (param2, param3)
+    >>> matrix(parameter_tuple0, parameter_tuple1)
+
+  will return all options of (param0 | param1) & (param2 | param3):
+
+    - param0_param2
+    - param0_param3
+    - param1_param2
+    - param1_param3
+  """
+  return {
+    "_".join([p.__name__ for p in params]): params
+    for params in itertools.product(*parameter_tuple)
+  }
+
+
+# Define parameters (function which receives instance params + modifies them).
+#
+# default runs tests without any adaption
+def default(instance_parameter_dict): ...
+
+
+def zeo(instance_parameter_dict):
+  instance_parameter_dict['zodb'] = [{"type": "zeo", "server": {}}]
+
+
+def neo(instance_parameter_dict):
+   # We don't provide encryption certificates in test runs for the sake
+  # of simplicity. By default SSL is turned on, we need to explicitly
+  # deactivate it:
+  #   https://lab.nexedi.com/nexedi/slapos/blob/a8150a1ac/software/neoppod/instance-neo-input-schema.json#L61-65
+  instance_parameter_dict['zodb'] = [{"type": "neo", "server": {"ssl": False}}]
+
+
+class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMeta):
   """ERP5 base test case
   """
+  __test_matrix__ = matrix((zeo, neo))  # switch between NEO and ZEO mode
+
   @classmethod
   def getRootPartitionConnectionParameterDict(cls):
     """Return the output paramters from the root partition"""

software/erp5/test/test/test_balancer.py

@@ -25,7 +25,7 @@ from slapos.testing.testcase import ManagedResource
 from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
                                   findFreeTCPPort)
 
-from . import ERP5InstanceTestCase, setUpModule
+from . import ERP5InstanceTestCase, setUpModule, matrix, default
 
 setUpModule  # pyflakes
 
@@ -132,6 +132,12 @@ class CaucaseService(ManagedResource):
 
 
 class BalancerTestCase(ERP5InstanceTestCase):
+  # We explicitly specify 'balancer' as our software type here,
+  # therefore we don't request ZODB. We therefore don't
+  # need to run these tests with both NEO and ZEO mode,
+  # it wouldn't make any difference.
+  #   https://lab.nexedi.com/nexedi/slapos/blob/273037c8/stack/erp5/instance.cfg.in#L216-230
+  __test_matrix__ = matrix((default,))
 
   @classmethod
   def getInstanceSoftwareType(cls):

software/erp5/test/test/test_erp5.py

 ##############################################################################
 #
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -46,7 +46,7 @@ import xmlrpc.client
 import urllib3
 from slapos.testing.utils import CrontabMixin
 
-from . import ERP5InstanceTestCase, setUpModule
+from . import ERP5InstanceTestCase, setUpModule, matrix, default
 
 setUpModule # pyflakes
 
@@ -119,6 +119,7 @@ class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
   """Test ERP5 can be instantiated with no parameters
   """
   __partition_reference__ = 'defp'
+  __test_matrix__ = matrix((default,))
 
 
 class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
@@ -310,6 +311,7 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
   """Test override zope node parameters
   """
   __partition_reference__ = 'override'
+  __test_matrix__ = matrix((default,))
 
   @classmethod
   def getInstanceParameterDict(cls):

software/erp5/test/test/test_mariadb.py

@@ -45,6 +45,9 @@ from slapos.testing.utils import getPromisePluginParameterDict
 
 from . import ERP5InstanceTestCase
 from . import setUpModule
+from . import matrix
+from . import default
+
 setUpModule  # pyflakes
 
 
@@ -52,6 +55,12 @@ class MariaDBTestCase(ERP5InstanceTestCase):
   """Base test case for mariadb tests.
   """
   __partition_reference__ = 'm'
+  # We explicitly specify 'mariadb' as our software type here,
+  # therefore we don't request ZODB. We therefore don't
+  # need to run these tests with both NEO and ZEO mode,
+  # it wouldn't make any difference.
+  #   https://lab.nexedi.com/nexedi/slapos/blob/273037c8/stack/erp5/instance.cfg.in#L216-230
+  __test_matrix__ = matrix((default,))
 
   @classmethod
   def getInstanceSoftwareType(cls):

software/erp5/test/test/test_wcfs.py

-# Copyright (C) 2021  Nexedi SA and Contributors.
+# Copyright (C) 2022  Nexedi SA and Contributors.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -43,6 +43,18 @@ class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   """
   __partition_reference__ = 'wcfs'
 
+  # Only run in ZEO mode; don't run with NEO.
+  # Current NEO/py and NEO/go versions have interoperability
+  # issues. Once these issues are fixed the following
+  # lines have to be removed so that test case runs agains NEO.
+  # Please see the following MR for more context:
+  # https://lab.nexedi.com/nexedi/slapos/merge_requests/1283#note_174854
+  @classmethod
+  def setUpClass(cls):
+    if json.loads(cls.getInstanceParameterDict()["_"])['zodb'][0]["type"] == "neo":
+      raise unittest.SkipTest("Not yet fixed WCFS+NEO interoperability issue.")
+    super().setUpClass()
+
   @classmethod
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wcfs': {'enable': True}})}