software/erp5/tests: Extend WCFS test scope

This patch enhances the ERP5 + WCFS integration tests. Before it was
only minimal; now it runs actual workload of writing and reading
ZBigArray to the ZODB. This is necessary to securely verify WC2 is
working with ERP5.

We had to implement the new tests in a separate test environment,
because SlapOS Integration tests run with python3, but we need python2
for NEO support. We need NEO, because we want to test if WCFS works with
NEO client.

software/erp5/instance-erp5-wcfs-test.cfg.in

+[buildout]
+extends =
+  {{ template_monitor }}
+parts +=
+  publish
+
+[publish]
+<= monitor-publish
+recipe = slapos.cookbook:publish.serialised
+erp5_wcfs_test_bin = {{ erp5_wcfs_test_bin }}

software/erp5/instance-test.cfg.in

+[buildout]
+extends =
+  {{ template }}
+
+[dynamic-template-erp5-wcfs-test-parameters]
+bin = {{ erp5_wcfs_test_bin }}
+
+[dynamic-template-erp5-wcfs-test]
+<= jinja2-template-base
+url = {{ template_erp5_wcfs_test }}
+filename = instance-erp5-wcfs-test.cfg
+extra-context = =
+  key erp5_wcfs_test_bin dynamic-template-erp5-wcfs-test-parameters:bin
+
+[switch-softwaretype]
+erp5-wcfs-test = dynamic-template-erp5-wcfs-test:output

software/erp5/test.cfg

@@ -11,3 +11,51 @@ major = 4-wc2
 [neoppod-repository]
 repository = https://lab.nexedi.com/kirr/neo.git
 branch = t
+
+# We use external standalone test program to verify WCFS is working
+# as expected. We can't use the default erp5 test suite, because this
+# test suite runs with py3. We need NEO for testing, but this doesn't
+# support py3 yet.
+[template-erp5-wcfs-test]
+<= download-base
+filename = "instance-test.cfg.in"
+
+[_template]
+<= template
+output = ${buildout:directory}/_template.cfg
+
+[template]
+url = ${:_profile_base_location_}/instance-test.cfg.in
+context =
+  key template _template:output
+  key template_erp5_wcfs_test template-erp5-wcfs-test:target
+  key erp5_wcfs_test_bin erp5-wcfs-test:bin
+
+[erp5-wcfs-test]
+recipe = zc.recipe.egg
+eggs =
+  ${pytest:eggs}
+  ${ZODB4-wc2:egg}
+  ${numpy:egg}
+  ${wendelin.core:egg}
+  ${neoppod:eggs}
+  ${ZEO:egg}
+  ${erp5-wcfs-test-egg:egg}
+entry-points = ${:_buildout_section_name_}=__main__:main
+initialization =
+  import sys
+  import pytest
+  zurl = sys.argv[1]
+  error_code = pytest.main(["--zurl", zurl, "${erp5-wcfs-test-egg:setup}"])
+  sys.exit(error_code)
+bin = ${:bin-directory}/${:_buildout_section_name_}
+
+[erp5-wcfs-test-egg]
+recipe = zc.recipe.egg:develop
+egg = slapos.test.erp5_wcfs
+setup = ${slapos-repository:location}/software/erp5/test_wcfs/
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+branch = master

software/erp5/test/test/test_wcfs.py

@@ -18,6 +18,7 @@
 
 import json
 import os.path
+import subprocess
 import unittest
 
 from slapos.grid.utils import md5digest
@@ -47,20 +48,28 @@ class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wcfs': {'enable': True}})}
 
+  def getZUrl(self):
+    return json.loads(
+      self.getComputerPartition('wcfs').getConnectionParameter('_')
+    )['serving-zurl']
+
   def test_wcfs_accessible(self):
     """Verify that wcfs filesystem is basically accessible.
 
        - we can read .wcfs/zurl
        - its content is equal to published `serving-zurl`
     """
-    zurl = json.loads(
-             self.getComputerPartition('wcfs').getConnectionParameter('_')
-           )['serving-zurl']
-
+    zurl = self.getZUrl()
     mntpt = lookupMount(zurl)
     zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
     self.assertEqual(zurl_, zurl)
 
+  def test_workload(self):
+    """Verify simple workload (reading/writing operation) on WCFS is successful"""
+    wcfs_test_bin = self.getRootPartitionConnectionParameterDict()['erp5_wcfs_test_bin']
+    # If exit code = 0, all subtests were successful
+    self.assertEqual(0, subprocess.call([wcfs_test_bin, self.getZUrl()]))
+
 
 # lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
 def lookupMount(zurl):

software/erp5/test_wcfs/README.md

+Test wendelin.core v2 integration with ERP5 SR

software/erp5/test_wcfs/conftest.py

+def pytest_addoption(parser):
+    parser.addoption("--zurl", action="store")

software/erp5/test_wcfs/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.erp5_wcfs'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test WCFS integration in SlapOS' ERP5 software release",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'ZODB',
+        'numpy',
+        'wendelin.core',
+        'ZEO',
+        'neoppod',
+      ],
+    )

software/erp5/test_wcfs/test/test_wcfs.py

+##############################################################################
+#
+# Copyright (c) 2022 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import pytest
+import numpy as np
+from wendelin.bigarray.array_zodb import ZBigArray
+from wendelin.lib.zodb import dbopen, dbclose
+from wendelin.wcfs import join
+from ZODB import DB
+import transaction
+
+
+@pytest.fixture(scope="session")
+def zurl(pytestconfig):
+  return pytestconfig.getoption("zurl")
+
+
+def test_access_wcfs(zurl):
+  wcfs = join(zurl)
+  assert wcfs
+
+
+def test_array_is_writeable_and_readable(zurl):
+  """Verify writing/reading ZBigArray to/from ZODB works."""
+  _ = getRunner(zurl)
+  @_
+  def create_array_and_commit(root):
+    assert not hasattr(root, 'zarray')
+    root.zarray = ZBigArray(shape=(4,), dtype=int)
+    transaction.commit()
+  @_
+  def verify_array_has_been_written_to_zodb(root):
+    assert hasattr(root, 'zarray')
+    assert root.zarray.shape, (4,)
+    assert root.zarray.shape, (4,)
+    np.testing.assert_array_equal(root.zarray[:], [0,0,0,0])
+  @_
+  def mutate_array_and_commit(root):
+    root.zarray[:][0] = 100; transaction.commit()
+  @_
+  def verify_mutation_has_been_written_to_zodb(root):
+    assert root.zarray[:][0] == 100
+  @_
+  def cleanup(root):
+    del root.zarray; transaction.commit()
+
+
+def getRunner(zurl):
+  def run(func):
+    root = dbopen(zurl)
+    func(root)
+    dbclose(root)
+
+  return run