commit: test: Verify it with both FileStorage and ZEO

There is currently one problem with `zodb commit` that exhibits it only
with ZEO. And it could be triggered only via commandline.

As a preparatory step for the fix add infrastructure to test zodbcommit
with both FileStorage and ZEO via zsrv fixture.

There is currently no breakage - test_commit continues to work ok. In
the next patch we will add test for commandline handling of `zodb
commit` and fix the issue.

P.S. Support code for testing wrt ZEO is based on wendelin.core :

https://lab.nexedi.com/nexedi/wendelin.core/-/blob/07087ec8/lib/testing.py#L230-308

Maybe in the future it would be a good idea to switch wendelin.core to
use zodbtools-provided ZSrv fixture.

setup.py

@@ -23,7 +23,7 @@ setup(
     install_requires = ['ZODB', 'zodbpickle', 'zodburi', 'zope.interface', 'pygolang >= 0.0.0.dev6', 'six', 'dateparser'],
 
     extras_require = {
-                  'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"', 'random2'],
+                  'test': ['pytest', 'freezegun', 'pytz', 'mock;python_version<="2.7"', 'random2', 'ZEO[test]'],
     },
 
     entry_points= {'console_scripts': ['zodb = zodbtools.zodb:main']},

zodbtools/test/conftest.py

@@ -99,3 +99,97 @@ def zext(request):
         if not zext_supported():
             request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
         return _
+
+
+
+# TestZSrv is base class for all test ZODB storages.
+class TestZSrv(object):
+    # .idname    string to use as subtest ID
+    # .zurl      URI to access the storage
+    # .all_logs  returns string with all current server logs
+    # .teardown  should be called when the server is no longer used
+    pass
+
+
+# TestFileStorage provides FileStorage for tests.
+class TestFileStorage(TestZSrv):
+    idname = 'FileStorage'
+    def __init__(self):
+        self.tmpd = mkdtemp('', 'test_filestorage.')
+        self.zurl = '%s/1.fs' % self.tmpd
+
+    def all_logs(self):
+        return "FileStorage: no logs"
+
+    def teardown(self):
+        rmtree(self.tmpd)
+
+
+# TestZEOSrv provides ZEO server for tests.
+class TestZEOSrv(TestZSrv):
+    idname = 'ZEO'
+    def __init__(self):
+        from ZEO.tests import forker
+        self.zeo_forker = forker
+
+        # .z5 represents whether we are running with ZEO5 or earlier
+        dzeo    = pkg_resources.working_set.find(pkg_resources.Requirement.parse('ZEO'))
+        v5      = pkg_resources.parse_version('5.0dev')
+
+        assert dzeo is not None
+        self.z5 = (dzeo.parsed_version >= v5)
+
+        self.tmpd = mkdtemp('', 'test_zeo.')
+        self.log  = '%s/zeo.log' % self.tmpd
+        port  = self.zeo_forker.get_port()
+        zconf = self.zeo_forker.ZEOConfig(('', port), log=self.log)
+        _ = self.zeo_forker.start_zeo_server(path='%s/1.fs' % self.tmpd, zeo_conf=zconf, port=port)
+        if self.z5:
+            self.addr, self.stop = _
+        else:
+            self.addr, self.adminaddr, self.pid, self.path = _
+
+        self.zurl = 'zeo://localhost:%d/' % port
+
+    def all_logs(self):
+        log = '%s:\n\n' % basename(self.log)
+        with open(self.log) as f:
+            log += f.read()
+        return log
+
+    def teardown(self):
+        if self.z5:
+            self.stop()
+        else:
+            self.zeo_forker.shutdown_zeo_server(self.adminaddr)
+            os.waitpid(self.pid, 0)
+
+        rmtree(self.tmpd)
+
+
+# zsrv is test fixture to run a test wrt particular ZODB storage server.
+#
+# It currently yields FileStorage and ZEO.
+#
+# Clients should use zsrv.zurl to connect to the storage.
+# See TestZSrv and its children classes for details.
+@pytest.fixture(params=[TestFileStorage, TestZEOSrv],
+                ids = lambda _: _.idname)
+def zsrv(request): # -> ~TestZSrv
+    nfail = request.session.testsfailed
+    zsrv = request.param()
+
+    yield zsrv
+
+    # see if current test failed
+    # https://stackoverflow.com/a/43268134/9456786
+    failed = False
+    if request.session.testsfailed > nfail:
+        failed = True
+
+    # dump server logs on test failure
+    if failed:
+        print()
+        print(zsrv.all_logs())
+
+    zsrv.teardown()

zodbtools/test/test_commit.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2022  Nexedi SA and Contributors.
+# Copyright (C) 2018-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -25,17 +25,12 @@ from zodbtools.util import storageFromURL, sha1
 from ZODB.utils import p64, u64, z64
 from ZODB._compat import BytesIO, dumps, _protocol   # XXX can't yet commit with arbitrary ext.bytes
 
-from tempfile import mkdtemp
-from shutil import rmtree
 from golang import func, defer, b
 
 # verify zodbcommit.
 @func
-def test_zodbcommit(zext):
-    tmpd = mkdtemp('', 'zodbcommit.')
-    defer(lambda: rmtree(tmpd))
-
-    stor = storageFromURL('%s/2.fs' % tmpd)
+def test_zodbcommit(zsrv, zext):
+    stor = storageFromURL(zsrv.zurl)
     defer(stor.close)
 
     head = stor.lastTransaction()