test/*: Start testing zodbtools behaviour on ZODB databases of multiple kinds

Since 0b6f99da (test/gen_testdata: Fix for ZODB5 > 5.5.1 + preserve
database compatibility with ZODB3/py2) we are generating our test
database with using pickle protocol=2. This was done in order to make
sure the zodbtools works ok with data generated by e.g. ZODB4. However
we still have lots of databases that are generated with pickle
protocol=1, and we also have newer databases that are generated with
pickle protocol=3. Note that it is not only data records who are
affected by specified pickle protocol. For example for FileStorage the
index is also saved via pickling and so used pickle protocol affects
index format. For example ZODB/go currently cannot load FileStorage
index if it was saved via protocol=3.

Since zodbtools should work ok with any data it creates a need to test
it against all kinds of ZODB databases: generated by either py2 or py3,
and saved via pickle protocol 1,2 and 3.

In this patch we add infrastructure for such testing and extend testdata
coverage to cover not only py2_pickle2, but also py2_pickle1. We will
add support for py2_pickle3 and py3_pickle3 in the follow-up patches.

Testdata files are now located inside dedicated subdirectories - one for
one ZODB kind. py2_pickle2/* is exactly the same compared to testdata
files we had before. py2_pickle1/* is bit different. It is handy to see
the difference via e.g.

    $ diff -u py2_pickle2/zdump.zpickledis.ok py2_pickle1/zdump.zpickledis.ok

In tests particular kind of testdata is now accessed via ztestdata
fixture. Please see changes in zodbtools/test/conftest.py and
zodbtools/test/gen_testdata.py for details.

zodbtools/test/conftest.py

-# Copyright (C) 2019  Nexedi SA and Contributors.
-#                     Kirill Smelkov <kirr@nexedi.com>
+# Copyright (C) 2019-2024  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -19,6 +19,51 @@
 
 import pytest
 from zodbtools.test.testutil import zext_supported
+import os
+from os.path import basename, dirname, relpath
+from tempfile import mkdtemp
+from shutil import rmtree
+import pkg_resources
+
+testdir = dirname(__file__)
+
+
+# ztestdata is test fixture to run a test wrt particular ZODB testdata case.
+#
+# It yields all testdata cases generated by gen_testdata.py for
+# all covered ZODB pickle kinds.
+#
+# ztestdata.prefix is where test database and other generated files live.
+# ztestdata.prefix + '/data.fs' , in particular, is the path to test database.
+@pytest.fixture(params=[
+        (name, zext, zkind)
+        # NOTE keep in sync with run_with_all_zodb_pickle_kinds
+        for name    in ('1',)
+        for zext    in (False, True)
+        for zkind   in ('py2_pickle1', 'py2_pickle2')
+    ],
+    ids = lambda _: '%s%s/%s' % (_[0], '' if _[1] else '_!zext', _[2]),
+)
+def ztestdata(request): # -> ZTestData
+    name, zext, zkind = request.param
+    _ = ZTestData()
+    _.name  = name
+    _.zext  = zext
+    _.zkind = zkind
+    return _
+
+class ZTestData(object):
+    __slots__ = (
+        'name',
+        'zext',
+        'zkind',
+    )
+
+    @property
+    def prefix(self):
+        _ = '%s/testdata/%s%s/%s' % (testdir, self.name, '' if self.zext else '_!zext', self.zkind)
+        return relpath(_)
+
 
 # zext is a test fixture function object that allows to exercise 2 cases:
 #

zodbtools/test/gen_testdata.py

@@ -18,9 +18,21 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-"""generate reference database and index for tests
+"""generate reference FileStorage databases and indices for tests
 
-Golden zodbdump & zodbanalyze outputs are also generated besides database itself.
+We generate test database with all, potentially tricky, cases of transaction,
+data and index records. This database is generated multiple times with
+different ZODB settings that try to mimic what notable ZODB versions would
+produce. The following combinations are covered:
+
+    py2: ZODB 4 and ZODB5 < 5.3     (pickle protocol 1)
+    py2: ZODB 5.3                   (pickle protocol 2)
+
+Each such combination is referred to by "zkind" which indicates major Python
+and pickle protocol versions used, for example "py2_pickle3". See
+run_with_all_zodb_pickle_kinds function for details.
+
+Golden zodbdump & zodbanalyze outputs are also generated besides databases themselves.
 """
 
 # NOTE result of this script must be saved in version control and should not be
@@ -50,8 +62,9 @@ from persistent import Persistent
 import transaction
 
 import os
-import glob
+import os.path
 import sys
+import shutil
 import struct
 import time
 import random2
@@ -151,13 +164,43 @@ def ext4subj(subj):
 
     return ext
 
-# run_with_zodb4py2_compat(f) runs f preserving database compatibility with
+
+# run_with_all_zodb_pickle_kinds runs f for all ZODB pickle kinds we care about.
+#
+# For each kind f is run separately under corresponding environment.
+# We currently support the following kinds:
+#
+#   py2: ZODB with pickle protocol = 1      generated by ZODB4 and ZODB5 < 5.3
+#   py2: ZODB with pickle protocol = 2      generated by ZODB5 5.3
+#
+# For convenience f can detect under which environment it is being executed via current_zkind.
+#
+# NOTE only the kinds supported under current python are executed.
+def run_with_all_zodb_pickle_kinds(f):
+    # NOTE keep in sync with ztestdata fixture.
+    def _(expect_protocol=None):
+        from ZODB import serialize as zserialize
+        if expect_protocol is not None:
+            assert zserialize._protocol == expect_protocol, (current_zkind(), expect_protocol)
+        f()
+    _run_with_zodb4py2_compat(_, 1)
+    _run_with_zodb4py2_compat(_, 2)
+
+# current_zkind returns string indicating currently activated ZODB environment,
+# for example "py2_pickle3".
+def current_zkind():
+    from ZODB import serialize as zserialize
+    zkind = "py%d_pickle%d" % (sys.version_info.major, zserialize._protocol)
+    return zkind
+
+# _run_with_zodb4py2_compat runs f preserving database compatibility with
 # ZODB4/py2, which generates pickles encoded with protocol < 3.
 #
 # ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0:
 # https://github.com/zopefoundation/ZODB/commit/12ee41c4
-# Undo it, while we generate test database.
-def run_with_zodb4py2_compat(f):
+# Undo it, while we generate test database as if produced by older ZODB.
+def _run_with_zodb4py2_compat(f, protocol):
+    assert protocol < 3
     import ZODB.ConflictResolution
     import ZODB.Connection
     import ZODB.ExportImport
@@ -168,21 +211,20 @@ def run_with_zodb4py2_compat(f):
     import ZODB.serialize
     binary    = getattr(ZODB.serialize, 'binary', None)
     _protocol = getattr(ZODB.serialize, '_protocol', None)
-    Pz4 = 2
     try:
         ZODB.serialize.binary    = bytes
         # XXX cannot change just ZODB._compat._protocol, because many modules
         # do `from ZODB._compat import _protocol` and just `import ZODB`
         # imports many ZODB.X modules. In other words we cannot change
         # _protocol just in one place.
-        ZODB.ConflictResolution._protocol       = Pz4
-        ZODB.Connection._protocol               = Pz4
-        ZODB.ExportImport._protocol             = Pz4
-        ZODB.FileStorage.FileStorage._protocol  = Pz4
-        ZODB._compat._protocol                  = Pz4
-        ZODB.broken._protocol                   = Pz4
-        ZODB.fsIndex._protocol                  = Pz4
-        ZODB.serialize._protocol                = Pz4
+        ZODB.ConflictResolution._protocol       = protocol
+        ZODB.Connection._protocol               = protocol
+        ZODB.ExportImport._protocol             = protocol
+        ZODB.FileStorage.FileStorage._protocol  = protocol
+        ZODB._compat._protocol                  = protocol
+        ZODB.broken._protocol                   = protocol
+        ZODB.fsIndex._protocol                  = protocol
+        ZODB.serialize._protocol                = protocol
 
         f()
     finally:
@@ -196,15 +238,11 @@ def run_with_zodb4py2_compat(f):
         ZODB.fsIndex._protocol                  = _protocol
         ZODB.serialize._protocol                = _protocol
 
+
 # gen_testdb generates test FileStorage database @ outfs_path.
 #
 # zext indicates whether or not to include non-empty extension into transactions.
 def gen_testdb(outfs_path, zext=True):
-    def _():
-        _gen_testdb(outfs_path, zext)
-    run_with_zodb4py2_compat(_)
-
-def _gen_testdb(outfs_path, zext):
     xtime_reset()
 
     def ext(subj):
@@ -317,30 +355,34 @@ def main():
     if not zext_supported():
         raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
 
-    out = "testdata/1"
-    for zext in [True, False]:
-        dbname = out
-        if not zext:
-            dbname += "_!zext"
-        for f in glob.glob(dbname + '.*'):
-            os.remove(f)
-        gen_testdb("%s.fs" % dbname, zext=zext)
-
-        # prepare zdump.ok for generated database
-        stor = FileStorage("%s.fs" % dbname, read_only=True)
-        for pretty in ('raw', 'zpickledis'):
-            with open("%s.zdump.%s.ok" % (dbname, pretty), "wb") as f:
-                zodbdump(stor, None, None, pretty=pretty, out=f)
-
-        # prepare zanalyze.csv.ok
-        sys_stdout = sys.stdout
-        sys.stdout = open("%s.zanalyze.csv.ok" % dbname, "w")
-        zodbanalyze.report(
-            zodbanalyze.analyze("%s.fs" % dbname, use_dbm=False, delta_fs=False, tidmin=None, tidmax=None),
-            csv=True,
-        )
-        sys.stdout.close()
-        sys.stdout = sys_stdout
+    top = "testdata/1"
+    def _():
+        for zext in [True, False]:
+            prefix = "%s%s/%s" % (top, "" if zext else "_!zext", current_zkind())
+            if os.path.exists(prefix):
+                shutil.rmtree(prefix)
+            os.makedirs(prefix)
+
+            outfs = "%s/data.fs" % prefix
+            gen_testdb(outfs, zext=zext)
+
+            # prepare zdump.ok for generated database
+            stor = FileStorage(outfs, read_only=True)
+            for pretty in ('raw', 'zpickledis'):
+                with open("%s/zdump.%s.ok" % (prefix, pretty), "wb") as f:
+                    zodbdump(stor, None, None, pretty=pretty, out=f)
+
+            # prepare zanalyze.csv.ok
+            sys_stdout = sys.stdout
+            sys.stdout = open("%s/zanalyze.csv.ok" % prefix, "w")
+            zodbanalyze.report(
+                zodbanalyze.analyze(outfs, use_dbm=False, delta_fs=False, tidmin=None, tidmax=None),
+                csv=True,
+            )
+            sys.stdout.close()
+            sys.stdout = sys_stdout
+
+    run_with_all_zodb_pickle_kinds(_)
 
 
 if __name__ == '__main__':

zodbtools/test/test_analyze.py

@@ -24,10 +24,9 @@ import os.path
 from golang import b
 
 
-def test_zodbanalyze(tmpdir, capsys):
-    testdata = os.path.join(os.path.dirname(__file__), "testdata")
+def test_zodbanalyze(tmpdir, ztestdata, capsys):
     tfs1 = fs1_testdata_py23(tmpdir,
-                    os.path.join(testdata, "1.fs"))
+                    os.path.join(ztestdata.prefix, "data.fs"))
 
     for use_dbm in (False, True):
         report(
@@ -57,7 +56,7 @@ def test_zodbanalyze(tmpdir, capsys):
     )
     captured = capsys.readouterr()
 
-    with open('%s/1.zanalyze.csv.ok' % testdata, 'r') as f:
+    with open('%s/zanalyze.csv.ok' % ztestdata.prefix, 'r') as f:
         zanalyze_csv_ok = f.read()
 
     assert captured.out == zanalyze_csv_ok

zodbtools/test/test_dump.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2022  Nexedi SA and Contributors.
+# Copyright (C) 2017-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -28,20 +28,17 @@ from ZODB.FileStorage import FileStorage
 from ZODB.utils import p64
 from io import BytesIO
 
-from os.path import dirname
-
-from zodbtools.test.testutil import zext_supported, fs1_testdata_py23
+from zodbtools.test.testutil import fs1_testdata_py23
 from pytest import mark, raises, xfail
 
+
 # verify zodbdump output against golden
 @mark.parametrize('pretty', ('raw', 'zpickledis'))
-def test_zodbdump(tmpdir, zext, pretty):
-    tdir  = dirname(__file__)
-    zkind = '_!zext' if zext.disabled else ''
-    tfs1  = fs1_testdata_py23(tmpdir, '%s/testdata/1%s.fs' % (tdir, zkind))
+def test_zodbdump(tmpdir, ztestdata, pretty):
+    tfs1  = fs1_testdata_py23(tmpdir, '%s/data.fs' % ztestdata.prefix)
     stor  = FileStorage(tfs1, read_only=True)
 
-    with open('%s/testdata/1%s.zdump.%s.ok' % (tdir, zkind, pretty), 'rb') as f:
+    with open('%s/zdump.%s.ok' % (ztestdata.prefix, pretty), 'rb') as f:
         dumpok = f.read()
 
     out = BytesIO()

zodbtools/test/test_restore.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2021-2022  Nexedi SA and Contributors.
+# Copyright (C) 2021-2024  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -24,22 +24,16 @@ from zodbtools.zodbrestore import zodbrestore
 from zodbtools.util import storageFromURL, readfile
 from zodbtools.test.testutil import fs1_testdata_py23
 
-from os.path import dirname
-from tempfile import mkdtemp
-from shutil import rmtree
 from golang import func, defer
 
 # verify zodbrestore.
 @func
-def test_zodbrestore(tmpdir, zext):
-    zkind = '_!zext' if zext.disabled else ''
-
-    # restore from testdata/1.zdump.ok and verify it gives result that is
-    # bit-to-bit identical to testdata/1.fs
-    tdata = dirname(__file__) + "/testdata"
+def test_zodbrestore(tmpdir, ztestdata):
+    # restore from zdump.ok and verify it gives result that is
+    # bit-to-bit identical to data.fs
     @func
     def _():
-        zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb')
+        zdump = open("%s/zdump.raw.ok" % ztestdata.prefix, 'rb')
         defer(zdump.close)
 
         stor = storageFromURL('%s/2.fs' % tmpdir)
@@ -48,6 +42,6 @@ def test_zodbrestore(tmpdir, zext):
         zodbrestore(stor, zdump)
     _()
 
-    zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/1%s.fs" % (tdata, zkind)))
+    zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/data.fs" % ztestdata.prefix))
     zfs2 = readfile("%s/2.fs" % tmpdir)
     assert zfs1 == zfs2

zodbtools/test/testdata/.gitignore

-/*.lock
-/*.tmp
-/*.old
+*.lock
+*.tmp
+*.old

zodbtools/test/testdata/1/py2_pickle1/zanalyze.csv.ok

+Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
+persistent.mapping.PersistentMapping,3,648,25.038640%,216.000000,1,216,2,432
+__main__.Object,65,1940,74.961360%,29.846154,9,283,56,1657

zodbtools/test/testdata/1_!zext/py2_pickle1/zanalyze.csv.ok

+Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
+persistent.mapping.PersistentMapping,3,648,25.038640%,216.000000,1,216,2,432
+__main__.Object,65,1940,74.961360%,29.846154,9,283,56,1657