Commit bf772ce0 authored by Kirill Smelkov's avatar Kirill Smelkov

test/*: Start testing zodbtools behaviour on ZODB databases of multiple kinds

Since 0b6f99da (test/gen_testdata: Fix for ZODB5 > 5.5.1 + preserve
database compatibility with ZODB3/py2) we are generating our test
database with using pickle protocol=2. This was done in order to make
sure the zodbtools works ok with data generated by e.g. ZODB4. However
we still have lots of databases that are generated with pickle
protocol=1, and we also have newer databases that are generated with
pickle protocol=3. Note that it is not only data records who are
affected by specified pickle protocol. For example for FileStorage the
index is also saved via pickling and so used pickle protocol affects
index format. For example ZODB/go currently cannot load FileStorage
index if it was saved via protocol=3.

Since zodbtools should work ok with any data it creates a need to test
it against all kinds of ZODB databases: generated by either py2 or py3,
and saved via pickle protocol 1,2 and 3.

In this patch we add infrastructure for such testing and extend testdata
coverage to cover not only py2_pickle2, but also py2_pickle1. We will
add support for py2_pickle3 and py3_pickle3 in the follow-up patches.

Testdata files are now located inside dedicated subdirectories - one for
one ZODB kind. py2_pickle2/* is exactly the same compared to testdata
files we had before. py2_pickle1/* is bit different. It is handy to see
the difference via e.g.

    $ diff -u py2_pickle2/zdump.zpickledis.ok py2_pickle1/zdump.zpickledis.ok

In tests particular kind of testdata is now accessed via ztestdata
fixture. Please see changes in zodbtools/test/conftest.py and
zodbtools/test/gen_testdata.py for details.
parent e66451c0
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -19,6 +19,51 @@
import pytest
from zodbtools.test.testutil import zext_supported
import os
from os.path import basename, dirname, relpath
from tempfile import mkdtemp
from shutil import rmtree
import pkg_resources
testdir = dirname(__file__)
# ztestdata is test fixture to run a test wrt particular ZODB testdata case.
#
# It yields all testdata cases generated by gen_testdata.py for
# all covered ZODB pickle kinds.
#
# ztestdata.prefix is where test database and other generated files live.
# ztestdata.prefix + '/data.fs' , in particular, is the path to test database.
@pytest.fixture(params=[
(name, zext, zkind)
# NOTE keep in sync with run_with_all_zodb_pickle_kinds
for name in ('1',)
for zext in (False, True)
for zkind in ('py2_pickle1', 'py2_pickle2')
],
ids = lambda _: '%s%s/%s' % (_[0], '' if _[1] else '_!zext', _[2]),
)
def ztestdata(request): # -> ZTestData
name, zext, zkind = request.param
_ = ZTestData()
_.name = name
_.zext = zext
_.zkind = zkind
return _
class ZTestData(object):
__slots__ = (
'name',
'zext',
'zkind',
)
@property
def prefix(self):
_ = '%s/testdata/%s%s/%s' % (testdir, self.name, '' if self.zext else '_!zext', self.zkind)
return relpath(_)
# zext is a test fixture function object that allows to exercise 2 cases:
#
......
......@@ -18,9 +18,21 @@
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""generate reference database and index for tests
"""generate reference FileStorage databases and indices for tests
Golden zodbdump & zodbanalyze outputs are also generated besides database itself.
We generate test database with all, potentially tricky, cases of transaction,
data and index records. This database is generated multiple times with
different ZODB settings that try to mimic what notable ZODB versions would
produce. The following combinations are covered:
py2: ZODB 4 and ZODB5 < 5.3 (pickle protocol 1)
py2: ZODB 5.3 (pickle protocol 2)
Each such combination is referred to by "zkind" which indicates major Python
and pickle protocol versions used, for example "py2_pickle3". See
run_with_all_zodb_pickle_kinds function for details.
Golden zodbdump & zodbanalyze outputs are also generated besides databases themselves.
"""
# NOTE result of this script must be saved in version control and should not be
......@@ -50,8 +62,9 @@ from persistent import Persistent
import transaction
import os
import glob
import os.path
import sys
import shutil
import struct
import time
import random2
......@@ -151,13 +164,43 @@ def ext4subj(subj):
return ext
# run_with_zodb4py2_compat(f) runs f preserving database compatibility with
# run_with_all_zodb_pickle_kinds runs f for all ZODB pickle kinds we care about.
#
# For each kind f is run separately under corresponding environment.
# We currently support the following kinds:
#
# py2: ZODB with pickle protocol = 1 generated by ZODB4 and ZODB5 < 5.3
# py2: ZODB with pickle protocol = 2 generated by ZODB5 5.3
#
# For convenience f can detect under which environment it is being executed via current_zkind.
#
# NOTE only the kinds supported under current python are executed.
def run_with_all_zodb_pickle_kinds(f):
# NOTE keep in sync with ztestdata fixture.
def _(expect_protocol=None):
from ZODB import serialize as zserialize
if expect_protocol is not None:
assert zserialize._protocol == expect_protocol, (current_zkind(), expect_protocol)
f()
_run_with_zodb4py2_compat(_, 1)
_run_with_zodb4py2_compat(_, 2)
# current_zkind returns string indicating currently activated ZODB environment,
# for example "py2_pickle3".
def current_zkind():
from ZODB import serialize as zserialize
zkind = "py%d_pickle%d" % (sys.version_info.major, zserialize._protocol)
return zkind
# _run_with_zodb4py2_compat runs f preserving database compatibility with
# ZODB4/py2, which generates pickles encoded with protocol < 3.
#
# ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0:
# https://github.com/zopefoundation/ZODB/commit/12ee41c4
# Undo it, while we generate test database.
def run_with_zodb4py2_compat(f):
# Undo it, while we generate test database as if produced by older ZODB.
def _run_with_zodb4py2_compat(f, protocol):
assert protocol < 3
import ZODB.ConflictResolution
import ZODB.Connection
import ZODB.ExportImport
......@@ -168,21 +211,20 @@ def run_with_zodb4py2_compat(f):
import ZODB.serialize
binary = getattr(ZODB.serialize, 'binary', None)
_protocol = getattr(ZODB.serialize, '_protocol', None)
Pz4 = 2
try:
ZODB.serialize.binary = bytes
# XXX cannot change just ZODB._compat._protocol, because many modules
# do `from ZODB._compat import _protocol` and just `import ZODB`
# imports many ZODB.X modules. In other words we cannot change
# _protocol just in one place.
ZODB.ConflictResolution._protocol = Pz4
ZODB.Connection._protocol = Pz4
ZODB.ExportImport._protocol = Pz4
ZODB.FileStorage.FileStorage._protocol = Pz4
ZODB._compat._protocol = Pz4
ZODB.broken._protocol = Pz4
ZODB.fsIndex._protocol = Pz4
ZODB.serialize._protocol = Pz4
ZODB.ConflictResolution._protocol = protocol
ZODB.Connection._protocol = protocol
ZODB.ExportImport._protocol = protocol
ZODB.FileStorage.FileStorage._protocol = protocol
ZODB._compat._protocol = protocol
ZODB.broken._protocol = protocol
ZODB.fsIndex._protocol = protocol
ZODB.serialize._protocol = protocol
f()
finally:
......@@ -196,15 +238,11 @@ def run_with_zodb4py2_compat(f):
ZODB.fsIndex._protocol = _protocol
ZODB.serialize._protocol = _protocol
# gen_testdb generates test FileStorage database @ outfs_path.
#
# zext indicates whether or not to include non-empty extension into transactions.
def gen_testdb(outfs_path, zext=True):
def _():
_gen_testdb(outfs_path, zext)
run_with_zodb4py2_compat(_)
def _gen_testdb(outfs_path, zext):
xtime_reset()
def ext(subj):
......@@ -317,31 +355,35 @@ def main():
if not zext_supported():
raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
out = "testdata/1"
top = "testdata/1"
def _():
for zext in [True, False]:
dbname = out
if not zext:
dbname += "_!zext"
for f in glob.glob(dbname + '.*'):
os.remove(f)
gen_testdb("%s.fs" % dbname, zext=zext)
prefix = "%s%s/%s" % (top, "" if zext else "_!zext", current_zkind())
if os.path.exists(prefix):
shutil.rmtree(prefix)
os.makedirs(prefix)
outfs = "%s/data.fs" % prefix
gen_testdb(outfs, zext=zext)
# prepare zdump.ok for generated database
stor = FileStorage("%s.fs" % dbname, read_only=True)
stor = FileStorage(outfs, read_only=True)
for pretty in ('raw', 'zpickledis'):
with open("%s.zdump.%s.ok" % (dbname, pretty), "wb") as f:
with open("%s/zdump.%s.ok" % (prefix, pretty), "wb") as f:
zodbdump(stor, None, None, pretty=pretty, out=f)
# prepare zanalyze.csv.ok
sys_stdout = sys.stdout
sys.stdout = open("%s.zanalyze.csv.ok" % dbname, "w")
sys.stdout = open("%s/zanalyze.csv.ok" % prefix, "w")
zodbanalyze.report(
zodbanalyze.analyze("%s.fs" % dbname, use_dbm=False, delta_fs=False, tidmin=None, tidmax=None),
zodbanalyze.analyze(outfs, use_dbm=False, delta_fs=False, tidmin=None, tidmax=None),
csv=True,
)
sys.stdout.close()
sys.stdout = sys_stdout
run_with_all_zodb_pickle_kinds(_)
if __name__ == '__main__':
main()
......@@ -24,10 +24,9 @@ import os.path
from golang import b
def test_zodbanalyze(tmpdir, capsys):
testdata = os.path.join(os.path.dirname(__file__), "testdata")
def test_zodbanalyze(tmpdir, ztestdata, capsys):
tfs1 = fs1_testdata_py23(tmpdir,
os.path.join(testdata, "1.fs"))
os.path.join(ztestdata.prefix, "data.fs"))
for use_dbm in (False, True):
report(
......@@ -57,7 +56,7 @@ def test_zodbanalyze(tmpdir, capsys):
)
captured = capsys.readouterr()
with open('%s/1.zanalyze.csv.ok' % testdata, 'r') as f:
with open('%s/zanalyze.csv.ok' % ztestdata.prefix, 'r') as f:
zanalyze_csv_ok = f.read()
assert captured.out == zanalyze_csv_ok
......
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2022 Nexedi SA and Contributors.
# Copyright (C) 2017-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -28,20 +28,17 @@ from ZODB.FileStorage import FileStorage
from ZODB.utils import p64
from io import BytesIO
from os.path import dirname
from zodbtools.test.testutil import zext_supported, fs1_testdata_py23
from zodbtools.test.testutil import fs1_testdata_py23
from pytest import mark, raises, xfail
# verify zodbdump output against golden
@mark.parametrize('pretty', ('raw', 'zpickledis'))
def test_zodbdump(tmpdir, zext, pretty):
tdir = dirname(__file__)
zkind = '_!zext' if zext.disabled else ''
tfs1 = fs1_testdata_py23(tmpdir, '%s/testdata/1%s.fs' % (tdir, zkind))
def test_zodbdump(tmpdir, ztestdata, pretty):
tfs1 = fs1_testdata_py23(tmpdir, '%s/data.fs' % ztestdata.prefix)
stor = FileStorage(tfs1, read_only=True)
with open('%s/testdata/1%s.zdump.%s.ok' % (tdir, zkind, pretty), 'rb') as f:
with open('%s/zdump.%s.ok' % (ztestdata.prefix, pretty), 'rb') as f:
dumpok = f.read()
out = BytesIO()
......
# -*- coding: utf-8 -*-
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,22 +24,16 @@ from zodbtools.zodbrestore import zodbrestore
from zodbtools.util import storageFromURL, readfile
from zodbtools.test.testutil import fs1_testdata_py23
from os.path import dirname
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# verify zodbrestore.
@func
def test_zodbrestore(tmpdir, zext):
zkind = '_!zext' if zext.disabled else ''
# restore from testdata/1.zdump.ok and verify it gives result that is
# bit-to-bit identical to testdata/1.fs
tdata = dirname(__file__) + "/testdata"
def test_zodbrestore(tmpdir, ztestdata):
# restore from zdump.ok and verify it gives result that is
# bit-to-bit identical to data.fs
@func
def _():
zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb')
zdump = open("%s/zdump.raw.ok" % ztestdata.prefix, 'rb')
defer(zdump.close)
stor = storageFromURL('%s/2.fs' % tmpdir)
......@@ -48,6 +42,6 @@ def test_zodbrestore(tmpdir, zext):
zodbrestore(stor, zdump)
_()
zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/1%s.fs" % (tdata, zkind)))
zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/data.fs" % ztestdata.prefix))
zfs2 = readfile("%s/2.fs" % tmpdir)
assert zfs1 == zfs2
/*.lock
/*.tmp
/*.old
*.lock
*.tmp
*.old
Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
persistent.mapping.PersistentMapping,3,648,25.038640%,216.000000,1,216,2,432
__main__.Object,65,1940,74.961360%,29.846154,9,283,56,1657
This diff is collapsed.
Class Name,T.Count,T.Bytes,Pct,AvgSize,C.Count,C.Bytes,O.Count,O.Bytes
persistent.mapping.PersistentMapping,3,648,25.038640%,216.000000,1,216,2,432
__main__.Object,65,1940,74.961360%,29.846154,9,283,56,1657
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment