Commit e825f80f authored by Kirill Smelkov's avatar Kirill Smelkov

tests: Adjust testdata FileStorage for current Python on the fly

FileStorage/py2 uses `FS21` magic in file header, whereas
FileStorage/py3 uses `FS30` magic:

    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/_compat.py#L39
    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/_compat.py#L74

And if, upon opening the database, file magic does not match to what ZODB
expects, open is rejected:

    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/FileStorage/FileStorage.py#L88
    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/FileStorage/FileStorage.py#L1625-L1630

This is done with the idea for a database, that was written from
Python2, to be rejected to be opened from Python3 and vice-versa because
strings/bytes semantics changed in between py23.

As the result, many zodbtools tests currently fail on py3 when they try
to access prepared FileStorage database in testdata, because that
database was originally prepared on py2. Here is, for example, how
test_zodbdump fails:

    ___________________________ test_zodbdump[zext-raw] ____________________________

    zext = <function zext.<locals>._ at 0x7f28530bf9d0>, pretty = 'raw'

        @mark.parametrize('pretty', ('raw', 'zpickledis'))
        def test_zodbdump(zext, pretty):
            tdir  = dirname(__file__)
            zkind = '_!zext' if zext.disabled else ''
    >       stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)

    zodbtools/test/test_dump.py:41:
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    ../ZODB/src/ZODB/FileStorage/FileStorage.py:315: in __init__
        self._pos, self._oid, tid = read_index(
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

    file = <_io.BufferedReader name='/home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs'>
    name = '/home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs'
    index = <ZODB.fsIndex.fsIndex object at 0x7f2852fee2b0>, tindex = {}
    stop = b'\xff\xff\xff\xff\xff\xff\xff\xff'
    ltid = b'\x00\x00\x00\x00\x00\x00\x00\x00', start = 4
    maxoid = b'\x00\x00\x00\x00\x00\x00\x00\x00', recover = 0, read_only = True

        def read_index(file, name, index, tindex, stop=b'\377'*8,
                       ltid=z64, start=4, maxoid=z64, recover=0, read_only=0):
            """Scan the file storage and update the index."""
            ...
            if file_size:
                if file_size < start:
                    raise FileStorageFormatError(file.name)
                seek(0)
                if read(4) != packed_version:
    >               raise FileStorageFormatError(name)
    E               ZODB.FileStorage.FileStorage.FileStorageFormatError: /home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs

    ../ZODB/src/ZODB/FileStorage/FileStorage.py:1630: FileStorageFormatError

Since zodbtools primarily work on raw data without decoding stored
pickles, unlike Zope or ERP5, it should not be a problem for zodbtools
to work on py3 with the database that was prepared on py2.

-> Adjust all tests to use FileStorage data generated on the fly based
on original files in testdata/ but with FileStorage header being
rewritten to match current python.
parent 3cb93096
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors. # Copyright (C) 2019-2022 Nexedi SA and Contributors.
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -18,14 +18,18 @@ ...@@ -18,14 +18,18 @@
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
from zodbtools.zodbanalyze import analyze, report from zodbtools.zodbanalyze import analyze, report
from zodbtools.test.testutil import fs1_testdata_py23
import os.path import os.path
def test_zodbanalyze(capsys): def test_zodbanalyze(tmpdir, capsys):
tfs1 = fs1_testdata_py23(tmpdir,
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"))
for use_dbm in (False, True): for use_dbm in (False, True):
report( report(
analyze( analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"), tfs1,
use_dbm=use_dbm, use_dbm=use_dbm,
delta_fs=False, delta_fs=False,
tidmin=None, tidmin=None,
...@@ -40,7 +44,7 @@ def test_zodbanalyze(capsys): ...@@ -40,7 +44,7 @@ def test_zodbanalyze(capsys):
# csv output # csv output
report( report(
analyze( analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"), tfs1,
use_dbm=False, use_dbm=False,
delta_fs=False, delta_fs=False,
tidmin=None, tidmin=None,
...@@ -61,7 +65,7 @@ __main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577 ...@@ -61,7 +65,7 @@ __main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
# empty range # empty range
report( report(
analyze( analyze(
os.path.join(os.path.dirname(__file__), "testdata", "1.fs"), tfs1,
use_dbm=False, use_dbm=False,
delta_fs=False, delta_fs=False,
tidmin="ffffffffffffffff", tidmin="ffffffffffffffff",
......
...@@ -30,15 +30,16 @@ from io import BytesIO ...@@ -30,15 +30,16 @@ from io import BytesIO
from os.path import dirname from os.path import dirname
from zodbtools.test.testutil import zext_supported from zodbtools.test.testutil import zext_supported, fs1_testdata_py23
from pytest import mark, raises, xfail from pytest import mark, raises, xfail
# verify zodbdump output against golden # verify zodbdump output against golden
@mark.parametrize('pretty', ('raw', 'zpickledis')) @mark.parametrize('pretty', ('raw', 'zpickledis'))
def test_zodbdump(zext, pretty): def test_zodbdump(tmpdir, zext, pretty):
tdir = dirname(__file__) tdir = dirname(__file__)
zkind = '_!zext' if zext.disabled else '' zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True) tfs1 = fs1_testdata_py23(tmpdir, '%s/testdata/1%s.fs' % (tdir, zkind))
stor = FileStorage(tfs1, read_only=True)
with open('%s/testdata/1%s.zdump.%s.ok' % (tdir, zkind, pretty), 'rb') as f: with open('%s/testdata/1%s.zdump.%s.ok' % (tdir, zkind, pretty), 'rb') as f:
dumpok = f.read() dumpok = f.read()
......
...@@ -22,6 +22,7 @@ from __future__ import print_function ...@@ -22,6 +22,7 @@ from __future__ import print_function
from zodbtools.zodbrestore import zodbrestore from zodbtools.zodbrestore import zodbrestore
from zodbtools.util import storageFromURL, readfile from zodbtools.util import storageFromURL, readfile
from zodbtools.test.testutil import fs1_testdata_py23
from os.path import dirname from os.path import dirname
from tempfile import mkdtemp from tempfile import mkdtemp
...@@ -30,9 +31,7 @@ from golang import func, defer ...@@ -30,9 +31,7 @@ from golang import func, defer
# verify zodbrestore. # verify zodbrestore.
@func @func
def test_zodbrestore(zext): def test_zodbrestore(tmpdir, zext):
tmpd = mkdtemp('', 'zodbrestore.')
defer(lambda: rmtree(tmpd))
zkind = '_!zext' if zext.disabled else '' zkind = '_!zext' if zext.disabled else ''
# restore from testdata/1.zdump.ok and verify it gives result that is # restore from testdata/1.zdump.ok and verify it gives result that is
...@@ -43,12 +42,12 @@ def test_zodbrestore(zext): ...@@ -43,12 +42,12 @@ def test_zodbrestore(zext):
zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb') zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb')
defer(zdump.close) defer(zdump.close)
stor = storageFromURL('%s/2.fs' % tmpd) stor = storageFromURL('%s/2.fs' % tmpdir)
defer(stor.close) defer(stor.close)
zodbrestore(stor, zdump) zodbrestore(stor, zdump)
_() _()
zfs1 = readfile("%s/1%s.fs" % (tdata, zkind)) zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/1%s.fs" % (tdata, zkind)))
zfs2 = readfile("%s/2.fs" % tmpd) zfs2 = readfile("%s/2.fs" % tmpdir)
assert zfs1 == zfs2 assert zfs1 == zfs2
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors. # Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -27,6 +27,10 @@ import transaction ...@@ -27,6 +27,10 @@ import transaction
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from golang import func, defer from golang import func, defer
from six import PY3
from os.path import basename
from zodbtools.util import readfile, writefile
# zext_supported checks whether ZODB supports txn.extension_bytes . # zext_supported checks whether ZODB supports txn.extension_bytes .
_zext_supported_memo = None _zext_supported_memo = None
...@@ -61,3 +65,19 @@ def _zext_supported(): ...@@ -61,3 +65,19 @@ def _zext_supported():
assert last_txn.extension == {'a': 'b'} assert last_txn.extension == {'a': 'b'}
return hasattr(last_txn, 'extension_bytes') return hasattr(last_txn, 'extension_bytes')
# fs1_testdata_py23 prepares and returns path to temprary FileStorage prepared
# from testdata with header adjusted to work on current Python.
def fs1_testdata_py23(tmpdir, path):
data = readfile(path)
index = readfile(path + ".index")
assert data[:4] == b"FS21" # FileStorage magic for Python2
if PY3:
data = b"FS30" + data[4:] # FileStorage magic for Python3
path_ = "%s/%s" % (tmpdir, basename(path))
writefile(path_, data)
writefile("%s.index" % path_, index)
return path_
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment