Merge branch 'y/restore' into xpy3-2

* y/restore:
  zodbrestore - Tool to restore content of a ZODB database from zodbdump output
  zodbcommit: Prepare to compute current serial of an oid lazily
  zodbcommit: Don't forget to call tpc_abort on an error
  Drop support for ZODB3
  tox: Don't run tests agains ZODB+PR183 anymore
  Add way to run tests via nxdtest
  tidrange: test: Fix for py3
  *: dict.keys() returns sequence, not [] on py3
  *: Pass bytes literal into BytesIO
  zodbdump: Use bytes to emit its output
  *: Zodbdump format is semi text-binary: Mark it as such + handle zdump output as binary
  *: Don't use %r to print/report lines/bytes to outside

.nxdtest

+# setup to run tests on Nexedi testing infrastructure.
+# https://stack.nexedi.com/test_status
+
+TestCase('pytest', ['python',  '-m', 'pytest'], summaryf=PyTest.summary)

README.rst

@@ -12,4 +12,5 @@ __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
 - `zodb cmp` - compare content of two ZODB databases bit-to-bit.
 - `zodb commit` - commit new transaction into a ZODB database.
 - `zodb dump` - dump content of a ZODB database.
+- `zodb restore` - restore content of a ZODB database.
 - `zodb info` - print general information about a ZODB database.

tox.ini

 # zodbtools | tox setup
 [tox]
-envlist = py27-ZODB3, py{27,36,37}-ZODB{4,5,devPR183}
+envlist = py{27,36,37}-ZODB{4,5}
 
 [testenv]
 deps	=
     .[test]
 
-    # latest ZODB from ZODB3 series
-    ZODB3:  ZODB3 >=3.10, <3.11dev
-    ZODB3:  transaction <2.0dev
-    # ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
-    # https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
-    ZODB3:  ZConfig <3.2.0
-
     # latest current ZODB 4
     ZODB4:  ZODB >=4.0, <5.0dev
     ZODB4:  ZEO  >=4.0, <5.0dev
@@ -20,11 +13,7 @@ deps	=
     ZODB4:  transaction <2.0dev
 
     # latest current ZODB 5
-    ZODB5:  ZODB >=5.0,	<6.0dev
+    ZODB5:  ZODB >=5.6,	<6.0dev
     ZODB5:  ZEO	 >=5.0, <6.0dev
 
-    # with the patches from https://github.com/zopefoundation/ZODB/pull/183
-    ZODBdevPR183: -e git+https://github.com/navytux/ZODB@7f5a67b3e0d2dcf2d13f18a42a82e526f6d4055c#egg=ZODB
-    ZODBdevPR183: ZEO >=5.0, <6.0dev
-
 commands= {envpython} -m pytest

zodbtools/test/gen_testdata.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2020  Nexedi SA and Contributors.
+# Copyright (C) 2017-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -121,13 +121,13 @@ def ext4subj(subj):
 
     return ext
 
-# run_with_zodb3py2_compat(f) runs f preserving database compatibility with
-# ZODB3/py2, which cannot load pickles encoded with protocol 3.
+# run_with_zodb4py2_compat(f) runs f preserving database compatibility with
+# ZODB4/py2, which generates pickles encoded with protocol < 3.
 #
 # ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0:
 # https://github.com/zopefoundation/ZODB/commit/12ee41c4
 # Undo it, while we generate test database.
-def run_with_zodb3py2_compat(f):
+def run_with_zodb4py2_compat(f):
     import ZODB.ConflictResolution
     import ZODB.Connection
     import ZODB.ExportImport
@@ -138,21 +138,21 @@ def run_with_zodb3py2_compat(f):
     import ZODB.serialize
     binary    = getattr(ZODB.serialize, 'binary', None)
     _protocol = getattr(ZODB.serialize, '_protocol', None)
-    Pz3 = 2
+    Pz4 = 2
     try:
         ZODB.serialize.binary    = bytes
         # XXX cannot change just ZODB._compat._protocol, because many modules
         # do `from ZODB._compat import _protocol` and just `import ZODB`
         # imports many ZODB.X modules. In other words we cannot change
         # _protocol just in one place.
-        ZODB.ConflictResolution._protocol       = Pz3
-        ZODB.Connection._protocol               = Pz3
-        ZODB.ExportImport._protocol             = Pz3
-        ZODB.FileStorage.FileStorage._protocol  = Pz3
-        ZODB._compat._protocol                  = Pz3
-        ZODB.broken._protocol                   = Pz3
-        ZODB.fsIndex._protocol                  = Pz3
-        ZODB.serialize._protocol                = Pz3
+        ZODB.ConflictResolution._protocol       = Pz4
+        ZODB.Connection._protocol               = Pz4
+        ZODB.ExportImport._protocol             = Pz4
+        ZODB.FileStorage.FileStorage._protocol  = Pz4
+        ZODB._compat._protocol                  = Pz4
+        ZODB.broken._protocol                   = Pz4
+        ZODB.fsIndex._protocol                  = Pz4
+        ZODB.serialize._protocol                = Pz4
 
         f()
     finally:
@@ -172,7 +172,7 @@ def run_with_zodb3py2_compat(f):
 def gen_testdb(outfs_path, zext=True):
     def _():
         _gen_testdb(outfs_path, zext)
-    run_with_zodb3py2_compat(_)
+    run_with_zodb4py2_compat(_)
 
 def _gen_testdb(outfs_path, zext):
     xtime_reset()

zodbtools/test/test_restore.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2021  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from __future__ import print_function
+
+from zodbtools.zodbrestore import zodbrestore
+from zodbtools.util import storageFromURL
+
+from os.path import dirname
+from tempfile import mkdtemp
+from shutil import rmtree
+from golang import func, defer
+
+# verify zodbrestore.
+@func
+def test_zodbrestore():
+    tmpd = mkdtemp('', 'zodbrestore.')
+    defer(lambda: rmtree(tmpd))
+
+    # restore from testdata/1.zdump.ok and verify it gives result that is
+    # bit-to-bit identical to testdata/1.fs
+    tdata = dirname(__file__) + "/testdata"
+    @func
+    def _():
+        zdump = open("%s/1.zdump.ok" % tdata, 'rb')
+        defer(zdump.close)
+
+        stor = storageFromURL('%s/2.fs' % tmpd)
+        defer(stor.close)
+
+        zodbrestore(stor, zdump)
+    _()
+
+    zfs1 = _readfile("%s/1.fs" % tdata)
+    zfs2 = _readfile("%s/2.fs" % tmpd)
+    assert zfs1 == zfs2
+
+
+# _readfile reads file at path.
+def _readfile(path): # -> data(bytes)
+    with open(path, 'rb') as _:
+        return _.read()

zodbtools/zodb.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2019  Nexedi SA and Contributors.
+# Copyright (C) 2017-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -37,7 +37,7 @@ def register_command(cmdname):
     command_module = importlib.import_module('zodbtools.zodb' + cmdname)
     command_dict[cmdname] = command_module
 
-for _ in ('analyze', 'cmp', 'commit', 'dump', 'info'):
+for _ in ('analyze', 'cmp', 'commit', 'dump', 'info', 'restore'):
     register_command(_)
 
 

zodbtools/zodbcommit.py

-# Copyright (C) 2018-2020  Nexedi SA and Contributors.
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -41,30 +41,53 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
 from __future__ import print_function
 from zodbtools import zodbdump
 from zodbtools.util import ashex, fromhex, storageFromURL
+from ZODB.interfaces import IStorageRestoreable
 from ZODB.utils import p64, u64, z64
 from ZODB.POSException import POSKeyError
 from ZODB._compat import BytesIO
 from golang import func, defer, panic
+import warnings
 
 
 # zodbcommit commits new transaction into ZODB storage with data specified by
 # zodbdump transaction.
 #
-# txn.tid is ignored.
-# tid of committed transaction is returned.
+# txn.tid acts as a flag:
+# - with tid=0 the transaction is committed regularly.
+# - with tid=!0 the transaction is recreated with exactly that tid via IStorageRestoreable.
+#
+# tid of created transaction is returned.
+_norestoreWarned = set() # of storage class
 def zodbcommit(stor, at, txn):
     assert isinstance(txn, zodbdump.Transaction)
 
-    before = p64(u64(at)+1)
-
+    want_restore = (txn.tid != z64)
+    have_restore = IStorageRestoreable.providedBy(stor)
+    # warn once if stor does not implement IStorageRestoreable
+    if want_restore and (not have_restore):
+        if type(stor) not in _norestoreWarned:
+            warnings.warn("restore: %s does not provide IStorageRestoreable ...\n"
+                          "\t... -> will try to emulate it on best-effort basis." %
+                          type(stor), RuntimeWarning)
+            _norestoreWarned.add(type(stor))
+
+    if want_restore:
+        # even if stor might be not providing IStorageRestoreable and not
+        # supporting .restore, it can still support .tpc_begin(tid=...). An example
+        # of this is NEO. We anyway need to be able to specify which transaction ID
+        # we need to restore transaction with.
+        stor.tpc_begin(txn, tid=txn.tid)
+    else:
         stor.tpc_begin(txn)
+
+    def _():
+        def current_serial(oid):
+            return _serial_at(stor, oid, at)
         for obj in txn.objv:
             data = None # data do be committed - setup vvv
+            copy_from = None
             if isinstance(obj, zodbdump.ObjectCopy):
-            # NEO does not support restore, and even if stor supports restore,
-            # going that way requires to already know tid for transaction we are
-            # committing. -> we just imitate copy by actually copying data and
-            # letting the storage deduplicate it.
+                copy_from = obj.copy_from
                 data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
 
             elif isinstance(obj, zodbdump.ObjectDelete):
@@ -80,37 +103,54 @@ def zodbcommit(stor, at, txn):
             else:
                 panic('invalid object record: %r' % (obj,))
 
-
-        # now we have the data.
-        # find out what is oid's serial as of <before state
-        try:
-            xdata = stor.loadBefore(obj.oid, before)
-        except POSKeyError:
-            serial_prev = z64
-        else:
-            if xdata is None:
-                serial_prev = z64
-            else:
-                _, serial_prev, _ = xdata
-
-        # store the object.
+            # we have the data -> restore/store the object.
             # if it will be ConflictError - we just fail and let the caller retry.
             if data is None:
-            stor.deleteObject(obj.oid, serial_prev, txn)
+                stor.deleteObject(obj.oid, current_serial(obj.oid), txn)
+            else:
+                if want_restore and have_restore:
+                    stor.restore(obj.oid, txn.tid, data, '', copy_from, txn)
                 else:
-            stor.store(obj.oid, serial_prev, data, '', txn)
+                    # FIXME we don't handle copy_from on commit
+                    # NEO does not support restore, and even if stor supports restore,
+                    # going that way requires to already know tid for transaction we are
+                    # committing. -> we just imitate copy by actually copying data and
+                    # letting the storage deduplicate it.
+                    stor.store(obj.oid, current_serial(obj.oid), data, '', txn)
 
+    try:
+        _()
         stor.tpc_vote(txn)
+    except:
+        stor.tpc_abort(txn)
+        raise
 
-    # in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 and ZODB 3 it
+    # in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 it
     # does not do so. Since we still need to support ZODB 4, utilize tpc_finish
     # callback to know with which tid the transaction was committed.
     _ = []
     stor.tpc_finish(txn, lambda tid: _.append(tid))
     assert len(_) == 1
     tid = _[0]
+    if want_restore and (tid != txn.tid):
+        panic('restore: restored transaction has tid=%s, but requested was tid=%s' %
+              (ashex(tid), ashex(txn.tid)))
     return tid
 
+# _serial_at returns oid's serial as of @at database state.
+def _serial_at(stor, oid, at):
+    before = p64(u64(at)+1)
+    try:
+        xdata = stor.loadBefore(oid, before)
+    except POSKeyError:
+        serial = z64
+    else:
+        if xdata is None:
+            serial = z64
+        else:
+            _, serial, _ = xdata
+    return serial
+
 
 # ----------------------------------------
 import sys, getopt
@@ -159,7 +199,9 @@ def main(argv):
     stor = storageFromURL(storurl)
     defer(stor.close)
 
-    zin = b'txn 0000000000000000 " "\n'  # artificial transaction header
+    # artificial transaction header with tid=0 to request regular commit
+    zin = b'txn 0000000000000000 " "\n'
+
     zin += sys.stdin.read()
     zin = BytesIO(zin)
     zr = zodbdump.DumpReader(zin)

zodbtools/zodbdump.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2020  Nexedi SA and Contributors.
+# Copyright (C) 2016-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #                          Jérome Perrin <jerome@nexedi.com>
 #
@@ -26,8 +26,8 @@ transaction prints transaction's header and information about changed objects.
 
 The information dumped is complete raw information as stored in ZODB storage
 and should be suitable for restoring the database from the dump file bit-to-bit
-identical to its original(*). It is dumped in semi text-binary format where
-object data is output as raw binary and everything else is text.
+identical to its original(*) via Zodbrestore. It is dumped in semi text-binary
+format where object data is output as raw binary and everything else is text.
 
 There is also shortened mode activated via --hashonly where only hash of object
 data is printed without content.

zodbtools/zodbrestore.py

+# Copyright (C) 2021  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Zodbrestore - Restore content of a ZODB database.
+
+Zodbrestore reads transactions from zodbdump output and recreates them in a
+ZODB storage. See Zodbdump documentation for details.
+"""
+
+from __future__ import print_function
+from zodbtools.zodbdump import DumpReader
+from zodbtools.zodbcommit import zodbcommit
+from zodbtools.util import asbinstream, ashex, storageFromURL
+from golang import func, defer
+
+
+# zodbrestore restores transactions read from reader r in zodbdump format.
+#
+# restoredf, if !None, is called for every restored transaction.
+def zodbrestore(stor, r, restoredf=None):
+    zr = DumpReader(r)
+    at = stor.lastTransaction()
+    while 1:
+        txn = zr.readtxn()
+        if txn is None:
+            break
+
+        zodbcommit(stor, at, txn)
+        if restoredf != None:
+            restoredf(txn)
+        at = txn.tid
+
+
+# ----------------------------------------
+import sys, getopt
+
+summary = "restore content of a ZODB database"
+
+def usage(out):
+    print("""\
+Usage: zodb restore [OPTIONS] <storage> < input
+Restore content of a ZODB database.
+
+The transactions to restore are read from stdin in zodbdump format.
+On success the ID of every restored transaction is printed to stdout.
+
+<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
+
+Options:
+
+    -h  --help      show this help
+""", file=out)
+
+@func
+def main(argv):
+    try:
+        optv, argv = getopt.getopt(argv[1:], "h", ["help"])
+    except getopt.GetoptError as e:
+        print(e, file=sys.stderr)
+        usage(sys.stderr)
+        sys.exit(2)
+
+    for opt, _ in optv:
+        if opt in ("-h", "--help"):
+            usage(sys.stdout)
+            sys.exit(0)
+
+    if len(argv) != 1:
+        usage(sys.stderr)
+        sys.exit(2)
+
+    storurl = argv[0]
+
+    stor = storageFromURL(storurl)
+    defer(stor.close)
+
+    def _(txn):
+        print(ashex(txn.tid))
+    zodbrestore(stor, asbinstream(sys.stdin), _)