Commit 4b3b9791 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'y/restore' into x/pack

* y/restore:
  zodbrestore - Tool to restore content of a ZODB database from zodbdump output
  zodbcommit: Prepare to compute current serial of an oid lazily
  zodbcommit: Don't forget to call tpc_abort on an error
  Drop support for ZODB3
  tox: Don't run tests agains ZODB+PR183 anymore
  Add way to run tests via nxdtest
parents ba4ffa31 198b8df4
# setup to run tests on Nexedi testing infrastructure.
# https://stack.nexedi.com/test_status
TestCase('pytest', ['python', '-m', 'pytest'], summaryf=PyTest.summary)
...@@ -12,4 +12,5 @@ __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932 ...@@ -12,4 +12,5 @@ __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
- `zodb cmp` - compare content of two ZODB databases bit-to-bit. - `zodb cmp` - compare content of two ZODB databases bit-to-bit.
- `zodb commit` - commit new transaction into a ZODB database. - `zodb commit` - commit new transaction into a ZODB database.
- `zodb dump` - dump content of a ZODB database. - `zodb dump` - dump content of a ZODB database.
- `zodb restore` - restore content of a ZODB database.
- `zodb info` - print general information about a ZODB database. - `zodb info` - print general information about a ZODB database.
# zodbtools | tox setup # zodbtools | tox setup
[tox] [tox]
envlist = py27-ZODB3, py{27,36,37}-ZODB{4,5,devPR183} envlist = py{27,36,37}-ZODB{4,5}
[testenv] [testenv]
deps = deps =
.[test] .[test]
# latest ZODB from ZODB3 series
ZODB3: ZODB3 >=3.10, <3.11dev
ZODB3: transaction <2.0dev
# ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
# https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
ZODB3: ZConfig <3.2.0
# latest current ZODB 4 # latest current ZODB 4
ZODB4: ZODB >=4.0, <5.0dev ZODB4: ZODB >=4.0, <5.0dev
ZODB4: ZEO >=4.0, <5.0dev ZODB4: ZEO >=4.0, <5.0dev
...@@ -20,11 +13,7 @@ deps = ...@@ -20,11 +13,7 @@ deps =
ZODB4: transaction <2.0dev ZODB4: transaction <2.0dev
# latest current ZODB 5 # latest current ZODB 5
ZODB5: ZODB >=5.0, <6.0dev ZODB5: ZODB >=5.6, <6.0dev
ZODB5: ZEO >=5.0, <6.0dev ZODB5: ZEO >=5.0, <6.0dev
# with the patches from https://github.com/zopefoundation/ZODB/pull/183
ZODBdevPR183: -e git+https://github.com/navytux/ZODB@7f5a67b3e0d2dcf2d13f18a42a82e526f6d4055c#egg=ZODB
ZODBdevPR183: ZEO >=5.0, <6.0dev
commands= {envpython} -m pytest commands= {envpython} -m pytest
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2017-2020 Nexedi SA and Contributors. # Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -121,13 +121,13 @@ def ext4subj(subj): ...@@ -121,13 +121,13 @@ def ext4subj(subj):
return ext return ext
# run_with_zodb3py2_compat(f) runs f preserving database compatibility with # run_with_zodb4py2_compat(f) runs f preserving database compatibility with
# ZODB3/py2, which cannot load pickles encoded with protocol 3. # ZODB4/py2, which generates pickles encoded with protocol < 3.
# #
# ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0: # ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0:
# https://github.com/zopefoundation/ZODB/commit/12ee41c4 # https://github.com/zopefoundation/ZODB/commit/12ee41c4
# Undo it, while we generate test database. # Undo it, while we generate test database.
def run_with_zodb3py2_compat(f): def run_with_zodb4py2_compat(f):
import ZODB.ConflictResolution import ZODB.ConflictResolution
import ZODB.Connection import ZODB.Connection
import ZODB.ExportImport import ZODB.ExportImport
...@@ -138,21 +138,21 @@ def run_with_zodb3py2_compat(f): ...@@ -138,21 +138,21 @@ def run_with_zodb3py2_compat(f):
import ZODB.serialize import ZODB.serialize
binary = getattr(ZODB.serialize, 'binary', None) binary = getattr(ZODB.serialize, 'binary', None)
_protocol = getattr(ZODB.serialize, '_protocol', None) _protocol = getattr(ZODB.serialize, '_protocol', None)
Pz3 = 2 Pz4 = 2
try: try:
ZODB.serialize.binary = bytes ZODB.serialize.binary = bytes
# XXX cannot change just ZODB._compat._protocol, because many modules # XXX cannot change just ZODB._compat._protocol, because many modules
# do `from ZODB._compat import _protocol` and just `import ZODB` # do `from ZODB._compat import _protocol` and just `import ZODB`
# imports many ZODB.X modules. In other words we cannot change # imports many ZODB.X modules. In other words we cannot change
# _protocol just in one place. # _protocol just in one place.
ZODB.ConflictResolution._protocol = Pz3 ZODB.ConflictResolution._protocol = Pz4
ZODB.Connection._protocol = Pz3 ZODB.Connection._protocol = Pz4
ZODB.ExportImport._protocol = Pz3 ZODB.ExportImport._protocol = Pz4
ZODB.FileStorage.FileStorage._protocol = Pz3 ZODB.FileStorage.FileStorage._protocol = Pz4
ZODB._compat._protocol = Pz3 ZODB._compat._protocol = Pz4
ZODB.broken._protocol = Pz3 ZODB.broken._protocol = Pz4
ZODB.fsIndex._protocol = Pz3 ZODB.fsIndex._protocol = Pz4
ZODB.serialize._protocol = Pz3 ZODB.serialize._protocol = Pz4
f() f()
finally: finally:
...@@ -172,7 +172,7 @@ def run_with_zodb3py2_compat(f): ...@@ -172,7 +172,7 @@ def run_with_zodb3py2_compat(f):
def gen_testdb(outfs_path, zext=True): def gen_testdb(outfs_path, zext=True):
def _(): def _():
_gen_testdb(outfs_path, zext) _gen_testdb(outfs_path, zext)
run_with_zodb3py2_compat(_) run_with_zodb4py2_compat(_)
def _gen_testdb(outfs_path, zext): def _gen_testdb(outfs_path, zext):
xtime_reset() xtime_reset()
......
# -*- coding: utf-8 -*-
# Copyright (C) 2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function
from zodbtools.zodbrestore import zodbrestore
from zodbtools.util import storageFromURL
from os.path import dirname
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# verify zodbrestore.
@func
def test_zodbrestore():
tmpd = mkdtemp('', 'zodbrestore.')
defer(lambda: rmtree(tmpd))
# restore from testdata/1.zdump.ok and verify it gives result that is
# bit-to-bit identical to testdata/1.fs
tdata = dirname(__file__) + "/testdata"
@func
def _():
zdump = open("%s/1.zdump.ok" % tdata, 'rb')
defer(zdump.close)
stor = storageFromURL('%s/2.fs' % tmpd)
defer(stor.close)
zodbrestore(stor, zdump)
_()
zfs1 = _readfile("%s/1.fs" % tdata)
zfs2 = _readfile("%s/2.fs" % tmpd)
assert zfs1 == zfs2
# _readfile reads file at path.
def _readfile(path): # -> data(bytes)
with open(path, 'rb') as _:
return _.read()
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2017-2020 Nexedi SA and Contributors. # Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com> # Jérome Perrin <jerome@nexedi.com>
# #
...@@ -37,7 +37,7 @@ def register_command(cmdname): ...@@ -37,7 +37,7 @@ def register_command(cmdname):
command_module = importlib.import_module('zodbtools.zodb' + cmdname) command_module = importlib.import_module('zodbtools.zodb' + cmdname)
command_dict[cmdname] = command_module command_dict[cmdname] = command_module
for _ in ('analyze', 'cmp', 'commit', 'dump', 'info', 'pack'): for _ in ('analyze', 'cmp', 'commit', 'dump', 'info', 'pack', 'restore'):
register_command(_) register_command(_)
......
# Copyright (C) 2018-2020 Nexedi SA and Contributors. # Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -41,30 +41,53 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`. ...@@ -41,30 +41,53 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
from __future__ import print_function from __future__ import print_function
from zodbtools import zodbdump from zodbtools import zodbdump
from zodbtools.util import ashex, fromhex, storageFromURL from zodbtools.util import ashex, fromhex, storageFromURL
from ZODB.interfaces import IStorageRestoreable
from ZODB.utils import p64, u64, z64 from ZODB.utils import p64, u64, z64
from ZODB.POSException import POSKeyError from ZODB.POSException import POSKeyError
from ZODB._compat import BytesIO from ZODB._compat import BytesIO
from golang import func, defer, panic from golang import func, defer, panic
import warnings
# zodbcommit commits new transaction into ZODB storage with data specified by # zodbcommit commits new transaction into ZODB storage with data specified by
# zodbdump transaction. # zodbdump transaction.
# #
# txn.tid is ignored. # txn.tid acts as a flag:
# tid of committed transaction is returned. # - with tid=0 the transaction is committed regularly.
# - with tid=!0 the transaction is recreated with exactly that tid via IStorageRestoreable.
#
# tid of created transaction is returned.
_norestoreWarned = set() # of storage class
def zodbcommit(stor, at, txn): def zodbcommit(stor, at, txn):
assert isinstance(txn, zodbdump.Transaction) assert isinstance(txn, zodbdump.Transaction)
before = p64(u64(at)+1) want_restore = (txn.tid != z64)
have_restore = IStorageRestoreable.providedBy(stor)
# warn once if stor does not implement IStorageRestoreable
if want_restore and (not have_restore):
if type(stor) not in _norestoreWarned:
warnings.warn("restore: %s does not provide IStorageRestoreable ...\n"
"\t... -> will try to emulate it on best-effort basis." %
type(stor), RuntimeWarning)
_norestoreWarned.add(type(stor))
if want_restore:
# even if stor might be not providing IStorageRestoreable and not
# supporting .restore, it can still support .tpc_begin(tid=...). An example
# of this is NEO. We anyway need to be able to specify which transaction ID
# we need to restore transaction with.
stor.tpc_begin(txn, tid=txn.tid)
else:
stor.tpc_begin(txn) stor.tpc_begin(txn)
def _():
def current_serial(oid):
return _serial_at(stor, oid, at)
for obj in txn.objv: for obj in txn.objv:
data = None # data do be committed - setup vvv data = None # data do be committed - setup vvv
copy_from = None
if isinstance(obj, zodbdump.ObjectCopy): if isinstance(obj, zodbdump.ObjectCopy):
# NEO does not support restore, and even if stor supports restore, copy_from = obj.copy_from
# going that way requires to already know tid for transaction we are
# committing. -> we just imitate copy by actually copying data and
# letting the storage deduplicate it.
data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1)) data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
elif isinstance(obj, zodbdump.ObjectDelete): elif isinstance(obj, zodbdump.ObjectDelete):
...@@ -80,37 +103,54 @@ def zodbcommit(stor, at, txn): ...@@ -80,37 +103,54 @@ def zodbcommit(stor, at, txn):
else: else:
panic('invalid object record: %r' % (obj,)) panic('invalid object record: %r' % (obj,))
# we have the data -> restore/store the object.
# now we have the data.
# find out what is oid's serial as of <before state
try:
xdata = stor.loadBefore(obj.oid, before)
except POSKeyError:
serial_prev = z64
else:
if xdata is None:
serial_prev = z64
else:
_, serial_prev, _ = xdata
# store the object.
# if it will be ConflictError - we just fail and let the caller retry. # if it will be ConflictError - we just fail and let the caller retry.
if data is None: if data is None:
stor.deleteObject(obj.oid, serial_prev, txn) stor.deleteObject(obj.oid, current_serial(obj.oid), txn)
else:
if want_restore and have_restore:
stor.restore(obj.oid, txn.tid, data, '', copy_from, txn)
else: else:
stor.store(obj.oid, serial_prev, data, '', txn) # FIXME we don't handle copy_from on commit
# NEO does not support restore, and even if stor supports restore,
# going that way requires to already know tid for transaction we are
# committing. -> we just imitate copy by actually copying data and
# letting the storage deduplicate it.
stor.store(obj.oid, current_serial(obj.oid), data, '', txn)
try:
_()
stor.tpc_vote(txn) stor.tpc_vote(txn)
except:
stor.tpc_abort(txn)
raise
# in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 and ZODB 3 it # in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 it
# does not do so. Since we still need to support ZODB 4, utilize tpc_finish # does not do so. Since we still need to support ZODB 4, utilize tpc_finish
# callback to know with which tid the transaction was committed. # callback to know with which tid the transaction was committed.
_ = [] _ = []
stor.tpc_finish(txn, lambda tid: _.append(tid)) stor.tpc_finish(txn, lambda tid: _.append(tid))
assert len(_) == 1 assert len(_) == 1
tid = _[0] tid = _[0]
if want_restore and (tid != txn.tid):
panic('restore: restored transaction has tid=%s, but requested was tid=%s' %
(ashex(tid), ashex(txn.tid)))
return tid return tid
# _serial_at returns oid's serial as of @at database state.
def _serial_at(stor, oid, at):
before = p64(u64(at)+1)
try:
xdata = stor.loadBefore(oid, before)
except POSKeyError:
serial = z64
else:
if xdata is None:
serial = z64
else:
_, serial, _ = xdata
return serial
# ---------------------------------------- # ----------------------------------------
import sys, getopt import sys, getopt
...@@ -159,7 +199,9 @@ def main(argv): ...@@ -159,7 +199,9 @@ def main(argv):
stor = storageFromURL(storurl) stor = storageFromURL(storurl)
defer(stor.close) defer(stor.close)
zin = b'txn 0000000000000000 " "\n' # artificial transaction header # artificial transaction header with tid=0 to request regular commit
zin = b'txn 0000000000000000 " "\n'
zin += sys.stdin.read() zin += sys.stdin.read()
zin = BytesIO(zin) zin = BytesIO(zin)
zr = zodbdump.DumpReader(zin) zr = zodbdump.DumpReader(zin)
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2016-2020 Nexedi SA and Contributors. # Copyright (C) 2016-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com> # Jérome Perrin <jerome@nexedi.com>
# #
...@@ -26,8 +26,8 @@ transaction prints transaction's header and information about changed objects. ...@@ -26,8 +26,8 @@ transaction prints transaction's header and information about changed objects.
The information dumped is complete raw information as stored in ZODB storage The information dumped is complete raw information as stored in ZODB storage
and should be suitable for restoring the database from the dump file bit-to-bit and should be suitable for restoring the database from the dump file bit-to-bit
identical to its original(*). It is dumped in semi text-binary format where identical to its original(*) via Zodbrestore. It is dumped in semi text-binary
object data is output as raw binary and everything else is text. format where object data is output as raw binary and everything else is text.
There is also shortened mode activated via --hashonly where only hash of object There is also shortened mode activated via --hashonly where only hash of object
data is printed without content. data is printed without content.
......
# Copyright (C) 2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Zodbrestore - Restore content of a ZODB database.
Zodbrestore reads transactions from zodbdump output and recreates them in a
ZODB storage. See Zodbdump documentation for details.
"""
from __future__ import print_function
from zodbtools.zodbdump import DumpReader
from zodbtools.zodbcommit import zodbcommit
from zodbtools.util import asbinstream, ashex, storageFromURL
from golang import func, defer
# zodbrestore restores transactions read from reader r in zodbdump format.
#
# restoredf, if !None, is called for every restored transaction.
def zodbrestore(stor, r, restoredf=None):
zr = DumpReader(r)
at = stor.lastTransaction()
while 1:
txn = zr.readtxn()
if txn is None:
break
zodbcommit(stor, at, txn)
if restoredf != None:
restoredf(txn)
at = txn.tid
# ----------------------------------------
import sys, getopt
summary = "restore content of a ZODB database"
def usage(out):
print("""\
Usage: zodb restore [OPTIONS] <storage> < input
Restore content of a ZODB database.
The transactions to restore are read from stdin in zodbdump format.
On success the ID of every restored transaction is printed to stdout.
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
Options:
-h --help show this help
""", file=out)
@func
def main(argv):
try:
optv, argv = getopt.getopt(argv[1:], "h", ["help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
if len(argv) != 1:
usage(sys.stderr)
sys.exit(2)
storurl = argv[0]
stor = storageFromURL(storurl)
defer(stor.close)
def _(txn):
print(ashex(txn.tid))
zodbrestore(stor, asbinstream(sys.stdin), _)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment