...
 
Commits (11)
  • Kirill Smelkov's avatar
    . · 054e25d5
    Kirill Smelkov authored
    054e25d5
  • Kirill Smelkov's avatar
    . · badb3fee
    Kirill Smelkov authored
    badb3fee
  • Kirill Smelkov's avatar
    . · e9a36412
    Kirill Smelkov authored
    e9a36412
  • Kirill Smelkov's avatar
    . · 735ab157
    Kirill Smelkov authored
    735ab157
  • Kirill Smelkov's avatar
    . · 4f53d246
    Kirill Smelkov authored
    4f53d246
  • Kirill Smelkov's avatar
    . · b81bf3f5
    Kirill Smelkov authored
    b81bf3f5
  • Kirill Smelkov's avatar
    . · ba4ffa31
    Kirill Smelkov authored
    ba4ffa31
  • Kirill Smelkov's avatar
    zodbcommit: Don't forget to call tpc_abort on an error · 1844c7db
    Kirill Smelkov authored
    Two-phase commit protocol assumes that after tpc_begin, it will be
    either successful tpc_vote + tpc_finish, or tpc_abort. We were not
    calling tpc_abort on an error, potentially leaving storage in "commit is
    in progress" state on an error.
    1844c7db
  • Kirill Smelkov's avatar
    zodbcommit: Prepare to compute current serial of an oid lazily · e5fb6e7c
    Kirill Smelkov authored
    This current serial will not be needed on new codepaths to be added to
    zodbcommit in the next patch.
    
    -> Move the computation to function to trigger it only from places where
    knowing current serial is actually needed.
    e5fb6e7c
  • Kirill Smelkov's avatar
    zodbrestore - Tool to restore content of a ZODB database from zodbdump output · 198b8df4
    Kirill Smelkov authored
    Zodbrestore is long-coming counterpart to zodbdump.
    Implementation is internally based on reworked zodbcommit.
    
    For FileStorage restored database is verified via test to be bit-to-bit
    identical to the original.
    
    For NEO it won't be exactly the case, as NEO does not implement
    IStorageRestoreable: there is only tpc_begin(tid=...) but no restore().
    
    /helped-by @jerome
    198b8df4
  • Kirill Smelkov's avatar
    Merge branch 'y/restore' into x/pack · 4b3b9791
    Kirill Smelkov authored
    * y/restore:
      zodbrestore - Tool to restore content of a ZODB database from zodbdump output
      zodbcommit: Prepare to compute current serial of an oid lazily
      zodbcommit: Don't forget to call tpc_abort on an error
      Drop support for ZODB3
      tox: Don't run tests agains ZODB+PR183 anymore
      Add way to run tests via nxdtest
    4b3b9791
......@@ -12,4 +12,5 @@ __ https://github.com/zopefoundation/ZODB/pull/128#issuecomment-260970932
- `zodb cmp` - compare content of two ZODB databases bit-to-bit.
- `zodb commit` - commit new transaction into a ZODB database.
- `zodb dump` - dump content of a ZODB database.
- `zodb restore` - restore content of a ZODB database.
- `zodb info` - print general information about a ZODB database.
# -*- coding: utf-8 -*-
# Copyright (C) 2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function
from zodbtools.zodbrestore import zodbrestore
from zodbtools.util import storageFromURL
from os.path import dirname
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# verify zodbrestore.
@func
def test_zodbrestore():
tmpd = mkdtemp('', 'zodbrestore.')
defer(lambda: rmtree(tmpd))
# restore from testdata/1.zdump.ok and verify it gives result that is
# bit-to-bit identical to testdata/1.fs
tdata = dirname(__file__) + "/testdata"
@func
def _():
zdump = open("%s/1.zdump.ok" % tdata, 'rb')
defer(zdump.close)
stor = storageFromURL('%s/2.fs' % tmpd)
defer(stor.close)
zodbrestore(stor, zdump)
_()
zfs1 = _readfile("%s/1.fs" % tdata)
zfs2 = _readfile("%s/2.fs" % tmpd)
assert zfs1 == zfs2
# _readfile reads file at path.
def _readfile(path): # -> data(bytes)
with open(path, 'rb') as _:
return _.read()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -37,7 +37,7 @@ def register_command(cmdname):
command_module = importlib.import_module('zodbtools.zodb' + cmdname)
command_dict[cmdname] = command_module
for _ in ('analyze', 'cmp', 'commit', 'dump', 'info'):
for _ in ('analyze', 'cmp', 'commit', 'dump', 'info', 'pack', 'restore'):
register_command(_)
......
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -41,66 +41,89 @@ can query current database head (last_tid) with `zodb info <stor> last_tid`.
from __future__ import print_function
from zodbtools import zodbdump
from zodbtools.util import ashex, fromhex, storageFromURL
from ZODB.interfaces import IStorageRestoreable
from ZODB.utils import p64, u64, z64
from ZODB.POSException import POSKeyError
from ZODB._compat import BytesIO
from golang import func, defer, panic
import warnings
# zodbcommit commits new transaction into ZODB storage with data specified by
# zodbdump transaction.
#
# txn.tid is ignored.
# tid of committed transaction is returned.
# txn.tid acts as a flag:
# - with tid=0 the transaction is committed regularly.
# - with tid=!0 the transaction is recreated with exactly that tid via IStorageRestoreable.
#
# tid of created transaction is returned.
_norestoreWarned = set() # of storage class
def zodbcommit(stor, at, txn):
assert isinstance(txn, zodbdump.Transaction)
before = p64(u64(at)+1)
stor.tpc_begin(txn)
for obj in txn.objv:
data = None # data do be committed - setup vvv
if isinstance(obj, zodbdump.ObjectCopy):
# NEO does not support restore, and even if stor supports restore,
# going that way requires to already know tid for transaction we are
# committing. -> we just imitate copy by actually copying data and
# letting the storage deduplicate it.
data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
elif isinstance(obj, zodbdump.ObjectDelete):
data = None
elif isinstance(obj, zodbdump.ObjectData):
if isinstance(obj.data, zodbdump.HashOnly):
raise ValueError('cannot commit transaction with hashonly object')
data = obj.data
else:
panic('invalid object record: %r' % (obj,))
want_restore = (txn.tid != z64)
have_restore = IStorageRestoreable.providedBy(stor)
# warn once if stor does not implement IStorageRestoreable
if want_restore and (not have_restore):
if type(stor) not in _norestoreWarned:
warnings.warn("restore: %s does not provide IStorageRestoreable ...\n"
"\t... -> will try to emulate it on best-effort basis." %
type(stor), RuntimeWarning)
_norestoreWarned.add(type(stor))
if want_restore:
# even if stor might be not providing IStorageRestoreable and not
# supporting .restore, it can still support .tpc_begin(tid=...). An example
# of this is NEO. We anyway need to be able to specify which transaction ID
# we need to restore transaction with.
stor.tpc_begin(txn, tid=txn.tid)
else:
stor.tpc_begin(txn)
def _():
def current_serial(oid):
return _serial_at(stor, oid, at)
for obj in txn.objv:
data = None # data do be committed - setup vvv
copy_from = None
if isinstance(obj, zodbdump.ObjectCopy):
copy_from = obj.copy_from
data, _, _ = stor.loadBefore(obj.oid, p64(u64(obj.copy_from)+1))
elif isinstance(obj, zodbdump.ObjectDelete):
data = None
elif isinstance(obj, zodbdump.ObjectData):
if isinstance(obj.data, zodbdump.HashOnly):
raise ValueError('cannot commit transaction with hashonly object')
data = obj.data
# now we have the data.
# find out what is oid's serial as of <before state
try:
xdata = stor.loadBefore(obj.oid, before)
except POSKeyError:
serial_prev = z64
else:
if xdata is None:
serial_prev = z64
else:
_, serial_prev, _ = xdata
panic('invalid object record: %r' % (obj,))
# store the object.
# if it will be ConflictError - we just fail and let the caller retry.
if data is None:
stor.deleteObject(obj.oid, serial_prev, txn)
else:
stor.store(obj.oid, serial_prev, data, '', txn)
# we have the data -> restore/store the object.
# if it will be ConflictError - we just fail and let the caller retry.
if data is None:
stor.deleteObject(obj.oid, current_serial(obj.oid), txn)
else:
if want_restore and have_restore:
stor.restore(obj.oid, txn.tid, data, '', copy_from, txn)
else:
# FIXME we don't handle copy_from on commit
# NEO does not support restore, and even if stor supports restore,
# going that way requires to already know tid for transaction we are
# committing. -> we just imitate copy by actually copying data and
# letting the storage deduplicate it.
stor.store(obj.oid, current_serial(obj.oid), data, '', txn)
stor.tpc_vote(txn)
try:
_()
stor.tpc_vote(txn)
except:
stor.tpc_abort(txn)
raise
# in ZODB >= 5 tpc_finish returns tid directly, but on ZODB 4 it
# does not do so. Since we still need to support ZODB 4, utilize tpc_finish
......@@ -109,8 +132,25 @@ def zodbcommit(stor, at, txn):
stor.tpc_finish(txn, lambda tid: _.append(tid))
assert len(_) == 1
tid = _[0]
if want_restore and (tid != txn.tid):
panic('restore: restored transaction has tid=%s, but requested was tid=%s' %
(ashex(tid), ashex(txn.tid)))
return tid
# _serial_at returns oid's serial as of @at database state.
def _serial_at(stor, oid, at):
before = p64(u64(at)+1)
try:
xdata = stor.loadBefore(oid, before)
except POSKeyError:
serial = z64
else:
if xdata is None:
serial = z64
else:
_, serial, _ = xdata
return serial
# ----------------------------------------
import sys, getopt
......@@ -159,7 +199,9 @@ def main(argv):
stor = storageFromURL(storurl)
defer(stor.close)
zin = b'txn 0000000000000000 " "\n' # artificial transaction header
# artificial transaction header with tid=0 to request regular commit
zin = b'txn 0000000000000000 " "\n'
zin += sys.stdin.read()
zin = BytesIO(zin)
zr = zodbdump.DumpReader(zin)
......
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2020 Nexedi SA and Contributors.
# Copyright (C) 2016-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -26,8 +26,8 @@ transaction prints transaction's header and information about changed objects.
The information dumped is complete raw information as stored in ZODB storage
and should be suitable for restoring the database from the dump file bit-to-bit
identical to its original(*). It is dumped in semi text-binary format where
object data is output as raw binary and everything else is text.
identical to its original(*) via Zodbrestore. It is dumped in semi text-binary
format where object data is output as raw binary and everything else is text.
There is also shortened mode activated via --hashonly where only hash of object
data is printed without content.
......
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Zodbpack - Pack a ZODB database"""
from __future__ import print_function
from zodbtools.util import storageFromURL, parse_tid, ashex
import ZODB.serialize
from ZODB.utils import u64
from persistent.TimeStamp import TimeStamp
import sys, time
from golang import func, defer, panic
# XXX try to avoid
import numpy as np
# XXX
def zodbpack(stor, packat, gc):
kw = {}
if gc is not None:
kw['gc'] = gc
# packat(tid) -> t(float) (IStorage.pack requres float packtime)
# the conversion is not exact due to potential rounding.
# we are ok to go a bit before, but not after - so that if packat specifies
# transaction exactly, this transaction is not pruned and stays in [packat range.
# FIXME pack(packtime) prunes transaction specified by packtime, so maybe it should be (packat not to prune
t = TimeStamp(packat).timeTime()
while 1:
packat_ = TimeStamp(*time.gmtime(t)[:5]+(t%60,)).raw()
d = u64(packat_) - u64(packat)
if abs(d) > 1E2:
panic("too much divergence due to rounding:"
"\npackat: %s\npackat_: %s\nδ: %d" % (ashex(packat), ashex(packat_), d))
#print("\npackat: %s\npackat_: %s\nδ: %d" % (ashex(packat), ashex(packat_), d))
if d <= 0:
break
t = np.nextafter(t, -np.inf)
stor.pack(t, ZODB.serialize.referencesf, **kw)
# ----------------------------------------
import getopt
summary = "pack a ZODB database"
def usage(out):
print("""\
Usage: zodb pack [OPTIONS] <storage> <packat>
Pack a ZODB database.
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
XXX
Options:
--nogc don't perform garbage-collection
-h --help show this help
""", file=out)
@func
def main(argv):
gc = True
try:
optv, argv = getopt.getopt(argv[1:], "h", ["nogc", "help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
if opt in ("--nogc"):
gc = False
if len(argv) != 2:
usage(sys.stderr)
sys.exit(2)
storurl = argv[0]
packat = parse_tid(argv[1])
stor = storageFromURL(storurl)
defer(stor.close)
zodbpack(stor, packat, gc)
# Copyright (C) 2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Zodbrestore - Restore content of a ZODB database.
Zodbrestore reads transactions from zodbdump output and recreates them in a
ZODB storage. See Zodbdump documentation for details.
"""
from __future__ import print_function
from zodbtools.zodbdump import DumpReader
from zodbtools.zodbcommit import zodbcommit
from zodbtools.util import asbinstream, ashex, storageFromURL
from golang import func, defer
# zodbrestore restores transactions read from reader r in zodbdump format.
#
# restoredf, if !None, is called for every restored transaction.
def zodbrestore(stor, r, restoredf=None):
zr = DumpReader(r)
at = stor.lastTransaction()
while 1:
txn = zr.readtxn()
if txn is None:
break
zodbcommit(stor, at, txn)
if restoredf != None:
restoredf(txn)
at = txn.tid
# ----------------------------------------
import sys, getopt
summary = "restore content of a ZODB database"
def usage(out):
print("""\
Usage: zodb restore [OPTIONS] <storage> < input
Restore content of a ZODB database.
The transactions to restore are read from stdin in zodbdump format.
On success the ID of every restored transaction is printed to stdout.
<storage> is an URL (see 'zodb help zurl') of a ZODB-storage.
Options:
-h --help show this help
""", file=out)
@func
def main(argv):
try:
optv, argv = getopt.getopt(argv[1:], "h", ["help"])
except getopt.GetoptError as e:
print(e, file=sys.stderr)
usage(sys.stderr)
sys.exit(2)
for opt, _ in optv:
if opt in ("-h", "--help"):
usage(sys.stdout)
sys.exit(0)
if len(argv) != 1:
usage(sys.stderr)
sys.exit(2)
storurl = argv[0]
stor = storageFromURL(storurl)
defer(stor.close)
def _(txn):
print(ashex(txn.tid))
zodbrestore(stor, asbinstream(sys.stdin), _)