Commit d562bf8f authored by Julien Muchembled's avatar Julien Muchembled

client: clean up import/export code

Export:
- Remove leftover warning about a bug that was fixed in
  commit e76af297
- In neomigrate script, open NEO storage read-only.
- IStorageIteration is already implemented.

Import:
- Review comments.
- In neomigrate script, warn that IStorageRestoreable is not implemented.
- Do not call 'close' method on source iterator. BaseStorage does not do it and
  this is not part of ZODB API. In the case of FileStorage, resource are freed
  automatically during garbage collection.
parent fcff26db
...@@ -142,8 +142,8 @@ RC - Review output of pylint (CODE) ...@@ -142,8 +142,8 @@ RC - Review output of pylint (CODE)
- Use generic bootstrap module (CODE) - Use generic bootstrap module (CODE)
- If too many storage nodes are dead, the client should check the partition - If too many storage nodes are dead, the client should check the partition
table hasn't changed by pinging the master and retry if necessary. table hasn't changed by pinging the master and retry if necessary.
- Implement restore() ZODB API method to bypass consistency checks during - Implement IStorageRestoreable (ZODB API) in order to preserve data
imports. serials (i.e. undo information).
- tpc_finish might raise while transaction got successfully committed. - tpc_finish might raise while transaction got successfully committed.
This can happen if it gets disconnected from primary master while waiting This can happen if it gets disconnected from primary master while waiting
for AnswerFinishTransaction after primary received it and hence will for AnswerFinishTransaction after primary received it and hence will
......
...@@ -32,13 +32,8 @@ class Storage(BaseStorage.BaseStorage, ...@@ -32,13 +32,8 @@ class Storage(BaseStorage.BaseStorage,
implements( implements(
ZODB.interfaces.IStorage, ZODB.interfaces.IStorage,
# "restore" missing for the moment, but "store" implements this
# interface.
# ZODB.interfaces.IStorageRestoreable, # ZODB.interfaces.IStorageRestoreable,
# XXX: imperfect iterator implementation: ZODB.interfaces.IStorageIteration,
# - start & stop are not handled (raises if either is not None)
# - transaction isolation is not done
# ZODB.interfaces.IStorageIteration,
ZODB.interfaces.IStorageUndoable, ZODB.interfaces.IStorageUndoable,
ZODB.interfaces.IExternalGC, ZODB.interfaces.IExternalGC,
ZODB.interfaces.ReadVerifyingStorage, ZODB.interfaces.ReadVerifyingStorage,
...@@ -178,8 +173,7 @@ class Storage(BaseStorage.BaseStorage, ...@@ -178,8 +173,7 @@ class Storage(BaseStorage.BaseStorage,
def copyTransactionsFrom(self, source, verbose=False): def copyTransactionsFrom(self, source, verbose=False):
""" Zope compliant API """ """ Zope compliant API """
return self.app.importFrom(source, None, None, return self.importFrom(source)
self.tryToResolveConflict)
def importFrom(self, source, start=None, stop=None, preindex=None): def importFrom(self, source, start=None, stop=None, preindex=None):
""" Allow import only a part of the source storage """ """ Allow import only a part of the source storage """
......
...@@ -919,23 +919,25 @@ class Application(object): ...@@ -919,23 +919,25 @@ class Application(object):
def importFrom(self, source, start, stop, tryToResolveConflict, def importFrom(self, source, start, stop, tryToResolveConflict,
preindex=None): preindex=None):
# TODO: The main difference with BaseStorage implementation is that
# preindex can't be filled with the result 'store' (tid only
# known after 'tpc_finish'. This method could be dropped if we
# implemented IStorageRestoreable (a wrapper around source would
# still be required for partial import).
if preindex is None: if preindex is None:
preindex = {} preindex = {}
transaction_iter = source.iterator(start, stop) for transaction in source.iterator(start, stop):
for transaction in transaction_iter:
tid = transaction.tid tid = transaction.tid
self.tpc_begin(transaction, tid, transaction.status) self.tpc_begin(transaction, tid, transaction.status)
for r in transaction: for r in transaction:
oid = r.oid oid = r.oid
pre = preindex.get(oid) pre = preindex.get(oid)
# TODO: bypass conflict resolution, locks...
self.store(oid, pre, r.data, r.version, transaction) self.store(oid, pre, r.data, r.version, transaction)
preindex[oid] = tid preindex[oid] = tid
conflicted = self.tpc_vote(transaction, tryToResolveConflict) conflicted = self.tpc_vote(transaction, tryToResolveConflict)
assert not conflicted, conflicted assert not conflicted, conflicted
real_tid = self.tpc_finish(transaction, tryToResolveConflict) real_tid = self.tpc_finish(transaction, tryToResolveConflict)
assert real_tid == tid, (real_tid, tid) assert real_tid == tid, (real_tid, tid)
transaction_iter.close()
from .iterator import iterator from .iterator import iterator
......
...@@ -44,18 +44,17 @@ def main(args=None): ...@@ -44,18 +44,17 @@ def main(args=None):
# open storages # open storages
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
#from ZEO.ClientStorage import ClientStorage as ZEOStorage
from neo.client.Storage import Storage as NEOStorage from neo.client.Storage import Storage as NEOStorage
if os.path.exists(source): if os.path.exists(source):
print("NOTE: NEO does not implement IStorageRestoreable interface,"
" which means that undo information is not preserved: conflict"
" resolution could happen when undoing an old transaction.")
src = FileStorage(file_name=source, read_only=True) src = FileStorage(file_name=source, read_only=True)
dst = NEOStorage(master_nodes=destination, name=cluster, dst = NEOStorage(master_nodes=destination, name=cluster,
verbose=options.verbose) verbose=options.verbose)
else: else:
print("WARNING: due to a bug in FileStorage (at least up to ZODB trunk"
"@121629), output database may be corrupted if input database is"
" not packed.")
src = NEOStorage(master_nodes=source, name=cluster, src = NEOStorage(master_nodes=source, name=cluster,
verbose=options.verbose) verbose=options.verbose, read_only=True)
dst = FileStorage(file_name=destination) dst = FileStorage(file_name=destination)
# do the job # do the job
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment