Commit e76af297 authored by Julien Muchembled's avatar Julien Muchembled

client: in iterator records, export data serial as stored by NEO

There is simply no way to guess data serials and instead of producing random
values, the only solution is to retrieve the values from storages.

There are still differences for data serials between FileStorage and NEO:
- NEO always resolves to original serial, to avoid any indirection
  (which slightly speeds up undo at the expense of a more complex pack code)
- NEO does not make any difference between object deletion and creation undone
  (data serial always null in storage)
It has to be decided whether NEO implementation should be changed about this.

Apart from that, conversion database back from NEO should be fixed.
testExportFileStorageBug passes and there was in fact no FileStorage bug.

Another change is that iterator does not trash the client cache anymore.
parent 441145e5
......@@ -399,11 +399,11 @@ class Application(object):
# Do not get something more recent than the last invalidation
# we got from master.
before_tid = p64(u64(self.last_tid) + 1)
result = self._loadFromStorage(oid, tid, before_tid)
data, tid, next_tid, _ = self._loadFromStorage(oid, tid, before_tid)
if not (self._loading_oid or result[2]):
result = result[0], result[1], self._loading_invalidated
result = data, tid, (next_tid if self._loading_oid or next_tid
else self._loading_invalidated), *result)
return result
......@@ -415,7 +415,7 @@ class Application(object):
packet = Packets.AskObject(oid, at_tid, before_tid)
for node, conn in self.cp.iterateForObject(oid, readable=True):
noid, tid, next_tid, compression, checksum, data \
tid, next_tid, compression, checksum, data, data_tid \
= self._askStorage(conn, packet)
except ConnectionClosed:
......@@ -425,9 +425,8 @@ class Application(object):
logging.error('wrong checksum from %s for oid %s',
conn, dump(oid))
if compression:
data = decompress(data)
return data, tid, next_tid
return (decompress(data) if compression else data,
tid, next_tid, data_tid)
raise NEOStorageCreationUndoneError(dump(oid))
# We didn't got any object from all storage node because of
# connection error
......@@ -58,10 +58,8 @@ class StorageBootstrapHandler(AnswerBaseHandler):
class StorageAnswersHandler(AnswerBaseHandler):
""" Handle all messages related to ZODB operations """
def answerObject(self, conn, oid, start_serial, end_serial,
compression, checksum, data, data_serial):, start_serial, end_serial,
compression, checksum, data))
def answerObject(self, conn, oid, *args):
def answerStoreObject(self, conn, conflicting, oid, serial):
txn_context =
......@@ -34,31 +34,25 @@ class Record(BaseStorage.DataRecord):
class Transaction(BaseStorage.TransactionRecord):
""" Transaction object yielded by the NEO iterator """
def __init__(self, app, txn, prev_serial_dict):
def __init__(self, app, txn):
super(Transaction, self).__init__(txn['id'], ' ',
txn['user_name'], txn['description'], txn['ext']) = app
self.oid_list = txn['oids']
self.prev_serial_dict = prev_serial_dict
def __iter__(self):
""" Iterate over the transaction records """
load =
load =
for oid in self.oid_list:
data, _, next_tid = load(oid, self.tid)
data, _, _, data_tid = load(oid, self.tid, None)
except NEOStorageCreationUndoneError:
data = next_tid = None
data = data_tid = None
except NEOStorageNotFoundError:
# Transactions are not updated after a pack, so their object
# will not be found in the database. Skip them.
if next_tid is None:
prev_tid = self.prev_serial_dict.pop(oid, None)
prev_tid = self.prev_serial_dict.get(oid)
self.prev_serial_dict[oid] = self.tid
yield Record(oid, self.tid, data, prev_tid)
yield Record(oid, self.tid, data, data_tid)
def __str__(self):
return 'Transaction #%s: %s %s' \
......@@ -70,13 +64,10 @@ def iterator(app, start=None, stop=None):
if start is None:
start = ZERO_TID
stop = min(stop or MAX_TID, app.lastTransaction())
# OID -> previous TID mapping
# TODO: prune old entries while walking ?
prev_serial_dict = {}
while 1:
max_tid, chunk = app.transactionLog(start, stop, CHUNK_LENGTH)
if not chunk:
break # nothing more
for txn in chunk:
yield Transaction(app, txn, prev_serial_dict)
yield Transaction(app, txn)
start = add64(max_tid, 1)
......@@ -41,7 +41,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
tid2 = self.getNextTID(tid1)
the_object = (oid, tid1, tid2, 0, '', 'DATA', None)
self.handler.answerObject(conn, *the_object)
def _getAnswerStoreObjectHandler(self, object_stored_counter_dict,
conflict_serial_dict, resolved_conflict_serial_dict):
......@@ -171,19 +171,33 @@ class ClientTests(NEOFunctionalTest):
db = ZODB.DB(storage=storage)
return (db, storage)
def __populate(self, db, tree_size=TREE_SIZE, filestorage_bug=True):
def __populate(self, db, tree_size=TREE_SIZE):
if isinstance(, FileStorage):
from base64 import b64encode as undo_tid
undo_tid = lambda x: x
def undo(tid=None):
db.undo(undo_tid(tid or db.lastTransaction()))
conn =
root = conn.root()
root['trees'] = Tree(tree_size)
if filestorage_bug:
ob = root['trees'].right
left = ob.left
del ob.left
ob._p_changed = 1
ob.left = left
ob = root['trees'].right
left = ob.left
del ob.left
ob._p_changed = 1
t2 = db.lastTransaction()
ob.left = left
t4 = db.lastTransaction()
def testImport(self):
......@@ -203,12 +217,12 @@ class ClientTests(NEOFunctionalTest):
(neo_db, neo_conn) = self.neo.getZODBConnection()
def testExport(self, filestorage_bug=False):
def testExport(self):
# create a neo storage
(neo_db, neo_conn) = self.neo.getZODBConnection()
self.__populate(neo_db, filestorage_bug=filestorage_bug)
# copy neo to data fs
dfs_db, dfs_storage = self.__getDataFS(reset=True)
......@@ -221,11 +235,6 @@ class ClientTests(NEOFunctionalTest):
def testExportFileStorageBug(self):
# currently fails due to a bug in ZODB.FileStorage
def testLockTimeout(self):
""" Hold a lock on an object to block a second transaction """
def test():
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment