Fix protocol and DB schema so that storages can handle transactions of any size

- Change protocol to use SHA1 for all checksums:
  - Use SHA1 instead of CRC32 for data checksums.
  - Use SHA1 instead of MD5 for replication.

- Change DatabaseManager API so that backends can store raw data separately from
  object metadata:
  - When processing AskStoreObject, call the backend to store the data
    immediately, instead of keeping it in RAM or in the temporary object table.
    Data is then referenced only by its checksum.
    Without such change, the storage could fail to store the transaction due to
    lack of RAM, or it could make tpc_finish step very slow.
  - Backends have to store data in a separate space, and remove entries as soon
    as they get unreferenced. So they must have an index of checksums in object
    metadata space. A new '_uncommitted_data' backend attribute keeps references
    of uncommitted data.
  - New methods: _pruneData, _storeData, storeData, unlockData
  - MySQL: change vertical partitioning of 'obj' by having data in a separate
    'data' table instead of using a shortened 'obj_short' table.
  - BTree: data is moved from '_obj' to a new '_data' btree.

- Undo is optimized so that backpointers are not required anymore to fetch data:
  - The checksum of an object is None only when creation is undone.
  - Removed DatabaseManager methods: _getObjectData, _getDataTIDFromData
  - DatabaseManager: move some code from _getDataTID to findUndoTID so that
    _getDataTID only has what's specific to backend.

- Removed because already covered by ZODB tests:
  - neo.tests.storage.testStorageDBTests.StorageDBTests.test__getDataTID
  - neo.tests.storage.testStorageDBTests.StorageDBTests.test__getDataTIDFromData

CHANGES

@@ -4,6 +4,8 @@ Change History
 0.10 (unreleased)
 -----------------
 
+- Storage was unable or slow to process large-sized transactions.
+  This required to change protocol and MySQL tables format.
 - NEO learned to store empty values (although it's useless when managed by
   a ZODB Connection).
 

neo/client/app.py

@@ -28,7 +28,8 @@ from ZODB.ConflictResolution import ResolvedSerial
 from persistent.TimeStamp import TimeStamp
 
 import neo.lib
-from neo.lib.protocol import NodeTypes, Packets, INVALID_PARTITION, ZERO_TID
+from neo.lib.protocol import NodeTypes, Packets, \
+    INVALID_PARTITION, ZERO_HASH, ZERO_TID
 from neo.lib.event import EventManager
 from neo.lib.util import makeChecksum as real_makeChecksum, dump
 from neo.lib.locking import Lock
@@ -444,7 +445,7 @@ class Application(object):
             except ConnectionClosed:
                 continue
 
-            if data or checksum:
+            if data or checksum != ZERO_HASH:
                 if checksum != makeChecksum(data):
                     neo.lib.logging.error('wrong checksum from %s for oid %s',
                               conn, dump(oid))
@@ -509,7 +510,7 @@ class Application(object):
             # an older object revision).
             compressed_data = ''
             compression = 0
-            checksum = 0
+            checksum = ZERO_HASH
         else:
             assert data_serial is None
             compression = self.compress

neo/client/handlers/storage.py

@@ -66,9 +66,6 @@ class StorageAnswersHandler(AnswerBaseHandler):
 
     def answerObject(self, conn, oid, start_serial, end_serial,
             compression, checksum, data, data_serial):
-        if data_serial is not None:
-            raise NEOStorageError, 'Storage should never send non-None ' \
-                'data_serial to clients, got %s' % (dump(data_serial), )
         self.app.setHandlerData((oid, start_serial, end_serial,
                 compression, checksum, data))
 

neo/lib/protocol.py

@@ -112,6 +112,7 @@ INVALID_TID = '\xff' * 8
 INVALID_OID = '\xff' * 8
 INVALID_PARTITION = 0xffffffff
 INVALID_ADDRESS_TYPE = socket.AF_UNSPEC
+ZERO_HASH = '\0' * 20
 ZERO_TID = '\0' * 8
 ZERO_OID = '\0' * 8
 OID_LEN = len(INVALID_OID)
@@ -527,6 +528,17 @@ class PProtocol(PStructItem):
             raise ProtocolError('protocol version mismatch')
         return (major, minor)
 
+class PChecksum(PItem):
+    """
+        A hash (SHA1)
+    """
+    def _encode(self, writer, checksum):
+        assert len(checksum) == 20, (len(checksum), checksum)
+        writer(checksum)
+
+    def _decode(self, reader):
+        return reader(20)
+
 class PUUID(PItem):
     """
         An UUID (node identifier)
@@ -561,7 +573,6 @@ class PTID(PItem):
 
 # same definition, for now
 POID = PTID
-PChecksum = PUUID # (md5 is same length as uuid)
 
 # common definitions
 
@@ -908,7 +919,7 @@ class StoreObject(Packet):
         POID('oid'),
         PTID('serial'),
         PBoolean('compression'),
-        PNumber('checksum'),
+        PChecksum('checksum'),
         PString('data'),
         PTID('data_serial'),
         PTID('tid'),
@@ -964,7 +975,7 @@ class GetObject(Packet):
         PTID('serial_start'),
         PTID('serial_end'),
         PBoolean('compression'),
-        PNumber('checksum'),
+        PChecksum('checksum'),
         PString('data'),
         PTID('data_serial'),
     )

neo/lib/util.py

@@ -18,7 +18,7 @@
 
 import re
 import socket
-from zlib import adler32
+from hashlib import sha1
 from Queue import deque
 from struct import pack, unpack
 
@@ -62,8 +62,8 @@ def bin(s):
 
 
 def makeChecksum(s):
-    """Return a 4-byte integer checksum against a string."""
-    return adler32(s) & 0xffffffff
+    """Return a 20-byte checksum against a string."""
+    return sha1(s).digest()
 
 
 def resolve(hostname):

neo/storage/handlers/__init__.py

@@ -21,7 +21,7 @@ from neo.lib.handler import EventHandler
 from neo.lib import protocol
 from neo.lib.util import dump
 from neo.lib.exception import PrimaryFailure, OperationFailure
-from neo.lib.protocol import NodeStates, NodeTypes, Packets, Errors
+from neo.lib.protocol import NodeStates, NodeTypes, Packets, Errors, ZERO_HASH
 
 class BaseMasterHandler(EventHandler):
 
@@ -97,7 +97,7 @@ class BaseClientAndStorageOperationHandler(EventHandler):
             neo.lib.logging.debug('oid = %s, serial = %s, next_serial = %s',
                           dump(oid), dump(serial), dump(next_serial))
             if checksum is None:
-                checksum = 0
+                checksum = ZERO_HASH
                 data = ''
             p = Packets.AnswerObject(oid, serial, next_serial,
                 compression, checksum, data, data_serial)

neo/storage/handlers/client.py

@@ -18,7 +18,7 @@
 import neo.lib
 from neo.lib import protocol
 from neo.lib.util import dump, makeChecksum
-from neo.lib.protocol import Packets, LockState, Errors
+from neo.lib.protocol import Packets, LockState, Errors, ZERO_HASH
 from neo.storage.handlers import BaseClientAndStorageOperationHandler
 from neo.storage.transactions import ConflictError, DelayedError
 from neo.storage.exception import AlreadyPendingError
@@ -88,7 +88,7 @@ class ClientOperationHandler(BaseClientAndStorageOperationHandler):
             compression, checksum, data, data_serial, ttid, unlock):
         # register the transaction
         self.app.tm.register(conn.getUUID(), ttid)
-        if data or checksum:
+        if data or checksum != ZERO_HASH:
             # TODO: return an appropriate error packet
             assert makeChecksum(data) == checksum
             assert data_serial is None

neo/storage/handlers/replication.py

@@ -20,7 +20,7 @@ from functools import wraps
 import neo.lib
 
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import Packets, ZERO_TID, ZERO_OID
+from neo.lib.protocol import Packets, ZERO_HASH, ZERO_TID, ZERO_OID
 from neo.lib.util import add64, u64
 
 # TODO: benchmark how different values behave
@@ -173,12 +173,14 @@ class ReplicationHandler(EventHandler):
     @checkConnectionIsReplicatorConnection
     def answerObject(self, conn, oid, serial_start,
             serial_end, compression, checksum, data, data_serial):
-        app = self.app
+        dm = self.app.dm
+        if data or checksum != ZERO_HASH:
+            dm.storeData(checksum, data, compression)
+        else:
+            checksum = None
         # Directly store the transaction.
-        obj = (oid, compression, checksum, data, data_serial)
-        app.dm.storeTransaction(serial_start, [obj], None, False)
-        del obj
-        del data
+        obj = oid, checksum, data_serial
+        dm.storeTransaction(serial_start, [obj], None, False)
 
     def _doAskCheckSerialRange(self, min_oid, min_tid, max_tid,
             length=RANGE_LENGTH):

neo/storage/handlers/storage.py

@@ -21,7 +21,10 @@ from neo.lib.protocol import Packets
 class StorageOperationHandler(BaseClientAndStorageOperationHandler):
 
     def _askObject(self, oid, serial, tid):
-        return self.app.dm.getObject(oid, serial, tid, resolve_data=False)
+        result = self.app.dm.getObject(oid, serial, tid)
+        if result and result[5]:
+            return result[:2] + (None, None, None) + result[4:]
+        return result
 
     def askLastIDs(self, conn):
         app = self.app

neo/storage/transactions.py

@@ -98,22 +98,21 @@ class Transaction(object):
         # assert self._transaction is not None
         self._transaction = (oid_list, user, desc, ext, packed)
 
-    def addObject(self, oid, compression, checksum, data, value_serial):
+    def addObject(self, oid, checksum, value_serial):
         """
             Add an object to the transaction
         """
         assert oid not in self._checked_set, dump(oid)
-        self._object_dict[oid] = (oid, compression, checksum, data,
-            value_serial)
+        self._object_dict[oid] = oid, checksum, value_serial
 
     def delObject(self, oid):
         try:
-            del self._object_dict[oid]
+            return self._object_dict.pop(oid)[1]
         except KeyError:
             self._checked_set.remove(oid)
 
     def getObject(self, oid):
-        return self._object_dict.get(oid)
+        return self._object_dict[oid]
 
     def getObjectList(self):
         return self._object_dict.values()
@@ -163,10 +162,10 @@ class TransactionManager(object):
             Return object data for given running transaction.
             Return None if not found.
         """
-        result = self._transaction_dict.get(ttid)
-        if result is not None:
-            result = result.getObject(oid)
-        return result
+        try:
+            return self._transaction_dict[ttid].getObject(oid)
+        except KeyError:
+            return None
 
     def reset(self):
         """
@@ -242,7 +241,9 @@ class TransactionManager(object):
             # drop the lock it held on this object, and drop object data for
             # consistency.
             del self._store_lock_dict[oid]
-            self._transaction_dict[ttid].delObject(oid)
+            checksum = self._transaction_dict[ttid].delObject(oid)
+            if checksum:
+                self._app.dm.pruneData((checksum,))
             # Give a chance to pending events to take that lock now.
             self._app.executeQueuedEvents()
             # Attemp to acquire lock again.
@@ -252,7 +253,7 @@ class TransactionManager(object):
         elif locking_tid == ttid:
             # If previous store was an undo, next store must be based on
             # undo target.
-            previous_serial = self._transaction_dict[ttid].getObject(oid)[4]
+            previous_serial = self._transaction_dict[ttid].getObject(oid)[2]
             if previous_serial is None:
                 # XXX: use some special serial when previous store was not
                 # an undo ? Maybe it should just not happen.
@@ -301,8 +302,11 @@ class TransactionManager(object):
         self.lockObject(ttid, serial, oid, unlock=unlock)
         # store object
         assert ttid in self, "Transaction not registered"
-        transaction = self._transaction_dict[ttid]
-        transaction.addObject(oid, compression, checksum, data, value_serial)
+        if data is None:
+            checksum = None
+        else:
+            self._app.dm.storeData(checksum, data, compression)
+        self._transaction_dict[ttid].addObject(oid, checksum, value_serial)
 
     def abort(self, ttid, even_if_locked=False):
         """
@@ -320,8 +324,13 @@ class TransactionManager(object):
         transaction = self._transaction_dict[ttid]
         has_load_lock = transaction.isLocked()
         # if the transaction is locked, ensure we can drop it
-        if not even_if_locked and has_load_lock:
-            return
+        if has_load_lock:
+            if not even_if_locked:
+                return
+        else:
+            self._app.dm.unlockData([checksum
+                for oid, checksum, value_serial in transaction.getObjectList()
+                if checksum], True)
         # unlock any object
         for oid in transaction.getLockedOIDList():
             if has_load_lock:
@@ -370,19 +379,13 @@ class TransactionManager(object):
         for oid, ttid in self._store_lock_dict.items():
             neo.lib.logging.info('    %r by %r', dump(oid), dump(ttid))
 
-    def updateObjectDataForPack(self, oid, orig_serial, new_serial,
-            getObjectData):
+    def updateObjectDataForPack(self, oid, orig_serial, new_serial, checksum):
         lock_tid = self.getLockingTID(oid)
         if lock_tid is not None:
             transaction = self._transaction_dict[lock_tid]
-            oid, compression, checksum, data, value_serial = \
-                transaction.getObject(oid)
-            if value_serial == orig_serial:
+            if transaction.getObject(oid)[2] == orig_serial:
                 if new_serial:
-                    value_serial = new_serial
+                    checksum = None
                 else:
-                    compression, checksum, data = getObjectData()
-                    value_serial = None
-                transaction.addObject(oid, compression, checksum, data,
-                    value_serial)
-
+                    self._app.dm.storeData(checksum)
+                transaction.addObject(oid, checksum, new_serial)

neo/tests/client/testStorageHandler.py

@@ -88,10 +88,6 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
         the_object = (oid, tid1, tid2, 0, '', 'DATA', None)
         self.handler.answerObject(conn, *the_object)
         self._checkHandlerData(the_object[:-1])
-        # Check handler raises on non-None data_serial.
-        the_object = (oid, tid1, tid2, 0, '', 'DATA', self.getNextTID())
-        self.assertRaises(NEOStorageError, self.handler.answerObject, conn,
-            *the_object)
 
     def _getAnswerStoreObjectHandler(self, object_stored_counter_dict,
             conflict_serial_dict, resolved_conflict_serial_dict):

neo/tests/storage/testClientHandler.py

@@ -23,9 +23,8 @@ from neo.tests import NeoUnitTestBase
 from neo.storage.app import Application
 from neo.storage.transactions import ConflictError, DelayedError
 from neo.storage.handlers.client import ClientOperationHandler
-from neo.lib.protocol import INVALID_PARTITION
-from neo.lib.protocol import INVALID_TID, INVALID_OID
-from neo.lib.protocol import Packets, LockState
+from neo.lib.protocol import INVALID_PARTITION, INVALID_TID, INVALID_OID
+from neo.lib.protocol import Packets, LockState, ZERO_HASH
 
 class StorageClientHandlerTests(NeoUnitTestBase):
 
@@ -124,7 +123,8 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         next_serial = self.getNextTID()
         oid = self.getOID(1)
         tid = self.getNextTID()
-        self.app.dm = Mock({'getObject': (serial, next_serial, 0, 0, '', None)})
+        H = "0" * 20
+        self.app.dm = Mock({'getObject': (serial, next_serial, 0, H, '', None)})
         conn = self._getConnection()
         self.assertEqual(len(self.app.event_queue), 0)
         self.operation.askObject(conn, oid=oid, serial=serial, tid=tid)
@@ -239,7 +239,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         tid = self.getNextTID()
         oid, serial, comp, checksum, data = self._getObject()
         data_tid = self.getNextTID()
-        self.operation.askStoreObject(conn, oid, serial, comp, 0,
+        self.operation.askStoreObject(conn, oid, serial, comp, ZERO_HASH,
                 '', data_tid, tid, False)
         self._checkStoreObjectCalled(tid, serial, oid, comp,
                 None, None, data_tid, False)

neo/tests/storage/testReplication.py

@@ -128,8 +128,11 @@ class ReplicationTests(NeoUnitTestBase):
             transaction = ([ZERO_OID], 'user', 'desc', '', False)
             storage.storeTransaction(makeid(tid), [], transaction, False)
         # store object history
+        H = "0" * 20
+        storage.storeData(H, '', 0)
+        storage.unlockData((H,))
         for tid, oid_list in objects.iteritems():
-            object_list = [(makeid(oid), False, 0, '', None) for oid in oid_list]
+            object_list = [(makeid(oid), H, None) for oid in oid_list]
             storage.storeTransaction(makeid(tid), object_list, None, False)
         return storage
 

neo/tests/storage/testReplicationHandler.py

@@ -268,15 +268,15 @@ class StorageReplicationHandlerTests(NeoUnitTestBase):
         serial_start = self.getNextTID()
         serial_end = self.getNextTID()
         compression = 1
-        checksum = 2
+        checksum = "0" * 20
         data = 'foo'
         data_serial = None
         ReplicationHandler(app).answerObject(conn, oid, serial_start,
             serial_end, compression, checksum, data, data_serial)
         calls = app.dm.mockGetNamedCalls('storeTransaction')
         self.assertEqual(len(calls), 1)
-        calls[0].checkArgs(serial_start, [(oid, compression, checksum, data,
-            data_serial)], None, False)
+        calls[0].checkArgs(serial_start, [(oid, checksum, data_serial)],
+            None, False)
 
     # CheckTIDRange
     def test_answerCheckTIDFullRangeIdenticalChunkWithNext(self):

neo/tests/storage/testStorageDBTests.py

@@ -121,7 +121,10 @@ class StorageDBTests(NeoUnitTestBase):
 
     def getTransaction(self, oid_list):
         transaction = (oid_list, 'user', 'desc', 'ext', False)
-        object_list = [(oid, 1, 0, '', None) for oid in oid_list]
+        H = "0" * 20
+        for _ in oid_list:
+            self.db.storeData(H, '', 1)
+        object_list = [(oid, H, None) for oid in oid_list]
         return (transaction, object_list)
 
     def checkSet(self, list1, list2):
@@ -180,9 +183,9 @@ class StorageDBTests(NeoUnitTestBase):
         oid1, = self.getOIDs(1)
         tid1, tid2 = self.getTIDs(2)
         FOUND_BUT_NOT_VISIBLE = False
-        OBJECT_T1_NO_NEXT = (tid1, None, 1, 0, '', None)
-        OBJECT_T1_NEXT = (tid1, tid2, 1, 0, '', None)
-        OBJECT_T2 = (tid2, None, 1, 0, '', None)
+        OBJECT_T1_NO_NEXT = (tid1, None, 1, "0"*20, '', None)
+        OBJECT_T1_NEXT = (tid1, tid2, 1, "0"*20, '', None)
+        OBJECT_T2 = (tid2, None, 1, "0"*20, '', None)
         txn1, objs1 = self.getTransaction([oid1])
         txn2, objs2 = self.getTransaction([oid1])
         # non-present
@@ -277,14 +280,14 @@ class StorageDBTests(NeoUnitTestBase):
         self.db.storeTransaction(tid2, objs2, txn2)
         self.db.finishTransaction(tid1)
         result = self.db.getObject(oid1)
-        self.assertEqual(result, (tid1, None, 1, 0, '', None))
+        self.assertEqual(result, (tid1, None, 1, "0"*20, '', None))
         self.assertEqual(self.db.getObject(oid2), None)
         self.assertEqual(self.db.getUnfinishedTIDList(), [tid2])
         # drop it
         self.db.dropUnfinishedData()
         self.assertEqual(self.db.getUnfinishedTIDList(), [])
         result = self.db.getObject(oid1)
-        self.assertEqual(result, (tid1, None, 1, 0, '', None))
+        self.assertEqual(result, (tid1, None, 1, "0"*20, '', None))
         self.assertEqual(self.db.getObject(oid2), None)
 
     def test_storeTransaction(self):
@@ -393,8 +396,8 @@ class StorageDBTests(NeoUnitTestBase):
         self.assertEqual(self.db.getObject(oid1, tid=tid2), None)
         self.db.deleteObject(oid2, serial=tid1)
         self.assertFalse(self.db.getObject(oid2, tid=tid1))
-        self.assertEqual(self.db.getObject(oid2, tid=tid2), (tid2, None) + \
-            objs2[1][1:])
+        self.assertEqual(self.db.getObject(oid2, tid=tid2),
+            (tid2, None, 1, "0" * 20, '', None))
 
     def test_deleteObjectsAbove(self):
         self.setNumPartitions(2)
@@ -574,138 +577,6 @@ class StorageDBTests(NeoUnitTestBase):
         result = self.db.getReplicationTIDList(ZERO_TID, MAX_TID, 1, 2, 0)
         self.checkSet(result, [tid1])
 
-    def test__getObjectData(self):
-        self.setNumPartitions(4, True)
-        db = self.db
-        tid0 = self.getNextTID()
-        tid1 = self.getNextTID()
-        tid2 = self.getNextTID()
-        tid3 = self.getNextTID()
-        assert tid0 < tid1 < tid2 < tid3
-        oid1 = self.getOID(1)
-        oid2 = self.getOID(2)
-        oid3 = self.getOID(3)
-        db.storeTransaction(
-            tid1, (
-                (oid1, 0, 0, 'foo', None),
-                (oid2, None, None, None, tid0),
-                (oid3, None, None, None, tid2),
-            ), None, temporary=False)
-        db.storeTransaction(
-            tid2, (
-                (oid1, None, None, None, tid1),
-                (oid2, None, None, None, tid1),
-                (oid3, 0, 0, 'bar', None),
-            ), None, temporary=False)
-
-        original_getObjectData = db._getObjectData
-        def _getObjectData(*args, **kw):
-            call_counter.append(1)
-            return original_getObjectData(*args, **kw)
-        db._getObjectData = _getObjectData
-
-        # NOTE: all tests are done as if values were fetched by _getObject, so
-        # there is already one indirection level.
-
-        # oid1 at tid1: data is immediately found
-        call_counter = []
-        self.assertEqual(
-            db._getObjectData(u64(oid1), u64(tid1), u64(tid3)),
-            (u64(tid1), 0, 0, 'foo'))
-        self.assertEqual(sum(call_counter), 1)
-
-        # oid2 at tid1: missing data in table, raise IndexError on next
-        # recursive call
-        call_counter = []
-        self.assertRaises(IndexError, db._getObjectData, u64(oid2), u64(tid1),
-            u64(tid3))
-        self.assertEqual(sum(call_counter), 2)
-
-        # oid3 at tid1: data_serial grater than row's tid, raise ValueError
-        # on next recursive call - even if data does exist at that tid (see
-        # "oid3 at tid2" case below)
-        call_counter = []
-        self.assertRaises(ValueError, db._getObjectData, u64(oid3), u64(tid1),
-            u64(tid3))
-        self.assertEqual(sum(call_counter), 2)
-        # Same with wrong parameters (tid0 < tid1)
-        call_counter = []
-        self.assertRaises(ValueError, db._getObjectData, u64(oid3), u64(tid1),
-            u64(tid0))
-        self.assertEqual(sum(call_counter), 1)
-        # Same with wrong parameters (tid1 == tid1)
-        call_counter = []
-        self.assertRaises(ValueError, db._getObjectData, u64(oid3), u64(tid1),
-            u64(tid1))
-        self.assertEqual(sum(call_counter), 1)
-
-        # oid1 at tid2: data is found after ons recursive call
-        call_counter = []
-        self.assertEqual(
-            db._getObjectData(u64(oid1), u64(tid2), u64(tid3)),
-            (u64(tid1), 0, 0, 'foo'))
-        self.assertEqual(sum(call_counter), 2)
-
-        # oid2 at tid2: missing data in table, raise IndexError after two
-        # recursive calls
-        call_counter = []
-        self.assertRaises(IndexError, db._getObjectData, u64(oid2), u64(tid2),
-            u64(tid3))
-        self.assertEqual(sum(call_counter), 3)
-
-        # oid3 at tid2: data is immediately found
-        call_counter = []
-        self.assertEqual(
-            db._getObjectData(u64(oid3), u64(tid2), u64(tid3)),
-            (u64(tid2), 0, 0, 'bar'))
-        self.assertEqual(sum(call_counter), 1)
-
-    def test__getDataTIDFromData(self):
-        self.setNumPartitions(4, True)
-        db = self.db
-        tid1 = self.getNextTID()
-        tid2 = self.getNextTID()
-        oid1 = self.getOID(1)
-        db.storeTransaction(
-            tid1, (
-                (oid1, 0, 0, 'foo', None),
-            ), None, temporary=False)
-        db.storeTransaction(
-            tid2, (
-                (oid1, None, None, None, tid1),
-            ), None, temporary=False)
-
-        self.assertEqual(
-            db._getDataTIDFromData(u64(oid1),
-                db._getObject(u64(oid1), tid=u64(tid1))),
-            (u64(tid1), u64(tid1)))
-        self.assertEqual(
-            db._getDataTIDFromData(u64(oid1),
-                db._getObject(u64(oid1), tid=u64(tid2))),
-            (u64(tid2), u64(tid1)))
-
-    def test__getDataTID(self):
-        self.setNumPartitions(4, True)
-        db = self.db
-        tid1 = self.getNextTID()
-        tid2 = self.getNextTID()
-        oid1 = self.getOID(1)
-        db.storeTransaction(
-            tid1, (
-                (oid1, 0, 0, 'foo', None),
-            ), None, temporary=False)
-        db.storeTransaction(
-            tid2, (
-                (oid1, None, None, None, tid1),
-            ), None, temporary=False)
-
-        self.assertEqual(
-            db._getDataTID(u64(oid1), tid=u64(tid1)),
-            (u64(tid1), u64(tid1)))
-        self.assertEqual(
-            db._getDataTID(u64(oid1), tid=u64(tid2)),
-            (u64(tid2), u64(tid1)))
-
     def test_findUndoTID(self):
         self.setNumPartitions(4, True)
         db = self.db
@@ -715,9 +586,14 @@ class StorageDBTests(NeoUnitTestBase):
         tid4 = self.getNextTID()
         tid5 = self.getNextTID()
         oid1 = self.getOID(1)
+        foo = "3" * 20
+        bar = "4" * 20
+        db.storeData(foo, 'foo', 0)
+        db.storeData(bar, 'bar', 0)
+        db.unlockData((foo, bar))
         db.storeTransaction(
             tid1, (
-                (oid1, 0, 0, 'foo', None),
+                (oid1, foo, None),
             ), None, temporary=False)
 
         # Undoing oid1 tid1, OK: tid1 is latest
@@ -730,7 +606,7 @@ class StorageDBTests(NeoUnitTestBase):
         # Store a new transaction
         db.storeTransaction(
             tid2, (
-                (oid1, 0, 0, 'bar', None),
+                (oid1, bar, None),
             ), None, temporary=False)
 
         # Undoing oid1 tid2, OK: tid2 is latest
@@ -753,13 +629,13 @@ class StorageDBTests(NeoUnitTestBase):
         # to tid1
         self.assertEqual(
             db.findUndoTID(oid1, tid5, tid4, tid1,
-                (u64(oid1), None, None, None, tid1)),
+                (u64(oid1), None, tid1)),
             (tid1, None, True))
 
         # Store a new transaction
         db.storeTransaction(
             tid3, (
-                (oid1, None, None, None, tid1),
+                (oid1, None, tid1),
             ), None, temporary=False)
 
         # Undoing oid1 tid1, OK: tid3 is latest with tid1 data

neo/tests/storage/testStorageHandler.py

@@ -97,7 +97,7 @@ class StorageStorageHandlerTests(NeoUnitTestBase):
         calls = self.app.dm.mockGetNamedCalls('getObject')
         self.assertEqual(len(self.app.event_queue), 0)
         self.assertEqual(len(calls), 1)
-        calls[0].checkArgs(oid, serial, tid, resolve_data=False)
+        calls[0].checkArgs(oid, serial, tid)
         self.checkErrorPacket(conn)
 
     def test_24_askObject3(self):
@@ -105,8 +105,9 @@ class StorageStorageHandlerTests(NeoUnitTestBase):
         tid = self.getNextTID()
         serial = self.getNextTID()
         next_serial = self.getNextTID()
+        H = "0" * 20
         # object found => answer
-        self.app.dm = Mock({'getObject': (serial, next_serial, 0, 0, '', None)})
+        self.app.dm = Mock({'getObject': (serial, next_serial, 0, H, '', None)})
         conn = self.getFakeConnection()
         self.assertEqual(len(self.app.event_queue), 0)
         self.operation.askObject(conn, oid=oid, serial=serial, tid=tid)
@@ -149,7 +150,7 @@ class StorageStorageHandlerTests(NeoUnitTestBase):
 
     def test_askCheckTIDRange(self):
         count = 1
-        tid_checksum = self.getNewUUID()
+        tid_checksum = "1" * 20
         min_tid = self.getNextTID()
         num_partitions = 4
         length = 5
@@ -173,12 +174,12 @@ class StorageStorageHandlerTests(NeoUnitTestBase):
 
     def test_askCheckSerialRange(self):
         count = 1
-        oid_checksum = self.getNewUUID()
+        oid_checksum = "2" * 20
         min_oid = self.getOID(1)
         num_partitions = 4
         length = 5
         partition = 6
-        serial_checksum = self.getNewUUID()
+        serial_checksum = "3" * 20
         min_serial = self.getNextTID()
         max_serial = self.getNextTID()
         max_oid = self.getOID(2)

neo/tests/storage/testStorageMySQLdb.py

@@ -125,23 +125,6 @@ class StorageMySQSLdbTests(StorageDBTests):
         self.assertEqual(self.db.escape('a"b'), 'a\\"b')
         self.assertEqual(self.db.escape("a'b"), "a\\'b")
 
-    def test_setup(self):
-        # XXX: this test verifies irrelevant symptoms. It should instead check that
-        # - setup, store, setup, load -> data still there
-        # - setup, store, setup(reset=True), load -> data not found
-        # Then, it should be moved to generic test class.
-
-        # create all tables
-        self.db.conn = Mock()
-        self.db.setup()
-        calls = self.db.conn.mockGetNamedCalls('query')
-        self.assertEqual(len(calls), 7)
-        # create all tables but drop them first
-        self.db.conn = Mock()
-        self.db.setup(reset=True)
-        calls = self.db.conn.mockGetNamedCalls('query')
-        self.assertEqual(len(calls), 8)
-
 del StorageDBTests
 
 if __name__ == "__main__":

neo/tests/storage/testTransactions.py

@@ -63,8 +63,8 @@ class TransactionTests(NeoUnitTestBase):
     def testObjects(self):
         txn = Transaction(self.getNewUUID(), self.getNextTID())
         oid1, oid2 = self.getOID(1), self.getOID(2)
-        object1 = (oid1, 1, '1', 'O1', None)
-        object2 = (oid2, 1, '2', 'O2', None)
+        object1 = oid1, "0" * 20, None
+        object2 = oid2, "1" * 20, None
         self.assertEqual(txn.getObjectList(), [])
         self.assertEqual(txn.getOIDList(), [])
         txn.addObject(*object1)
@@ -78,9 +78,9 @@ class TransactionTests(NeoUnitTestBase):
         oid_1 = self.getOID(1)
         oid_2 = self.getOID(2)
         txn = Transaction(self.getNewUUID(), self.getNextTID())
-        object_info = (oid_1, None, None, None, None)
+        object_info = oid_1, None, None
         txn.addObject(*object_info)
-        self.assertEqual(txn.getObject(oid_2), None)
+        self.assertRaises(KeyError, txn.getObject, oid_2)
         self.assertEqual(txn.getObject(oid_1), object_info)
 
 class TransactionManagerTests(NeoUnitTestBase):
@@ -102,12 +102,12 @@ class TransactionManagerTests(NeoUnitTestBase):
     def _storeTransactionObjects(self, tid, txn):
         for i, oid in enumerate(txn[0]):
             self.manager.storeObject(tid, None,
-                    oid, 1, str(i), '0' + str(i), None)
+                    oid, 1, '%020d' % i, '0' + str(i), None)
 
     def _getObject(self, value):
         oid = self.getOID(value)
         serial = self.getNextTID()
-        return (serial, (oid, 1, str(value), 'O' + str(value), None))
+        return (serial, (oid, 1, '%020d' % value, 'O' + str(value), None))
 
     def _checkTransactionStored(self, *args):
         calls = self.app.dm.mockGetNamedCalls('storeTransaction')
@@ -136,7 +136,10 @@ class TransactionManagerTests(NeoUnitTestBase):
         self.manager.storeObject(ttid, serial2, *object2)
         self.assertTrue(ttid in self.manager)
         self.manager.lock(ttid, tid, txn[0])
-        self._checkTransactionStored(tid, [object1, object2], txn)
+        self._checkTransactionStored(tid, [
+            (object1[0], object1[2], object1[4]),
+            (object2[0], object2[2], object2[4]),
+            ], txn)
         self.manager.unlock(ttid)
         self.assertFalse(ttid in self.manager)
         self._checkTransactionFinished(tid)
@@ -340,7 +343,7 @@ class TransactionManagerTests(NeoUnitTestBase):
         self.assertEqual(self.manager.getObjectFromTransaction(tid1, obj2[0]),
             None)
         self.assertEqual(self.manager.getObjectFromTransaction(tid1, obj1[0]),
-            obj1)
+            (obj1[0], obj1[2], obj1[4]))
 
     def test_getLockingTID(self):
         uuid = self.getNewUUID()
@@ -360,26 +363,24 @@ class TransactionManagerTests(NeoUnitTestBase):
         locking_serial = self.getNextTID()
         other_serial = self.getNextTID()
         new_serial = self.getNextTID()
-        compression = 1
-        checksum = 42
-        value = 'foo'
+        checksum = "2" * 20
         self.manager.register(uuid, locking_serial)
-        def getObjectData():
-            return (compression, checksum, value)
         # Object not known, nothing happens
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
             oid), None)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, None)
+        self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
             oid), None)
         self.manager.abort(locking_serial, even_if_locked=True)
         # Object known, but doesn't point at orig_serial, it is not updated
         self.manager.register(uuid, locking_serial)
-        self.manager.storeObject(locking_serial, ram_serial, oid, 0, 512,
+        self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20,
             'bar', None)
+        storeData = self.app.dm.mockGetNamedCalls('storeData')
+        self.assertEqual(storeData.pop(0).params, ("3" * 20, 'bar', 0))
         orig_object = self.manager.getObjectFromTransaction(locking_serial,
             oid)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, None)
+        self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
             oid), orig_object)
         self.manager.abort(locking_serial, even_if_locked=True)
@@ -389,29 +390,29 @@ class TransactionManagerTests(NeoUnitTestBase):
             None, other_serial)
         orig_object = self.manager.getObjectFromTransaction(locking_serial,
             oid)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None, None)
+        self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
             oid), orig_object)
         self.manager.abort(locking_serial, even_if_locked=True)
         # Object known and points at undone data it gets updated
-        # ...with data_serial: getObjectData must not be called
         self.manager.register(uuid, locking_serial)
         self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
             None, orig_serial)
         self.manager.updateObjectDataForPack(oid, orig_serial, new_serial,
-            None)
+            checksum)
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), (oid, None, None, None, new_serial))
+            oid), (oid, None, new_serial))
         self.manager.abort(locking_serial, even_if_locked=True)
-        # with data
+
         self.manager.register(uuid, locking_serial)
         self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
             None, orig_serial)
-        self.manager.updateObjectDataForPack(oid, orig_serial, None,
-            getObjectData)
+        self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
+        self.assertEqual(storeData.pop(0).params, (checksum,))
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
-            oid), (oid, compression, checksum, value, None))
+            oid), (oid, checksum, None))
         self.manager.abort(locking_serial, even_if_locked=True)
+        self.assertFalse(storeData)
 
 if __name__ == "__main__":
     unittest.main()

neo/tests/testProtocol.py

@@ -387,7 +387,8 @@ class ProtocolTests(NeoUnitTestBase):
         tid = self.getNextTID()
         tid2 = self.getNextTID()
         unlock = False
-        p = Packets.AskStoreObject(oid, serial, 1, 55, "to", tid2, tid, unlock)
+        H = "1" * 20
+        p = Packets.AskStoreObject(oid, serial, 1, H, "to", tid2, tid, unlock)
         poid, pserial, compression, checksum, data, ptid2, ptid, punlock = \
             p.decode()
         self.assertEqual(oid, poid)
@@ -395,7 +396,7 @@ class ProtocolTests(NeoUnitTestBase):
         self.assertEqual(tid, ptid)
         self.assertEqual(tid2, ptid2)
         self.assertEqual(compression, 1)
-        self.assertEqual(checksum, 55)
+        self.assertEqual(checksum, H)
         self.assertEqual(data, "to")
         self.assertEqual(unlock, punlock)
 
@@ -423,7 +424,8 @@ class ProtocolTests(NeoUnitTestBase):
         serial_start = self.getNextTID()
         serial_end = self.getNextTID()
         data_serial = self.getNextTID()
-        p = Packets.AnswerObject(oid, serial_start, serial_end, 1, 55, "to",
+        H = "2" * 20
+        p = Packets.AnswerObject(oid, serial_start, serial_end, 1, H, "to",
             data_serial)
         poid, pserial_start, pserial_end, compression, checksum, data, \
             pdata_serial = p.decode()
@@ -431,7 +433,7 @@ class ProtocolTests(NeoUnitTestBase):
         self.assertEqual(serial_start, pserial_start)
         self.assertEqual(serial_end, pserial_end)
         self.assertEqual(compression, 1)
-        self.assertEqual(checksum, 55)
+        self.assertEqual(checksum, H)
         self.assertEqual(data, "to")
         self.assertEqual(pdata_serial, data_serial)
 
@@ -686,7 +688,7 @@ class ProtocolTests(NeoUnitTestBase):
         min_tid = self.getNextTID()
         length = 2
         count = 1
-        tid_checksum = self.getNewUUID()
+        tid_checksum = "3" * 20
         max_tid = self.getNextTID()
         p = Packets.AnswerCheckTIDRange(min_tid, length, count, tid_checksum,
             max_tid)
@@ -717,9 +719,9 @@ class ProtocolTests(NeoUnitTestBase):
         min_serial = self.getNextTID()
         length = 2
         count = 1
-        oid_checksum = self.getNewUUID()
+        oid_checksum = "4" * 20
         max_oid = self.getOID(5)
-        tid_checksum = self.getNewUUID()
+        tid_checksum = "5" * 20
         max_serial = self.getNextTID()
         p = Packets.AnswerCheckSerialRange(min_oid, min_serial, length, count,
             oid_checksum, max_oid, tid_checksum, max_serial)

neo/tests/threaded/__init__.py

@@ -259,6 +259,10 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
         if adapter == 'BTree':
             dm._obj, dm._tobj = dm._tobj, dm._obj
             dm._trans, dm._ttrans = dm._ttrans, dm._trans
+            uncommitted_data = dm._uncommitted_data
+            for checksum, (_, _, index) in dm._data.iteritems():
+                uncommitted_data[checksum] = len(index)
+                index.clear()
         elif adapter == 'MySQL':
             q = dm.query
             dm.begin()
@@ -266,11 +270,22 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
                 q('RENAME TABLE %s to tmp' % table)
                 q('RENAME TABLE t%s to %s' % (table, table))
                 q('RENAME TABLE tmp to t%s' % table)
-            q('TRUNCATE obj_short')
             dm.commit()
         else:
             assert False
 
+    def getDataLockInfo(self):
+        adapter = self._init_args[1]['getAdapter']
+        dm = self.dm
+        if adapter == 'BTree':
+            checksum_list = dm._data
+        elif adapter == 'MySQL':
+            checksum_list = [x for x, in dm.query("SELECT hash FROM data")]
+        else:
+            assert False
+        assert set(dm._uncommitted_data).issubset(checksum_list)
+        return dict((x, dm._uncommitted_data.get(x, 0)) for x in checksum_list)
+
 class ClientApplication(Node, neo.client.app.Application):
 
     @SerializedEventManager.decorate

neo/tests/threaded/test.py

@@ -26,6 +26,7 @@ from neo.lib.connection import MTClientConnection
 from neo.lib.protocol import NodeStates, Packets, ZERO_TID
 from neo.tests.threaded import NEOCluster, NEOThreadedTest, \
     Patch, ConnectionFilter
+from neo.lib.util import makeChecksum
 from neo.client.pool import CELL_CONNECTED, CELL_GOOD
 
 class PCounter(Persistent):
@@ -43,13 +44,19 @@ class Test(NEOThreadedTest):
         try:
             cluster.start()
             storage = cluster.getZODBStorage()
-            for data in 'foo', '':
+            data_info = {}
+            for data in 'foo', '', 'foo':
+                checksum = makeChecksum(data)
                 oid = storage.new_oid()
                 txn = transaction.Transaction()
                 storage.tpc_begin(txn)
                 r1 = storage.store(oid, None, data, '', txn)
                 r2 = storage.tpc_vote(txn)
+                data_info[checksum] = 1
+                self.assertEqual(data_info, cluster.storage.getDataLockInfo())
                 serial = storage.tpc_finish(txn)
+                data_info[checksum] = 0
+                self.assertEqual(data_info, cluster.storage.getDataLockInfo())
                 self.assertEqual((data, serial), storage.load(oid, ''))
                 storage._cache.clear()
                 self.assertEqual((data, serial), storage.load(oid, ''))
@@ -57,6 +64,51 @@ class Test(NEOThreadedTest):
         finally:
             cluster.stop()
 
+    def testStorageDataLock(self):
+        cluster = NEOCluster()
+        try:
+            cluster.start()
+            storage = cluster.getZODBStorage()
+            data_info = {}
+
+            data = 'foo'
+            checksum = makeChecksum(data)
+            oid = storage.new_oid()
+            txn = transaction.Transaction()
+            storage.tpc_begin(txn)
+            r1 = storage.store(oid, None, data, '', txn)
+            r2 = storage.tpc_vote(txn)
+            tid = storage.tpc_finish(txn)
+            data_info[checksum] = 0
+            storage.sync()
+
+            txn = [transaction.Transaction() for x in xrange(3)]
+            for t in txn:
+                storage.tpc_begin(t)
+                storage.store(tid and oid or storage.new_oid(),
+                              tid, data, '', t)
+                tid = None
+            for t in txn:
+                storage.tpc_vote(t)
+            data_info[checksum] = 3
+            self.assertEqual(data_info, cluster.storage.getDataLockInfo())
+
+            storage.tpc_abort(txn[1])
+            storage.sync()
+            data_info[checksum] -= 1
+            self.assertEqual(data_info, cluster.storage.getDataLockInfo())
+
+            tid1 = storage.tpc_finish(txn[2])
+            data_info[checksum] -= 1
+            self.assertEqual(data_info, cluster.storage.getDataLockInfo())
+
+            storage.tpc_abort(txn[0])
+            storage.sync()
+            data_info[checksum] -= 1
+            self.assertEqual(data_info, cluster.storage.getDataLockInfo())
+        finally:
+            cluster.stop()
+
     def testDelayedUnlockInformation(self):
         except_list = []
         def delayUnlockInformation(conn, packet):
@@ -273,16 +325,21 @@ class Test(NEOThreadedTest):
             t, c = cluster.getTransaction()
             c.root()[0] = 'ok'
             t.commit()
+            data_info = cluster.storage.getDataLockInfo()
+            self.assertEqual(data_info.values(), [0, 0])
+            # (obj|trans) become t(obj|trans)
+            cluster.storage.switchTables()
         finally:
             cluster.stop()
         cluster.reset()
-        # XXX: (obj|trans) become t(obj|trans)
-        cluster.storage.switchTables()
+        self.assertEqual(dict.fromkeys(data_info, 1),
+                         cluster.storage.getDataLockInfo())
         try:
             cluster.start(fast_startup=fast_startup)
             t, c = cluster.getTransaction()
             # transaction should be verified and commited
             self.assertEqual(c.root()[0], 'ok')
+            self.assertEqual(data_info, cluster.storage.getDataLockInfo())
         finally:
             cluster.stop()