storage: fix memory leak in replication

The following 3 methods are renamed:
 unlockData -> releaseData
 storeData -> holdData
 _storeData -> storeData

and StorageOperationHandler use the new storeData instead of the old one.

neo/storage/database/manager.py

@@ -308,7 +308,7 @@ class DatabaseManager(object):
         """
         raise NotImplementedError
 
-    def _storeData(self, checksum, data, compression):
+    def storeData(self, checksum, data, compression):
         """To be overriden by the backend to store object raw data
 
         If same data was already stored, the storage only has to check there's
@@ -316,24 +316,24 @@ class DatabaseManager(object):
         """
         raise NotImplementedError
 
-    def storeData(self, checksum_or_id, data=None, compression=None):
-        """Store object raw data
+    def holdData(self, checksum_or_id, data=None, compression=None):
+        """Store raw data of temporary object
 
         checksum must be the result of neo.lib.util.makeChecksum(data)
         'compression' indicates if 'data' is compressed.
-        A volatile reference is set to this data until 'unlockData' is called
+        A volatile reference is set to this data until 'releaseData' is called
         with this checksum.
         If called with only an id, it only increment the volatile
         reference to the data matching the id.
         """
         refcount = self._uncommitted_data
         if data is not None:
-            checksum_or_id = self._storeData(checksum_or_id, data, compression)
+            checksum_or_id = self.storeData(checksum_or_id, data, compression)
         refcount[checksum_or_id] = 1 + refcount.get(checksum_or_id, 0)
         return checksum_or_id
 
-    def unlockData(self, data_id_list, prune=False):
-        """Release 1 volatile reference to given list of checksums
+    def releaseData(self, data_id_list, prune=False):
+        """Release 1 volatile reference to given list of data ids
 
         If 'prune' is true, any data that is not referenced anymore (either by
         a volatile reference or by a fully-committed object) is deleted.

neo/storage/database/mysqldb.py

@@ -44,7 +44,7 @@ def splitOIDField(tid, oids):
 class MySQLDatabaseManager(DatabaseManager):
     """This class manages a database on MySQL."""
 
-    # WARNING: some parts are not concurrent safe (ex: storeData)
+    # WARNING: some parts are not concurrent safe (ex: holdData)
     # (there must be only 1 writable connection per DB)
 
     # Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because
@@ -370,7 +370,7 @@ class MySQLDatabaseManager(DatabaseManager):
         data_id_list = [x for x, in q("SELECT data_id FROM tobj") if x]
         q("TRUNCATE tobj")
         q("TRUNCATE ttrans")
-        self.unlockData(data_id_list, True)
+        self.releaseData(data_id_list, True)
 
     def storeTransaction(self, tid, object_list, transaction, temporary = True):
         e = self.escape
@@ -392,7 +392,7 @@ class MySQLDatabaseManager(DatabaseManager):
                     " WHERE partition=%d AND oid=%d AND tid=%d"
                     % (partition, oid, value_serial))
                 if temporary:
-                    self.storeData(data_id)
+                    self.holdData(data_id)
             else:
                 value_serial = 'NULL'
             q("REPLACE INTO %s VALUES (%d, %d, %d, %s, %s)" % (obj_table,
@@ -415,7 +415,7 @@ class MySQLDatabaseManager(DatabaseManager):
                 " WHERE id IN (%s) AND data_id IS NULL"
                 % ",".join(map(str, data_id_list)))
 
-    def _storeData(self, checksum, data, compression):
+    def storeData(self, checksum, data, compression):
         e = self.escape
         checksum = e(checksum)
         try:
@@ -454,7 +454,7 @@ class MySQLDatabaseManager(DatabaseManager):
         q("DELETE FROM tobj WHERE tid=%d" % tid)
         q("INSERT INTO trans SELECT * FROM ttrans WHERE tid=%d" % tid)
         q("DELETE FROM ttrans WHERE tid=%d" % tid)
-        self.unlockData(data_id_list)
+        self.releaseData(data_id_list)
         self.commit()
 
     def deleteTransaction(self, tid, oid_list=()):
@@ -464,7 +464,7 @@ class MySQLDatabaseManager(DatabaseManager):
         q = self.query
         sql = " FROM tobj WHERE tid=%d" % tid
         data_id_list = [x for x, in q("SELECT data_id" + sql) if x]
-        self.unlockData(data_id_list)
+        self.releaseData(data_id_list)
         q("DELETE" + sql)
         q("""DELETE FROM ttrans WHERE tid = %d""" % tid)
         q("""DELETE FROM trans WHERE partition = %d AND tid = %d""" %

neo/storage/database/sqlite.py

@@ -311,7 +311,7 @@ class SQLiteDatabaseManager(DatabaseManager):
         data_id_list = [x for x, in q("SELECT data_id FROM tobj") if x]
         q("DELETE FROM tobj")
         q("DELETE FROM ttrans")
-        self.unlockData(data_id_list, True)
+        self.releaseData(data_id_list, True)
 
     def storeTransaction(self, tid, object_list, transaction, temporary=True):
         u64 = util.u64
@@ -328,7 +328,7 @@ class SQLiteDatabaseManager(DatabaseManager):
                     " WHERE partition=? AND oid=? AND tid=?",
                     (partition, oid, value_serial))
                 if temporary:
-                    self.storeData(data_id)
+                    self.holdData(data_id)
             try:
                 q(obj_sql, (partition, oid, tid, data_id, value_serial))
             except sqlite3.IntegrityError:
@@ -361,7 +361,7 @@ class SQLiteDatabaseManager(DatabaseManager):
             q("DELETE FROM data WHERE id IN (%s)"
               % ",".join(map(str, data_id_list)))
 
-    def _storeData(self, checksum, data, compression,
+    def storeData(self, checksum, data, compression,
                    _dup_hash=unique_constraint_message("data", "hash")):
         H = buffer(checksum)
         try:
@@ -399,7 +399,7 @@ class SQLiteDatabaseManager(DatabaseManager):
         q("DELETE FROM tobj WHERE tid=?", args)
         q("INSERT OR FAIL INTO trans SELECT * FROM ttrans WHERE tid=?", args)
         q("DELETE FROM ttrans WHERE tid=?", args)
-        self.unlockData(data_id_list)
+        self.releaseData(data_id_list)
         self.commit()
 
     def deleteTransaction(self, tid, oid_list=()):
@@ -409,7 +409,7 @@ class SQLiteDatabaseManager(DatabaseManager):
         q = self.query
         sql = " FROM tobj WHERE tid=?"
         data_id_list = [x for x, in q("SELECT data_id" + sql, (tid,)) if x]
-        self.unlockData(data_id_list)
+        self.releaseData(data_id_list)
         q("DELETE" + sql, (tid,))
         q("DELETE FROM ttrans WHERE tid=?", (tid,))
         q("DELETE FROM trans WHERE partition=? AND tid=?",

neo/storage/transactions.py

@@ -299,7 +299,7 @@ class TransactionManager(object):
         if data is None:
             data_id = None
         else:
-            data_id = self._app.dm.storeData(checksum, data, compression)
+            data_id = self._app.dm.holdData(checksum, data, compression)
         self._transaction_dict[ttid].addObject(oid, data_id, value_serial)
 
     def abort(self, ttid, even_if_locked=False):
@@ -322,7 +322,7 @@ class TransactionManager(object):
             if not even_if_locked:
                 return
         else:
-            self._app.dm.unlockData([data_id
+            self._app.dm.releaseData([data_id
                 for oid, data_id, value_serial in transaction.getObjectList()
                 if data_id], True)
         # unlock any object
@@ -387,5 +387,5 @@ class TransactionManager(object):
                 if new_serial:
                     data_id = None
                 else:
-                    self._app.dm.storeData(data_id)
+                    self._app.dm.holdData(data_id)
                 transaction.addObject(oid, data_id, new_serial)

neo/tests/storage/testStorageDBTests.py

@@ -115,7 +115,7 @@ class StorageDBTests(NeoUnitTestBase):
         self._last_ttid = ttid = add64(self._last_ttid, 1)
         transaction = oid_list, 'user', 'desc', 'ext', False, ttid
         H = "0" * 20
-        object_list = [(oid, self.db.storeData(H, '', 1), None)
+        object_list = [(oid, self.db.holdData(H, '', 1), None)
                        for oid in oid_list]
         return (transaction, object_list)
 
@@ -528,9 +528,9 @@ class StorageDBTests(NeoUnitTestBase):
         tid4 = self.getNextTID()
         tid5 = self.getNextTID()
         oid1 = self.getOID(1)
-        foo = db.storeData("3" * 20, 'foo', 0)
-        bar = db.storeData("4" * 20, 'bar', 0)
-        db.unlockData((foo, bar))
+        foo = db.holdData("3" * 20, 'foo', 0)
+        bar = db.holdData("4" * 20, 'bar', 0)
+        db.releaseData((foo, bar))
         db.storeTransaction(
             tid1, (
                 (oid1, foo, None),

neo/tests/storage/testTransactions.py

@@ -120,7 +120,7 @@ class TransactionManagerTests(NeoUnitTestBase):
     def testSimpleCase(self):
         """ One node, one transaction, not abort """
         data_id_list = random.random(), random.random()
-        self.app.dm.mockAddReturnValues(storeData=ReturnValues(*data_id_list))
+        self.app.dm.mockAddReturnValues(holdData=ReturnValues(*data_id_list))
         uuid = self.getClientUUID()
         ttid = self.getNextTID()
         tid, txn = self._getTransaction()
@@ -328,7 +328,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def test_getObjectFromTransaction(self):
         data_id = random.random()
-        self.app.dm.mockAddReturnValues(storeData=ReturnValues(data_id))
+        self.app.dm.mockAddReturnValues(holdData=ReturnValues(data_id))
         uuid = self.getClientUUID()
         tid1, txn1 = self._getTransaction()
         tid2, txn2 = self._getTransaction()
@@ -374,8 +374,8 @@ class TransactionManagerTests(NeoUnitTestBase):
         self.manager.register(uuid, locking_serial)
         self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20,
             'bar', None)
-        storeData = self.app.dm.mockGetNamedCalls('storeData')
-        self.assertEqual(storeData.pop(0).params, ("3" * 20, 'bar', 0))
+        holdData = self.app.dm.mockGetNamedCalls('holdData')
+        self.assertEqual(holdData.pop(0).params, ("3" * 20, 'bar', 0))
         orig_object = self.manager.getObjectFromTransaction(locking_serial,
             oid)
         self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
@@ -406,11 +406,11 @@ class TransactionManagerTests(NeoUnitTestBase):
         self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
             None, orig_serial)
         self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
-        self.assertEqual(storeData.pop(0).params, (checksum,))
+        self.assertEqual(holdData.pop(0).params, (checksum,))
         self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
             oid), (oid, checksum, None))
         self.manager.abort(locking_serial, even_if_locked=True)
-        self.assertFalse(storeData)
+        self.assertFalse(holdData)
 
 if __name__ == "__main__":
     unittest.main()

neo/tests/threaded/testReplication.py

@@ -68,6 +68,7 @@ class ReplicationTests(NEOThreadedTest):
         checked = 0
         source_dict = {x.uuid: x for x in cluster.upstream.storage_list}
         for storage in cluster.storage_list:
+            self.assertFalse(storage.dm._uncommitted_data)
             self.assertEqual(np, storage.pt.getPartitions())
             for partition in pt.getAssignedPartitionList(storage.uuid):
                 cell_list = upstream_pt.getCellList(partition, readable=True)