Skip to content

N
neoppod
Commit a0bd2ae8 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

storage: fix memory leak in replication 

The following 3 methods are renamed:
 unlockData -> releaseData
 storeData -> holdData
 _storeData -> storeData

and StorageOperationHandler use the new storeData instead of the old one.
parent f1b72dfe
Show whitespace changes
Inline Side-by-side
Showing with 32 additions and 31 deletions
neo/storage/database/manager.py
View file @ a0bd2ae8
......@@ -308,7 +308,7 @@ class DatabaseManager(object):
"""
raise NotImplementedError
def _storeData(self, checksum, data, compression):
def storeData(self, checksum, data, compression):
"""To be overriden by the backend to store object raw data
If same data was already stored, the storage only has to check there's
......@@ -316,24 +316,24 @@ class DatabaseManager(object):
"""
raise NotImplementedError
def storeData(self, checksum_or_id, data=None, compression=None):
"""Store object raw data
def holdData(self, checksum_or_id, data=None, compression=None):
"""Store raw data of temporary object
checksum must be the result of neo.lib.util.makeChecksum(data)
'compression' indicates if 'data' is compressed.
A volatile reference is set to this data until 'unlockData' is called
A volatile reference is set to this data until 'releaseData' is called
with this checksum.
If called with only an id, it only increment the volatile
reference to the data matching the id.
"""
refcount = self._uncommitted_data
if data is not None:
checksum_or_id = self._storeData(checksum_or_id, data, compression)
checksum_or_id = self.storeData(checksum_or_id, data, compression)
refcount[checksum_or_id] = 1 + refcount.get(checksum_or_id, 0)
return checksum_or_id
def unlockData(self, data_id_list, prune=False):
"""Release 1 volatile reference to given list of checksums
def releaseData(self, data_id_list, prune=False):
"""Release 1 volatile reference to given list of data ids
If 'prune' is true, any data that is not referenced anymore (either by
a volatile reference or by a fully-committed object) is deleted.
......
neo/storage/database/mysqldb.py
View file @ a0bd2ae8
......@@ -44,7 +44,7 @@ def splitOIDField(tid, oids):
class MySQLDatabaseManager(DatabaseManager):
"""This class manages a database on MySQL."""
# WARNING: some parts are not concurrent safe (ex: storeData)
# WARNING: some parts are not concurrent safe (ex: holdData)
# (there must be only 1 writable connection per DB)
# Disabled even on MySQL 5.1-5.5 and MariaDB 5.2-5.3 because
......@@ -370,7 +370,7 @@ class MySQLDatabaseManager(DatabaseManager):
data_id_list = [x for x, in q("SELECT data_id FROM tobj") if x]
q("TRUNCATE tobj")
q("TRUNCATE ttrans")
self.unlockData(data_id_list, True)
self.releaseData(data_id_list, True)
def storeTransaction(self, tid, object_list, transaction, temporary = True):
e = self.escape
......@@ -392,7 +392,7 @@ class MySQLDatabaseManager(DatabaseManager):
" WHERE partition=%d AND oid=%d AND tid=%d"
% (partition, oid, value_serial))
if temporary:
self.storeData(data_id)
self.holdData(data_id)
else:
value_serial = 'NULL'
q("REPLACE INTO %s VALUES (%d, %d, %d, %s, %s)" % (obj_table,
......@@ -415,7 +415,7 @@ class MySQLDatabaseManager(DatabaseManager):
" WHERE id IN (%s) AND data_id IS NULL"
% ",".join(map(str, data_id_list)))
def _storeData(self, checksum, data, compression):
def storeData(self, checksum, data, compression):
e = self.escape
checksum = e(checksum)
try:
......@@ -454,7 +454,7 @@ class MySQLDatabaseManager(DatabaseManager):
q("DELETE FROM tobj WHERE tid=%d" % tid)
q("INSERT INTO trans SELECT * FROM ttrans WHERE tid=%d" % tid)
q("DELETE FROM ttrans WHERE tid=%d" % tid)
self.unlockData(data_id_list)
self.releaseData(data_id_list)
self.commit()
def deleteTransaction(self, tid, oid_list=()):
......@@ -464,7 +464,7 @@ class MySQLDatabaseManager(DatabaseManager):
q = self.query
sql = " FROM tobj WHERE tid=%d" % tid
data_id_list = [x for x, in q("SELECT data_id" + sql) if x]
self.unlockData(data_id_list)
self.releaseData(data_id_list)
q("DELETE" + sql)
q("""DELETE FROM ttrans WHERE tid = %d""" % tid)
q("""DELETE FROM trans WHERE partition = %d AND tid = %d""" %
......
neo/storage/database/sqlite.py
View file @ a0bd2ae8
......@@ -311,7 +311,7 @@ class SQLiteDatabaseManager(DatabaseManager):
data_id_list = [x for x, in q("SELECT data_id FROM tobj") if x]
q("DELETE FROM tobj")
q("DELETE FROM ttrans")
self.unlockData(data_id_list, True)
self.releaseData(data_id_list, True)
def storeTransaction(self, tid, object_list, transaction, temporary=True):
u64 = util.u64
......@@ -328,7 +328,7 @@ class SQLiteDatabaseManager(DatabaseManager):
" WHERE partition=? AND oid=? AND tid=?",
(partition, oid, value_serial))
if temporary:
self.storeData(data_id)
self.holdData(data_id)
try:
q(obj_sql, (partition, oid, tid, data_id, value_serial))
except sqlite3.IntegrityError:
......@@ -361,7 +361,7 @@ class SQLiteDatabaseManager(DatabaseManager):
q("DELETE FROM data WHERE id IN (%s)"
% ",".join(map(str, data_id_list)))
def _storeData(self, checksum, data, compression,
def storeData(self, checksum, data, compression,
_dup_hash=unique_constraint_message("data", "hash")):
H = buffer(checksum)
try:
......@@ -399,7 +399,7 @@ class SQLiteDatabaseManager(DatabaseManager):
q("DELETE FROM tobj WHERE tid=?", args)
q("INSERT OR FAIL INTO trans SELECT * FROM ttrans WHERE tid=?", args)
q("DELETE FROM ttrans WHERE tid=?", args)
self.unlockData(data_id_list)
self.releaseData(data_id_list)
self.commit()
def deleteTransaction(self, tid, oid_list=()):
......@@ -409,7 +409,7 @@ class SQLiteDatabaseManager(DatabaseManager):
q = self.query
sql = " FROM tobj WHERE tid=?"
data_id_list = [x for x, in q("SELECT data_id" + sql, (tid,)) if x]
self.unlockData(data_id_list)
self.releaseData(data_id_list)
q("DELETE" + sql, (tid,))
q("DELETE FROM ttrans WHERE tid=?", (tid,))
q("DELETE FROM trans WHERE partition=? AND tid=?",
......
neo/storage/transactions.py
View file @ a0bd2ae8
......@@ -299,7 +299,7 @@ class TransactionManager(object):
if data is None:
data_id = None
else:
data_id = self._app.dm.storeData(checksum, data, compression)
data_id = self._app.dm.holdData(checksum, data, compression)
self._transaction_dict[ttid].addObject(oid, data_id, value_serial)
def abort(self, ttid, even_if_locked=False):
......@@ -322,7 +322,7 @@ class TransactionManager(object):
if not even_if_locked:
return
else:
self._app.dm.unlockData([data_id
self._app.dm.releaseData([data_id
for oid, data_id, value_serial in transaction.getObjectList()
if data_id], True)
# unlock any object
......@@ -387,5 +387,5 @@ class TransactionManager(object):
if new_serial:
data_id = None
else:
self._app.dm.storeData(data_id)
self._app.dm.holdData(data_id)
transaction.addObject(oid, data_id, new_serial)
neo/tests/storage/testStorageDBTests.py
View file @ a0bd2ae8
......@@ -115,7 +115,7 @@ class StorageDBTests(NeoUnitTestBase):
self._last_ttid = ttid = add64(self._last_ttid, 1)
transaction = oid_list, 'user', 'desc', 'ext', False, ttid
H = "0" * 20
object_list = [(oid, self.db.storeData(H, '', 1), None)
object_list = [(oid, self.db.holdData(H, '', 1), None)
for oid in oid_list]
return (transaction, object_list)
......@@ -528,9 +528,9 @@ class StorageDBTests(NeoUnitTestBase):
tid4 = self.getNextTID()
tid5 = self.getNextTID()
oid1 = self.getOID(1)
foo = db.storeData("3" * 20, 'foo', 0)
bar = db.storeData("4" * 20, 'bar', 0)
db.unlockData((foo, bar))
foo = db.holdData("3" * 20, 'foo', 0)
bar = db.holdData("4" * 20, 'bar', 0)
db.releaseData((foo, bar))
db.storeTransaction(
tid1, (
(oid1, foo, None),
......
neo/tests/storage/testTransactions.py
View file @ a0bd2ae8
......@@ -120,7 +120,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def testSimpleCase(self):
""" One node, one transaction, not abort """
data_id_list = random.random(), random.random()
self.app.dm.mockAddReturnValues(storeData=ReturnValues(*data_id_list))
self.app.dm.mockAddReturnValues(holdData=ReturnValues(*data_id_list))
uuid = self.getClientUUID()
ttid = self.getNextTID()
tid, txn = self._getTransaction()
......@@ -328,7 +328,7 @@ class TransactionManagerTests(NeoUnitTestBase):
def test_getObjectFromTransaction(self):
data_id = random.random()
self.app.dm.mockAddReturnValues(storeData=ReturnValues(data_id))
self.app.dm.mockAddReturnValues(holdData=ReturnValues(data_id))
uuid = self.getClientUUID()
tid1, txn1 = self._getTransaction()
tid2, txn2 = self._getTransaction()
......@@ -374,8 +374,8 @@ class TransactionManagerTests(NeoUnitTestBase):
self.manager.register(uuid, locking_serial)
self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20,
'bar', None)
storeData = self.app.dm.mockGetNamedCalls('storeData')
self.assertEqual(storeData.pop(0).params, ("3" * 20, 'bar', 0))
holdData = self.app.dm.mockGetNamedCalls('holdData')
self.assertEqual(holdData.pop(0).params, ("3" * 20, 'bar', 0))
orig_object = self.manager.getObjectFromTransaction(locking_serial,
oid)
self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
......@@ -406,11 +406,11 @@ class TransactionManagerTests(NeoUnitTestBase):
self.manager.storeObject(locking_serial, ram_serial, oid, None, None,
None, orig_serial)
self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum)
self.assertEqual(storeData.pop(0).params, (checksum,))
self.assertEqual(holdData.pop(0).params, (checksum,))
self.assertEqual(self.manager.getObjectFromTransaction(locking_serial,
oid), (oid, checksum, None))
self.manager.abort(locking_serial, even_if_locked=True)
self.assertFalse(storeData)
self.assertFalse(holdData)
if __name__ == "__main__":
unittest.main()
neo/tests/threaded/testReplication.py
View file @ a0bd2ae8
......@@ -68,6 +68,7 @@ class ReplicationTests(NEOThreadedTest):
checked = 0
source_dict = {x.uuid: x for x in cluster.upstream.storage_list}
for storage in cluster.storage_list:
self.assertFalse(storage.dm._uncommitted_data)
self.assertEqual(np, storage.pt.getPartitions())
for partition in pt.getAssignedPartitionList(storage.uuid):
cell_list = upstream_pt.getCellList(partition, readable=True)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7