Commit 52c5d862 by Julien Muchembled

storage: fix severe performance issue by committing backend only at key moments

1 parent 4741e38e
......@@ -15,7 +15,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from neo.lib import logging, util
from neo.lib.exception import DatabaseFailure
from neo.lib.protocol import ZERO_TID
class CreationUndone(Exception):
......@@ -28,7 +27,6 @@ class DatabaseManager(object):
"""
Initialize the object.
"""
self._under_transaction = False
self._wait = wait
self._parse(database)
......@@ -50,34 +48,9 @@ class DatabaseManager(object):
"""
raise NotImplementedError
def __enter__(self):
"""
Begin a transaction
"""
if self._under_transaction:
raise DatabaseFailure('A transaction has already begun')
r = self.begin()
self._under_transaction = True
return r
def __exit__(self, exc_type, exc_value, tb):
if not self._under_transaction:
raise DatabaseFailure('The transaction has not begun')
self._under_transaction = False
if exc_type is None:
self.commit()
else:
self.rollback()
def begin(self):
pass
def commit(self):
pass
def rollback(self):
pass
def _getPartition(self, oid_or_tid):
return oid_or_tid % self.getNumPartitions()
......@@ -91,11 +64,8 @@ class DatabaseManager(object):
"""
Set a configuration value
"""
if self._under_transaction:
self._setConfiguration(key, value)
else:
with self:
self._setConfiguration(key, value)
self._setConfiguration(key, value)
self.commit()
def _setConfiguration(self, key, value):
raise NotImplementedError
......@@ -344,8 +314,8 @@ class DatabaseManager(object):
else:
del refcount[data_id]
if prune:
with self:
self._pruneData(data_id_list)
self._pruneData(data_id_list)
self.commit()
__getDataTID = set()
def _getDataTID(self, oid, tid=None, before_tid=None):
......@@ -465,11 +435,11 @@ class DatabaseManager(object):
def truncate(self, tid):
assert tid not in (None, ZERO_TID), tid
with self:
assert self.getBackupTID()
self.setBackupTID(tid)
for partition in xrange(self.getNumPartitions()):
self._deleteRange(partition, tid)
assert self.getBackupTID()
self.setBackupTID(tid)
for partition in xrange(self.getNumPartitions()):
self._deleteRange(partition, tid)
self.commit()
def getTransaction(self, tid, all = False):
"""Return a tuple of the list of OIDs, user information,
......
......@@ -98,6 +98,7 @@ class StorageOperationHandler(EventHandler):
for serial, oid_list in object_dict.iteritems():
for oid in oid_list:
deleteObject(oid, serial)
self.app.dm.commit()
assert not pack_tid, "TODO"
if next_tid:
self.app.replicator.fetchObjects(next_tid, next_oid)
......
......@@ -20,7 +20,6 @@ from mock import Mock
from neo.lib.util import add64, dump, p64, u64
from neo.lib.protocol import CellStates, ZERO_HASH, ZERO_OID, ZERO_TID, MAX_TID
from .. import NeoUnitTestBase
from neo.lib.exception import DatabaseFailure
class StorageDBTests(NeoUnitTestBase):
......@@ -93,29 +92,6 @@ class StorageDBTests(NeoUnitTestBase):
db = self.getDB()
self.checkConfigEntry(db.getPTID, db.setPTID, self.getPTID(1))
def test_transaction(self):
db = self.getDB()
x = []
class DB(db.__class__):
begin = lambda self: x.append('begin')
commit = lambda self: x.append('commit')
rollback = lambda self: x.append('rollback')
db.__class__ = DB
with db:
self.assertEqual(x.pop(), 'begin')
self.assertEqual(x.pop(), 'commit')
try:
with db:
self.assertEqual(x.pop(), 'begin')
with db:
self.fail()
self.fail()
except DatabaseFailure:
pass
self.assertEqual(x.pop(), 'rollback')
self.assertRaises(DatabaseFailure, db.__exit__, None, None, None)
self.assertFalse(x)
def test_getPartitionTable(self):
db = self.getDB()
ptid = self.getPTID(1)
......
......@@ -300,11 +300,11 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
pass
def switchTables(self):
with self.dm as q:
for table in ('trans', 'obj'):
q('ALTER TABLE %s RENAME TO tmp' % table)
q('ALTER TABLE t%s RENAME TO %s' % (table, table))
q('ALTER TABLE tmp RENAME TO t%s' % table)
q = self.dm.query
for table in 'trans', 'obj':
q('ALTER TABLE %s RENAME TO tmp' % table)
q('ALTER TABLE t%s RENAME TO %s' % (table, table))
q('ALTER TABLE tmp RENAME TO t%s' % table)
def getDataLockInfo(self):
dm = self.dm
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!