Commit 0accf110 authored by Julien Muchembled's avatar Julien Muchembled

storage: backend optimizations

Transaction manager uses a new getLastObjectTID method instead of
getObjectHistory, because the latter fetches the data and this is
a waste of time here.

Similarly, minimize SQL work in getObject if the object can't be found.
parent edb1b866
...@@ -207,6 +207,12 @@ class DatabaseManager(object): ...@@ -207,6 +207,12 @@ class DatabaseManager(object):
searched from unfinished transactions as well.""" searched from unfinished transactions as well."""
raise NotImplementedError raise NotImplementedError
@fallback
def getLastObjectTID(self, oid):
"""Return the latest tid of given oid or None if it does not exist"""
r = self.getObject(oid)
return r and r[0]
def _getObject(self, oid, tid=None, before_tid=None): def _getObject(self, oid, tid=None, before_tid=None):
""" """
oid (int) oid (int)
...@@ -241,27 +247,17 @@ class DatabaseManager(object): ...@@ -241,27 +247,17 @@ class DatabaseManager(object):
- data_serial (packed, None) - data_serial (packed, None)
""" """
u64 = util.u64 u64 = util.u64
p64 = util.p64 r = self._getObject(u64(oid), tid and u64(tid),
oid = u64(oid) before_tid and u64(before_tid))
if tid is not None: try:
tid = u64(tid) serial, next_serial, compression, checksum, data, data_serial = r
if before_tid is not None: except TypeError:
before_tid = u64(before_tid)
result = self._getObject(oid, tid, before_tid)
if result:
serial, next_serial, compression, checksum, data, data_serial = \
result
assert before_tid is None or next_serial is None or \
before_tid <= next_serial
if serial is not None:
serial = p64(serial)
if next_serial is not None:
next_serial = p64(next_serial)
if data_serial is not None:
data_serial = p64(data_serial)
return serial, next_serial, compression, checksum, data, data_serial
# See if object exists at all # See if object exists at all
return self._getObject(oid) and False return (tid or before_tid) and self.getLastObjectTID(oid) and False
return (util.p64(serial),
None if next_serial is None else util.p64(next_serial),
compression, checksum, data,
None if data_serial is None else util.p64(data_serial))
def changePartitionTable(self, ptid, cell_list, reset=False): def changePartitionTable(self, ptid, cell_list, reset=False):
"""Change a part of a partition table. The list of cells is """Change a part of a partition table. The list of cells is
...@@ -460,7 +456,7 @@ class DatabaseManager(object): ...@@ -460,7 +456,7 @@ class DatabaseManager(object):
area as well.""" area as well."""
raise NotImplementedError raise NotImplementedError
def getObjectHistory(self, oid, offset = 0, length = 1): def getObjectHistory(self, oid, offset, length):
"""Return a list of serials and sizes for a given object ID. """Return a list of serials and sizes for a given object ID.
The length specifies the maximum size of such a list. Result starts The length specifies the maximum size of such a list. Result starts
with latest serial, and the list must be sorted in descending order. with latest serial, and the list must be sorted in descending order.
......
...@@ -282,6 +282,14 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -282,6 +282,14 @@ class MySQLDatabaseManager(DatabaseManager):
% (self._getPartition(oid), oid, tid)) or all and \ % (self._getPartition(oid), oid, tid)) or all and \
q("SELECT 1 FROM tobj WHERE tid=%d AND oid=%d" % (tid, oid)) q("SELECT 1 FROM tobj WHERE tid=%d AND oid=%d" % (tid, oid))
def getLastObjectTID(self, oid):
oid = util.u64(oid)
r = self.query("SELECT tid FROM obj"
" WHERE partition=%d AND oid=%d"
" ORDER BY tid DESC LIMIT 1"
% (self._getPartition(oid), oid))
return util.p64(r[0][0]) if r else None
def _getObject(self, oid, tid=None, before_tid=None): def _getObject(self, oid, tid=None, before_tid=None):
q = self.query q = self.query
partition = self._getPartition(oid) partition = self._getPartition(oid)
...@@ -511,7 +519,7 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -511,7 +519,7 @@ class MySQLDatabaseManager(DatabaseManager):
oid_list = splitOIDField(tid, oids) oid_list = splitOIDField(tid, oids)
return oid_list, user, desc, ext, bool(packed), util.p64(ttid) return oid_list, user, desc, ext, bool(packed), util.p64(ttid)
def getObjectHistory(self, oid, offset = 0, length = 1): def getObjectHistory(self, oid, offset, length):
# FIXME: This method doesn't take client's current ransaction id as # FIXME: This method doesn't take client's current ransaction id as
# parameter, which means it can return transactions in the future of # parameter, which means it can return transactions in the future of
# client's transaction. # client's transaction.
......
...@@ -244,6 +244,14 @@ class SQLiteDatabaseManager(DatabaseManager): ...@@ -244,6 +244,14 @@ class SQLiteDatabaseManager(DatabaseManager):
q("SELECT 1 FROM tobj WHERE tid=? AND oid=?", q("SELECT 1 FROM tobj WHERE tid=? AND oid=?",
(tid, oid)).fetchone() (tid, oid)).fetchone()
def getLastObjectTID(self, oid):
oid = util.u64(oid)
r = self.query("SELECT tid FROM obj"
" WHERE partition=? AND oid=?"
" ORDER BY tid DESC LIMIT 1",
(self._getPartition(oid), oid)).fetchone()
return r and util.p64(r[0])
def _getObject(self, oid, tid=None, before_tid=None): def _getObject(self, oid, tid=None, before_tid=None):
q = self.query q = self.query
partition = self._getPartition(oid) partition = self._getPartition(oid)
...@@ -461,7 +469,7 @@ class SQLiteDatabaseManager(DatabaseManager): ...@@ -461,7 +469,7 @@ class SQLiteDatabaseManager(DatabaseManager):
return splitOIDField(tid, oids), str(user), \ return splitOIDField(tid, oids), str(user), \
str(description), str(ext), packed, util.p64(ttid) str(description), str(ext), packed, util.p64(ttid)
def getObjectHistory(self, oid, offset=0, length=1): def getObjectHistory(self, oid, offset, length):
# FIXME: This method doesn't take client's current transaction id as # FIXME: This method doesn't take client's current transaction id as
# parameter, which means it can return transactions in the future of # parameter, which means it can return transactions in the future of
# client's transaction. # client's transaction.
......
...@@ -274,9 +274,7 @@ class TransactionManager(object): ...@@ -274,9 +274,7 @@ class TransactionManager(object):
dump(oid), dump(ttid), dump(locking_tid)) dump(oid), dump(ttid), dump(locking_tid))
raise ConflictError(ZERO_TID) raise ConflictError(ZERO_TID)
if previous_serial is None: if previous_serial is None:
history_list = self._app.dm.getObjectHistory(oid) previous_serial = self._app.dm.getLastObjectTID(oid)
if history_list:
previous_serial = history_list[0][0]
if previous_serial is not None and previous_serial != serial: if previous_serial is not None and previous_serial != serial:
logging.info('Resolvable conflict on %r:%r', logging.info('Resolvable conflict on %r:%r',
dump(oid), dump(ttid)) dump(oid), dump(ttid))
......
...@@ -189,7 +189,7 @@ class TransactionManagerTests(NeoUnitTestBase): ...@@ -189,7 +189,7 @@ class TransactionManagerTests(NeoUnitTestBase):
serial, obj = self._getObject(1) serial, obj = self._getObject(1)
next_serial = self.getNextTID(serial) next_serial = self.getNextTID(serial)
# try to store without the last revision # try to store without the last revision
self.app.dm = Mock({'getObjectHistory': [next_serial]}) self.app.dm = Mock({'getLastObjectTID': next_serial})
self.manager.register(uuid, tid) self.manager.register(uuid, tid)
self.manager.storeTransaction(tid, *txn) self.manager.storeTransaction(tid, *txn)
self.assertRaises(ConflictError, self.manager.storeObject, self.assertRaises(ConflictError, self.manager.storeObject,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment