Commit 6aa372d9 authored by Julien Muchembled's avatar Julien Muchembled

client: many fixes to 'transactionLog'

- Do not fetch data from outdated/discarded cells.
- Do not return more than transactions than requested by 'limit' parameter.
  Anyway, all results above this 'limit' could contain holes.
parent d9adfcbc
...@@ -19,6 +19,7 @@ from cPickle import dumps, loads ...@@ -19,6 +19,7 @@ from cPickle import dumps, loads
from zlib import compress as real_compress, decompress from zlib import compress as real_compress, decompress
from neo.lib.locking import Empty from neo.lib.locking import Empty
from random import shuffle from random import shuffle
import heapq
import time import time
import os import os
...@@ -946,30 +947,28 @@ class Application(object): ...@@ -946,30 +947,28 @@ class Application(object):
return undo_info return undo_info
def transactionLog(self, start, stop, limit): def transactionLog(self, start, stop, limit):
node_map = self.pt.getNodeMap() tid_list = []
node_list = node_map.keys()
node_list.sort(key=self.cp.getCellSortKey)
partition_set = set(range(self.pt.getPartitions()))
queue = self._getThreadQueue()
tid_set = set()
# request a tid list for each partition # request a tid list for each partition
for node in node_list: for offset in xrange(self.pt.getPartitions()):
conn = self.cp.getConnForNode(node) p = Packets.AskTIDsFrom(start, stop, limit, [offset])
request_set = set(node_map[node]) & partition_set for node, conn in self.cp.iterateForObject(offset, readable=True):
if conn is None or not request_set: try:
continue r = self._askStorage(conn, p)
partition_set -= set(request_set)
packet = Packets.AskTIDsFrom(start, stop, limit, request_set)
conn.ask(packet, queue=queue, tid_set=tid_set)
if not partition_set:
break break
assert not partition_set except ConnectionClosed:
self.waitResponses(queue) pass
else:
raise NEOStorageError('transactionLog failed')
if r:
tid_list = list(heapq.merge(tid_list, r))
if len(tid_list) >= limit:
del tid_list[limit:]
stop = tid_list[-1]
# request transactions informations # request transactions informations
txn_list = [] txn_list = []
append = txn_list.append append = txn_list.append
tid = None tid = None
for tid in sorted(tid_set): for tid in tid_list:
(txn_info, txn_ext) = self._getTransactionInformation(tid) (txn_info, txn_ext) = self._getTransactionInformation(tid)
txn_info['ext'] = loads(txn_ext) txn_info['ext'] = loads(txn_ext)
append(txn_info) append(txn_info)
......
...@@ -100,10 +100,9 @@ class StorageAnswersHandler(AnswerBaseHandler): ...@@ -100,10 +100,9 @@ class StorageAnswersHandler(AnswerBaseHandler):
def answerStoreTransaction(self, conn, _): def answerStoreTransaction(self, conn, _):
pass pass
def answerTIDsFrom(self, conn, tid_list, tid_set): def answerTIDsFrom(self, conn, tid_list):
neo.lib.logging.debug('Get %d TIDs from %r', len(tid_list), conn) neo.lib.logging.debug('Get %d TIDs from %r', len(tid_list), conn)
assert not tid_set.intersection(tid_list) self.app.setHandlerData(tid_list)
tid_set.update(tid_list)
def answerTransactionInformation(self, conn, tid, def answerTransactionInformation(self, conn, tid,
user, desc, ext, packed, oid_list): user, desc, ext, packed, oid_list):
......
...@@ -126,7 +126,9 @@ class ConnectionPool(object): ...@@ -126,7 +126,9 @@ class ConnectionPool(object):
def iterateForObject(self, object_id, readable=False, writable=False): def iterateForObject(self, object_id, readable=False, writable=False):
""" Iterate over nodes managing an object """ """ Iterate over nodes managing an object """
pt = self.app.getPartitionTable() pt = self.app.getPartitionTable()
cell_list = pt.getCellListForOID(object_id, readable, writable) if type(object_id) is str:
object_id = pt.getPartition(object_id)
cell_list = pt.getCellList(object_id, readable, writable)
if not cell_list: if not cell_list:
raise NEOStorageError('no storage available') raise NEOStorageError('no storage available')
getConnForNode = self.getConnForNode getConnForNode = self.getConnForNode
......
...@@ -140,12 +140,6 @@ class PartitionTable(object): ...@@ -140,12 +140,6 @@ class PartitionTable(object):
except (TypeError, KeyError): except (TypeError, KeyError):
return [] return []
def getCellListForTID(self, tid, readable=False, writable=False):
return self.getCellList(self.getPartition(tid), readable, writable)
def getCellListForOID(self, oid, readable=False, writable=False):
return self.getCellList(self.getPartition(oid), readable, writable)
def getPartition(self, oid_or_tid): def getPartition(self, oid_or_tid):
return u64(oid_or_tid) % self.getPartitions() return u64(oid_or_tid) % self.getPartitions()
...@@ -326,14 +320,6 @@ class PartitionTable(object): ...@@ -326,14 +320,6 @@ class PartitionTable(object):
getRow = self.getRow getRow = self.getRow
return [(x, getRow(x)) for x in xrange(self.np)] return [(x, getRow(x)) for x in xrange(self.np)]
def getNodeMap(self):
""" Return a list of 2-tuple: (uuid, partition_list) """
uuid_map = {}
for index, row in enumerate(self.partition_list):
for cell in row:
uuid_map.setdefault(cell.getNode(), []).append(index)
return uuid_map
def thread_safe(method): def thread_safe(method):
def wrapper(self, *args, **kwargs): def wrapper(self, *args, **kwargs):
self.lock() self.lock()
...@@ -358,14 +344,6 @@ class MTPartitionTable(PartitionTable): ...@@ -358,14 +344,6 @@ class MTPartitionTable(PartitionTable):
def unlock(self): def unlock(self):
self._lock.release() self._lock.release()
@thread_safe
def getCellListForTID(self, *args, **kwargs):
return PartitionTable.getCellListForTID(self, *args, **kwargs)
@thread_safe
def getCellListForOID(self, *args, **kwargs):
return PartitionTable.getCellListForOID(self, *args, **kwargs)
@thread_safe @thread_safe
def setCell(self, *args, **kwargs): def setCell(self, *args, **kwargs):
return PartitionTable.setCell(self, *args, **kwargs) return PartitionTable.setCell(self, *args, **kwargs)
...@@ -381,8 +359,3 @@ class MTPartitionTable(PartitionTable): ...@@ -381,8 +359,3 @@ class MTPartitionTable(PartitionTable):
@thread_safe @thread_safe
def getNodeList(self, *args, **kwargs): def getNodeList(self, *args, **kwargs):
return PartitionTable.getNodeList(self, *args, **kwargs) return PartitionTable.getNodeList(self, *args, **kwargs)
@thread_safe
def getNodeMap(self, *args, **kwargs):
return PartitionTable.getNodeMap(self, *args, **kwargs)
...@@ -41,10 +41,7 @@ def _getMasterConnection(self): ...@@ -41,10 +41,7 @@ def _getMasterConnection(self):
self.uuid = 'C' * 16 self.uuid = 'C' * 16
self.num_partitions = 10 self.num_partitions = 10
self.num_replicas = 1 self.num_replicas = 1
self.pt = Mock({ self.pt = Mock({'getCellList': ()})
'getCellListForOID': (),
'getCellListForTID': (),
})
self.master_conn = Mock() self.master_conn = Mock()
return self.master_conn return self.master_conn
...@@ -293,13 +290,12 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -293,13 +290,12 @@ class ClientApplicationTests(NeoUnitTestBase):
None, txn) None, txn)
# check partition_id and an empty cell list -> NEOStorageError # check partition_id and an empty cell list -> NEOStorageError
self._begin(app, txn, self.makeTID()) self._begin(app, txn, self.makeTID())
app.pt = Mock({ 'getCellListForOID': (), }) app.pt = Mock({'getCellList': ()})
app.num_partitions = 2 app.num_partitions = 2
self.assertRaises(NEOStorageError, app.store, oid, tid, '', None, self.assertRaises(NEOStorageError, app.store, oid, tid, '', None,
txn) txn)
calls = app.pt.mockGetNamedCalls('getCellListForOID') calls = app.pt.mockGetNamedCalls('getCellList')
self.assertEqual(len(calls), 1) self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].getParam(0), oid) # oid=11
def test_store2(self): def test_store2(self):
app = self.getApp() app = self.getApp()
...@@ -312,7 +308,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -312,7 +308,7 @@ class ClientApplicationTests(NeoUnitTestBase):
packet.setId(0) packet.setId(0)
storage_address = ('127.0.0.1', 10020) storage_address = ('127.0.0.1', 10020)
node, cell, conn = self.getNodeCellConn(address=storage_address) node, cell, conn = self.getNodeCellConn(address=storage_address)
app.pt = Mock({ 'getCellListForOID': (cell, cell)}) app.pt = Mock()
app.cp = self.getConnectionPool([(node, conn)]) app.cp = self.getConnectionPool([(node, conn)])
app.dispatcher = Dispatcher() app.dispatcher = Dispatcher()
app.nm.createStorage(address=storage_address) app.nm.createStorage(address=storage_address)
...@@ -341,7 +337,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -341,7 +337,7 @@ class ClientApplicationTests(NeoUnitTestBase):
node, cell, conn = self.getNodeCellConn(address=storage_address, node, cell, conn = self.getNodeCellConn(address=storage_address,
uuid=uuid) uuid=uuid)
app.cp = self.getConnectionPool([(node, conn)]) app.cp = self.getConnectionPool([(node, conn)])
app.pt = Mock({ 'getCellListForOID': (cell, cell, ) }) app.pt = Mock()
app.dispatcher = Dispatcher() app.dispatcher = Dispatcher()
app.nm.createStorage(address=storage_address) app.nm.createStorage(address=storage_address)
app.store(oid, tid, 'DATA', None, txn) app.store(oid, tid, 'DATA', None, txn)
...@@ -391,7 +387,6 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -391,7 +387,6 @@ class ClientApplicationTests(NeoUnitTestBase):
app.master_conn = Mock() app.master_conn = Mock()
conn = Mock() conn = Mock()
cell = Mock() cell = Mock()
app.pt = Mock({'getCellListForTID': (cell, cell)})
app.cp = Mock({'getConnForCell': ReturnValues(None, cell)}) app.cp = Mock({'getConnForCell': ReturnValues(None, cell)})
app.tpc_abort(txn) app.tpc_abort(txn)
# no packet sent # no packet sent
...@@ -456,14 +451,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -456,14 +451,7 @@ class ClientApplicationTests(NeoUnitTestBase):
node1 = Mock({'__repr__': 'node1', '__hash__': 1, 'getConnection': conn1}) node1 = Mock({'__repr__': 'node1', '__hash__': 1, 'getConnection': conn1})
node2 = Mock({'__repr__': 'node2', '__hash__': 2, 'getConnection': conn2}) node2 = Mock({'__repr__': 'node2', '__hash__': 2, 'getConnection': conn2})
node3 = Mock({'__repr__': 'node3', '__hash__': 3, 'getConnection': conn3}) node3 = Mock({'__repr__': 'node3', '__hash__': 3, 'getConnection': conn3})
cell1 = Mock({ 'getNode': node1, '__hash__': 1, 'getConnection': conn1})
cell2 = Mock({ 'getNode': node2, '__hash__': 2, 'getConnection': conn2})
cell3 = Mock({ 'getNode': node3, '__hash__': 3, 'getConnection': conn3})
# fake environment # fake environment
app.pt = Mock({
'getCellListForTID': [cell1],
'getCellListForOID': ReturnValues([cell2], [cell3]),
})
app.cp = Mock({'getConnForCell': ReturnValues(conn2, conn3, conn1)}) app.cp = Mock({'getConnForCell': ReturnValues(conn2, conn3, conn1)})
app.cp = Mock({ app.cp = Mock({
'getConnForNode': ReturnValues(conn2, conn3, conn1), 'getConnForNode': ReturnValues(conn2, conn3, conn1),
...@@ -544,11 +532,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -544,11 +532,7 @@ class ClientApplicationTests(NeoUnitTestBase):
'getAddress': 'FakeServer', 'getAddress': 'FakeServer',
'getState': 'FakeState', 'getState': 'FakeState',
}) })
app.pt = Mock({ app.pt = Mock({'getCellList': [cell]})
'getCellListForTID': [cell, ],
'getCellListForOID': [cell, ],
'getCellList': [cell, ],
})
transaction_info = Packets.AnswerTransactionInformation(tid1, '', '', transaction_info = Packets.AnswerTransactionInformation(tid1, '', '',
'', False, (oid0, )) '', False, (oid0, ))
transaction_info.setId(1) transaction_info.setId(1)
...@@ -752,7 +736,6 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -752,7 +736,6 @@ class ClientApplicationTests(NeoUnitTestBase):
app.dispatcher = Dispatcher() app.dispatcher = Dispatcher()
app.pt = Mock({ app.pt = Mock({
'getNodeList': (Mock(), Mock()), 'getNodeList': (Mock(), Mock()),
'getCellListForTID': ReturnValues([Mock()], [Mock()]),
}) })
app.cp = Mock({ app.cp = Mock({
'getConnForNode': ReturnValues(answerTIDs(p1), answerTIDs(p2)), 'getConnForNode': ReturnValues(answerTIDs(p1), answerTIDs(p2)),
......
...@@ -72,7 +72,7 @@ class ConnectionPoolTests(NeoUnitTestBase): ...@@ -72,7 +72,7 @@ class ConnectionPoolTests(NeoUnitTestBase):
def test_iterateForObject_noStorageAvailable(self): def test_iterateForObject_noStorageAvailable(self):
# no node available # no node available
oid = self.getOID(1) oid = self.getOID(1)
pt = Mock({'getCellListForOID': []}) pt = Mock({'getCellList': []})
app = Mock({'getPartitionTable': pt}) app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app) pool = ConnectionPool(app)
self.assertRaises(NEOStorageError, pool.iterateForObject(oid).next) self.assertRaises(NEOStorageError, pool.iterateForObject(oid).next)
...@@ -83,7 +83,7 @@ class ConnectionPoolTests(NeoUnitTestBase): ...@@ -83,7 +83,7 @@ class ConnectionPoolTests(NeoUnitTestBase):
node = Mock({'__repr__': 'node', 'isRunning': True}) node = Mock({'__repr__': 'node', 'isRunning': True})
cell = Mock({'__repr__': 'cell', 'getNode': node}) cell = Mock({'__repr__': 'cell', 'getNode': node})
conn = Mock({'__repr__': 'conn'}) conn = Mock({'__repr__': 'conn'})
pt = Mock({'getCellListForOID': [cell]}) pt = Mock({'getCellList': [cell]})
app = Mock({'getPartitionTable': pt}) app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app) pool = ConnectionPool(app)
pool.getConnForNode = Mock({'__call__': ReturnValues(None, conn)}) pool.getConnForNode = Mock({'__call__': ReturnValues(None, conn)})
...@@ -95,7 +95,7 @@ class ConnectionPoolTests(NeoUnitTestBase): ...@@ -95,7 +95,7 @@ class ConnectionPoolTests(NeoUnitTestBase):
node = Mock({'__repr__': 'node', 'isRunning': True}) node = Mock({'__repr__': 'node', 'isRunning': True})
cell = Mock({'__repr__': 'cell', 'getNode': node}) cell = Mock({'__repr__': 'cell', 'getNode': node})
conn = Mock({'__repr__': 'conn'}) conn = Mock({'__repr__': 'conn'})
pt = Mock({'getCellListForOID': [cell]}) pt = Mock({'getCellList': [cell]})
app = Mock({'getPartitionTable': pt}) app = Mock({'getPartitionTable': pt})
pool = ConnectionPool(app) pool = ConnectionPool(app)
pool.getConnForNode = Mock({'__call__': conn}) pool.getConnForNode = Mock({'__call__': conn})
......
...@@ -421,35 +421,6 @@ class PartitionTableTests(NeoUnitTestBase): ...@@ -421,35 +421,6 @@ class PartitionTableTests(NeoUnitTestBase):
# unknwon row # unknwon row
self.assertRaises(IndexError, pt.getRow, 5) self.assertRaises(IndexError, pt.getRow, 5)
def test_getNodeMap(self):
num_partitions = 5
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
uuid1 = self.getNewUUID()
uuid2 = self.getNewUUID()
uuid3 = self.getNewUUID()
sn1 = StorageNode(Mock(),("127.0.0.1", 19001) , uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
self.assertEqual(pt.getNodeMap(), {
sn1: [0, 1, 2],
})
sn2 = StorageNode(Mock(), ("127.0.0.2", 19001), uuid2)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
self.assertEqual(pt.getNodeMap(), {
sn1: [0, 1, 2],
sn2: [0, 1],
})
sn3 = StorageNode(Mock(), ("127.0.0.3", 19001), uuid3)
pt.setCell(0, sn3, CellStates.UP_TO_DATE)
self.assertEqual(pt.getNodeMap(), {
sn1: [0, 1, 2],
sn2: [0, 1],
sn3: [0],
})
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment