Commit d250deca authored by Julien Muchembled's avatar Julien Muchembled

Drop support for Python < 2.7

parent 14fd9cd3
graft tools graft tools
include neo.conf CHANGES TODO TESTS.txt ZODB3.patch include neo.conf CHANGES TODO TESTS.txt ZODB3.patch
include neo/client/component.xml # required for Python < 2.7
...@@ -54,7 +54,7 @@ Requirements ...@@ -54,7 +54,7 @@ Requirements
- Linux 2.6 or later - Linux 2.6 or later
- Python 2.6.x or 2.7.x - Python 2.7.x
- For storage nodes using MySQL backend: - For storage nodes using MySQL backend:
......
...@@ -27,7 +27,6 @@ if 1: ...@@ -27,7 +27,6 @@ if 1:
# to tpc_finish/tpc_abort). # to tpc_finish/tpc_abort).
_check(Connection.tpc_finish, _check(Connection.tpc_finish,
'f50ed2e5a74f584fa1ecaf1cd70a6b15', # Python 2.6
'ab9b1b8d82c40e5fffa84f7bc4ea3a8b', # Python 2.7 'ab9b1b8d82c40e5fffa84f7bc4ea3a8b', # Python 2.7
) )
...@@ -72,7 +71,6 @@ if 1: ...@@ -72,7 +71,6 @@ if 1:
# know any legitimate use of DB access outside a transaction. # know any legitimate use of DB access outside a transaction.
_check(Connection.afterCompletion, _check(Connection.afterCompletion,
'70dfc3df8a455d9f663dec619c607eb5', # Python 2.6
'cd3a080b80fd957190ff3bb867149448', # Python 2.7 'cd3a080b80fd957190ff3bb867149448', # Python 2.7
) )
......
...@@ -21,5 +21,5 @@ class NeoStorage(BaseConfig): ...@@ -21,5 +21,5 @@ class NeoStorage(BaseConfig):
def open(self): def open(self):
from .Storage import Storage from .Storage import Storage
config = self.config config = self.config
return Storage(**dict((k, getattr(config, k)) return Storage(**{k: getattr(config, k)
for k in config.getSectionAttributes())) for k in config.getSectionAttributes()})
...@@ -181,7 +181,7 @@ UUID_NAMESPACES = { ...@@ -181,7 +181,7 @@ UUID_NAMESPACES = {
} }
uuid_str = (lambda ns: lambda uuid: uuid_str = (lambda ns: lambda uuid:
ns[uuid >> 24] + str(uuid & 0xffffff) if uuid else str(uuid) ns[uuid >> 24] + str(uuid & 0xffffff) if uuid else str(uuid)
)(dict((v, str(k)[0]) for k, v in UUID_NAMESPACES.iteritems())) )({v: str(k)[0] for k, v in UUID_NAMESPACES.iteritems()})
class ProtocolError(Exception): class ProtocolError(Exception):
""" Base class for protocol errors, close the connection """ """ Base class for protocol errors, close the connection """
......
...@@ -129,9 +129,9 @@ class PartitionTable(object): ...@@ -129,9 +129,9 @@ class PartitionTable(object):
def getNodeSet(self, readable=False): def getNodeSet(self, readable=False):
if readable: if readable:
return set(x.getNode() for row in self.partition_list for x in row return {x.getNode() for row in self.partition_list for x in row
if x.isReadable()) if x.isReadable()}
return set(x.getNode() for row in self.partition_list for x in row) return {x.getNode() for row in self.partition_list for x in row}
def getConnectedNodeList(self): def getConnectedNodeList(self):
return [node for node in self.getNodeSet() if node.isConnected()] return [node for node in self.getNodeSet() if node.isConnected()]
...@@ -276,8 +276,8 @@ class PartitionTable(object): ...@@ -276,8 +276,8 @@ class PartitionTable(object):
if row is None: if row is None:
line.append('X' * len(node_list)) line.append('X' * len(node_list))
else: else:
cell_dict = dict((x.getNode(), cell_state_dict[x.getState()]) cell_dict = {x.getNode(): cell_state_dict[x.getState()]
for x in row) for x in row}
line.append(''.join(cell_dict.get(x, '.') for x in node_list)) line.append(''.join(cell_dict.get(x, '.') for x in node_list))
if line: if line:
append('pt: %0*u: %s' % (prefix_len, prefix, '|'.join(line))) append('pt: %0*u: %s' % (prefix_len, prefix, '|'.join(line)))
......
...@@ -282,8 +282,8 @@ class Application(object): ...@@ -282,8 +282,8 @@ class Application(object):
if e.args[0] != ClusterStates.STARTING_BACKUP: if e.args[0] != ClusterStates.STARTING_BACKUP:
raise raise
self.backup_tid = tid = self.getLastTransaction() self.backup_tid = tid = self.getLastTransaction()
self.pt.setBackupTidDict(dict((node.getUUID(), tid) self.pt.setBackupTidDict({node.getUUID(): tid
for node in self.nm.getStorageList(only_identified=True))) for node in self.nm.getStorageList(only_identified=True)})
def playPrimaryRole(self): def playPrimaryRole(self):
logging.info('play the primary role with %r', self.listening_conn) logging.info('play the primary role with %r', self.listening_conn)
......
...@@ -56,25 +56,21 @@ class ClientServiceHandler(MasterHandler): ...@@ -56,25 +56,21 @@ class ClientServiceHandler(MasterHandler):
def askFinishTransaction(self, conn, ttid, oid_list): def askFinishTransaction(self, conn, ttid, oid_list):
app = self.app app = self.app
pt = app.pt
# Collect partitions related to this transaction. # Collect partitions related to this transaction.
getPartition = app.pt.getPartition partition_set = set(map(pt.getPartition, oid_list))
partition_set = set(getPartition(oid) for oid in oid_list) partition_set.add(pt.getPartition(ttid))
partition_set.add(getPartition(ttid))
# Collect the UUIDs of nodes related to this transaction. # Collect the UUIDs of nodes related to this transaction.
uuid_set = set() uuid_list = filter(app.isStorageReady, {cell.getUUID()
isStorageReady = app.isStorageReady for part in partition_set
for part in partition_set: for cell in pt.getCellList(part)
uuid_set.update((uuid for uuid in ( if cell.getNodeState() != NodeStates.HIDDEN})
cell.getUUID() for cell in app.pt.getCellList(part) if not uuid_list:
if cell.getNodeState() != NodeStates.HIDDEN)
if isStorageReady(uuid)))
if not uuid_set:
raise ProtocolError('No storage node ready for transaction') raise ProtocolError('No storage node ready for transaction')
identified_node_list = app.nm.getIdentifiedList(pool_set=uuid_set) identified_node_list = app.nm.getIdentifiedList(pool_set=set(uuid_list))
# Request locking data. # Request locking data.
# build a new set as we may not send the message to all nodes as some # build a new set as we may not send the message to all nodes as some
...@@ -83,9 +79,9 @@ class ClientServiceHandler(MasterHandler): ...@@ -83,9 +79,9 @@ class ClientServiceHandler(MasterHandler):
ttid, ttid,
app.tm.prepare( app.tm.prepare(
ttid, ttid,
app.pt.getPartitions(), pt.getPartitions(),
oid_list, oid_list,
set(x.getUUID() for x in identified_node_list), {x.getUUID() for x in identified_node_list},
conn.getPeerId(), conn.getPeerId(),
), ),
oid_list, oid_list,
...@@ -98,7 +94,7 @@ class ClientServiceHandler(MasterHandler): ...@@ -98,7 +94,7 @@ class ClientServiceHandler(MasterHandler):
if app.packing is None: if app.packing is None:
storage_list = app.nm.getStorageList(only_identified=True) storage_list = app.nm.getStorageList(only_identified=True)
app.packing = (conn, conn.getPeerId(), app.packing = (conn, conn.getPeerId(),
set(x.getUUID() for x in storage_list)) {x.getUUID() for x in storage_list})
p = Packets.AskPack(tid) p = Packets.AskPack(tid)
for storage in storage_list: for storage in storage_list:
storage.getConnection().ask(p) storage.getConnection().ask(p)
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from collections import defaultdict
import neo.lib.pt import neo.lib.pt
from neo.lib.protocol import CellStates, ZERO_TID from neo.lib.protocol import CellStates, ZERO_TID
...@@ -188,17 +189,17 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -188,17 +189,17 @@ class PartitionTable(neo.lib.pt.PartitionTable):
This is done by computing a minimal diff between current partition table This is done by computing a minimal diff between current partition table
and what make() would do. and what make() would do.
""" """
assigned_dict = dict((x, {}) for x in self.count_dict) assigned_dict = {x: {} for x in self.count_dict}
readable_dict = dict((i, set()) for i in xrange(self.np)) readable_list = [set() for x in xrange(self.np)]
for offset, row in enumerate(self.partition_list): for offset, row in enumerate(self.partition_list):
for cell in row: for cell in row:
if cell.isReadable(): if cell.isReadable():
readable_dict[offset].add(cell) readable_list[offset].add(cell)
assigned_dict[cell.getNode()][offset] = cell assigned_dict[cell.getNode()][offset] = cell
pt = PartitionTable(self.np, self.nr) pt = PartitionTable(self.np, self.nr)
drop_list = set(x for x in drop_list if x in assigned_dict) drop_list = set(drop_list).intersection(assigned_dict)
node_set = set(MappedNode(x) for x in assigned_dict node_set = {MappedNode(x) for x in assigned_dict
if x not in drop_list) if x not in drop_list}
pt.make(node_set) pt.make(node_set)
for offset, row in enumerate(pt.partition_list): for offset, row in enumerate(pt.partition_list):
for cell in row: for cell in row:
...@@ -210,8 +211,8 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -210,8 +211,8 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if node in drop_list: if node in drop_list:
yield node, frozenset() yield node, frozenset()
continue continue
readable = set(offset for offset, cell in assigned.iteritems() readable = {offset for offset, cell in assigned.iteritems()
if cell.isReadable()) if cell.isReadable()}
# the criterion on UUID is purely cosmetic # the criterion on UUID is purely cosmetic
node_list.append((len(readable), len(assigned), node_list.append((len(readable), len(assigned),
-node.getUUID(), readable, node)) -node.getUUID(), readable, node))
...@@ -226,7 +227,7 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -226,7 +227,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
assert not node_set assert not node_set
changed_list = [] changed_list = []
uptodate_set = set() uptodate_set = set()
remove_dict = dict((i, []) for i in xrange(self.np)) remove_dict = defaultdict(list)
for node, mapped in map_nodes(): for node, mapped in map_nodes():
uuid = node.getUUID() uuid = node.getUUID()
assigned = assigned_dict[node] assigned = assigned_dict[node]
...@@ -234,7 +235,7 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -234,7 +235,7 @@ class PartitionTable(neo.lib.pt.PartitionTable):
if offset in mapped: if offset in mapped:
if cell.isReadable(): if cell.isReadable():
uptodate_set.add(offset) uptodate_set.add(offset)
readable_dict[offset].remove(cell) readable_list[offset].remove(cell)
if cell.isFeeding(): if cell.isFeeding():
self.count_dict[node] += 1 self.count_dict[node] += 1
state = CellStates.UP_TO_DATE state = CellStates.UP_TO_DATE
...@@ -251,11 +252,9 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -251,11 +252,9 @@ class PartitionTable(neo.lib.pt.PartitionTable):
changed_list.append((offset, uuid, state)) changed_list.append((offset, uuid, state))
count_dict = self.count_dict.copy() count_dict = self.count_dict.copy()
for offset, cell_list in remove_dict.iteritems(): for offset, cell_list in remove_dict.iteritems():
if not cell_list:
continue
row = self.partition_list[offset] row = self.partition_list[offset]
feeding = None if offset in uptodate_set else min( feeding = None if offset in uptodate_set else min(
readable_dict[offset], key=lambda x: count_dict[x.getNode()]) readable_list[offset], key=lambda x: count_dict[x.getNode()])
for cell in cell_list: for cell in cell_list:
if cell is feeding: if cell is feeding:
count_dict[cell.getNode()] += 1 count_dict[cell.getNode()] += 1
...@@ -305,10 +304,10 @@ class PartitionTable(neo.lib.pt.PartitionTable): ...@@ -305,10 +304,10 @@ class PartitionTable(neo.lib.pt.PartitionTable):
Return a set of all nodes which are part of at least one UP TO DATE Return a set of all nodes which are part of at least one UP TO DATE
partition. partition.
""" """
return set(cell.getNode() return {cell.getNode()
for row in self.partition_list for row in self.partition_list
for cell in row for cell in row
if cell.isReadable()) if cell.isReadable()}
def clearReplicating(self): def clearReplicating(self):
for row in self.partition_list: for row in self.partition_list:
......
...@@ -63,8 +63,8 @@ class RecoveryManager(MasterHandler): ...@@ -63,8 +63,8 @@ class RecoveryManager(MasterHandler):
# A partition table exists, we are starting an existing # A partition table exists, we are starting an existing
# cluster. # cluster.
partition_node_set = pt.getReadableCellNodeSet() partition_node_set = pt.getReadableCellNodeSet()
pending_node_set = set(x for x in partition_node_set pending_node_set = {x for x in partition_node_set
if x.isPending()) if x.isPending()}
if app._startup_allowed or \ if app._startup_allowed or \
partition_node_set == pending_node_set: partition_node_set == pending_node_set:
allowed_node_set = pending_node_set allowed_node_set = pending_node_set
......
...@@ -41,7 +41,7 @@ action_dict = { ...@@ -41,7 +41,7 @@ action_dict = {
uuid_int = (lambda ns: lambda uuid: uuid_int = (lambda ns: lambda uuid:
(ns[uuid[0]] << 24) + int(uuid[1:]) (ns[uuid[0]] << 24) + int(uuid[1:])
)(dict((str(k)[0], v) for k, v in UUID_NAMESPACES.iteritems())) )({str(k)[0]: v for k, v in UUID_NAMESPACES.iteritems()})
class TerminalNeoCTL(object): class TerminalNeoCTL(object):
def __init__(self, address): def __init__(self, address):
......
...@@ -44,8 +44,8 @@ def main(): ...@@ -44,8 +44,8 @@ def main():
options, args = parser.parse_args() options, args = parser.parse_args()
if options.seed: if options.seed:
functional.random = random.Random(options.seed) functional.random = random.Random(options.seed)
cluster = functional.NEOCluster(args, **dict((x, getattr(options, x)) cluster = functional.NEOCluster(args, **{x: getattr(options, x)
for x, _ in option_list)) for x, _ in option_list})
try: try:
cluster.run() cluster.run()
logging.info("Cluster running ...") logging.info("Cluster running ...")
......
...@@ -200,7 +200,7 @@ class Application(object): ...@@ -200,7 +200,7 @@ class Application(object):
conn.close() conn.close()
# create/clear event queue # create/clear event queue
self.event_queue = deque() self.event_queue = deque()
self.event_queue_dict = dict() self.event_queue_dict = {}
try: try:
self.verifyData() self.verifyData()
self.initialize() self.initialize()
......
...@@ -259,12 +259,12 @@ class MySQLDatabaseManager(DatabaseManager): ...@@ -259,12 +259,12 @@ class MySQLDatabaseManager(DatabaseManager):
def _getLastIDs(self, all=True): def _getLastIDs(self, all=True):
p64 = util.p64 p64 = util.p64
q = self.query q = self.query
trans = dict((partition, p64(tid)) trans = {partition: p64(tid)
for partition, tid in q("SELECT partition, MAX(tid)" for partition, tid in q("SELECT partition, MAX(tid)"
" FROM trans GROUP BY partition")) " FROM trans GROUP BY partition")}
obj = dict((partition, p64(tid)) obj = {partition: p64(tid)
for partition, tid in q("SELECT partition, MAX(tid)" for partition, tid in q("SELECT partition, MAX(tid)"
" FROM obj GROUP BY partition")) " FROM obj GROUP BY partition")}
oid = q("SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj" oid = q("SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj"
" GROUP BY partition) as t")[0][0] " GROUP BY partition) as t")[0][0]
if all: if all:
......
...@@ -221,12 +221,12 @@ class SQLiteDatabaseManager(DatabaseManager): ...@@ -221,12 +221,12 @@ class SQLiteDatabaseManager(DatabaseManager):
def _getLastIDs(self, all=True): def _getLastIDs(self, all=True):
p64 = util.p64 p64 = util.p64
q = self.query q = self.query
trans = dict((partition, p64(tid)) trans = {partition: p64(tid)
for partition, tid in q("SELECT partition, MAX(tid)" for partition, tid in q("SELECT partition, MAX(tid)"
" FROM trans GROUP BY partition")) " FROM trans GROUP BY partition")}
obj = dict((partition, p64(tid)) obj = {partition: p64(tid)
for partition, tid in q("SELECT partition, MAX(tid)" for partition, tid in q("SELECT partition, MAX(tid)"
" FROM obj GROUP BY partition")) " FROM obj GROUP BY partition")}
oid = q("SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj" oid = q("SELECT MAX(oid) FROM (SELECT MAX(oid) AS oid FROM obj"
" GROUP BY partition) as t").next()[0] " GROUP BY partition) as t").next()[0]
if all: if all:
......
...@@ -64,9 +64,8 @@ class MasterOperationHandler(BaseMasterHandler): ...@@ -64,9 +64,8 @@ class MasterOperationHandler(BaseMasterHandler):
conn.answer(Packets.AnswerPack(True)) conn.answer(Packets.AnswerPack(True))
def replicate(self, conn, tid, upstream_name, source_dict): def replicate(self, conn, tid, upstream_name, source_dict):
self.app.replicator.backup(tid, self.app.replicator.backup(tid, {p: a and (a, upstream_name)
dict((p, a and (a, upstream_name)) for p, a in source_dict.iteritems()})
for p, a in source_dict.iteritems()))
def truncate(self, conn, tid): def truncate(self, conn, tid):
self.app.replicator.cancel() self.app.replicator.cancel()
......
...@@ -514,28 +514,3 @@ class DoNothingConnector(Mock): ...@@ -514,28 +514,3 @@ class DoNothingConnector(Mock):
__builtin__.pdb = lambda depth=0: \ __builtin__.pdb = lambda depth=0: \
debug.getPdb().set_trace(sys._getframe(depth+1)) debug.getPdb().set_trace(sys._getframe(depth+1))
def _fixMockForInspect():
"""
inspect module change broke Mock, see http://bugs.python.org/issue1785
Monkey-patch Mock class if needed by replacing predicate parameter on 2nd
getmembers call with isroutine (was ismethod).
"""
import inspect
class A(object):
def f(self):
pass
if not inspect.getmembers(A, inspect.ismethod):
from mock import MockCallable
# _setupSubclassMethodInterceptors is under the FreeBSD license.
# See pyMock module for the whole license.
def _setupSubclassMethodInterceptors(self):
methods = inspect.getmembers(self.__class__, inspect.isroutine)
baseMethods = dict(inspect.getmembers(Mock, inspect.isroutine))
for m in methods:
name = m[0]
# Don't record calls to methods of Mock base class.
if not name in baseMethods:
self.__dict__[name] = MockCallable(name, self, handcrafted=True)
Mock._setupSubclassMethodInterceptors = _setupSubclassMethodInterceptors
_fixMockForInspect()
...@@ -337,7 +337,7 @@ class ClientApplicationTests(NeoUnitTestBase): ...@@ -337,7 +337,7 @@ class ClientApplicationTests(NeoUnitTestBase):
txn_context['queue'].put((conn, packet, {})) txn_context['queue'].put((conn, packet, {}))
app.waitStoreResponses(txn_context, resolving_tryToResolveConflict) app.waitStoreResponses(txn_context, resolving_tryToResolveConflict)
self.assertEqual(txn_context['object_stored_counter_dict'][oid], self.assertEqual(txn_context['object_stored_counter_dict'][oid],
{tid: set([uuid])}) {tid: {uuid}})
self.assertEqual(txn_context['cache_dict'][oid], 'DATA') self.assertEqual(txn_context['cache_dict'][oid], 'DATA')
self.assertFalse(oid in txn_context['data_dict']) self.assertFalse(oid in txn_context['data_dict'])
self.assertFalse(oid in txn_context['conflict_serial_dict']) self.assertFalse(oid in txn_context['conflict_serial_dict'])
......
...@@ -65,11 +65,11 @@ class StorageAnswerHandlerTests(NeoUnitTestBase): ...@@ -65,11 +65,11 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
self._getAnswerStoreObjectHandler(object_stored_counter_dict, self._getAnswerStoreObjectHandler(object_stored_counter_dict,
conflict_serial_dict, resolved_conflict_serial_dict, conflict_serial_dict, resolved_conflict_serial_dict,
).answerStoreObject(conn, 1, oid, tid) ).answerStoreObject(conn, 1, oid, tid)
self.assertEqual(conflict_serial_dict[oid], set([tid, ])) self.assertEqual(conflict_serial_dict[oid], {tid})
self.assertEqual(object_stored_counter_dict[oid], {}) self.assertEqual(object_stored_counter_dict[oid], {})
self.assertFalse(oid in resolved_conflict_serial_dict) self.assertFalse(oid in resolved_conflict_serial_dict)
# object was already accepted by another storage, raise # object was already accepted by another storage, raise
handler = self._getAnswerStoreObjectHandler({oid: {tid: set([1])}}, {}, {}) handler = self._getAnswerStoreObjectHandler({oid: {tid: {1}}}, {}, {})
self.assertRaises(NEOStorageError, handler.answerStoreObject, self.assertRaises(NEOStorageError, handler.answerStoreObject,
conn, 1, oid, tid) conn, 1, oid, tid)
...@@ -80,21 +80,21 @@ class StorageAnswerHandlerTests(NeoUnitTestBase): ...@@ -80,21 +80,21 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
tid_2 = self.getNextTID() tid_2 = self.getNextTID()
# resolution-pending conflict # resolution-pending conflict
object_stored_counter_dict = {oid: {}} object_stored_counter_dict = {oid: {}}
conflict_serial_dict = {oid: set([tid, ])} conflict_serial_dict = {oid: {tid}}
resolved_conflict_serial_dict = {} resolved_conflict_serial_dict = {}
self._getAnswerStoreObjectHandler(object_stored_counter_dict, self._getAnswerStoreObjectHandler(object_stored_counter_dict,
conflict_serial_dict, resolved_conflict_serial_dict, conflict_serial_dict, resolved_conflict_serial_dict,
).answerStoreObject(conn, 1, oid, tid) ).answerStoreObject(conn, 1, oid, tid)
self.assertEqual(conflict_serial_dict[oid], set([tid, ])) self.assertEqual(conflict_serial_dict[oid], {tid})
self.assertFalse(oid in resolved_conflict_serial_dict) self.assertFalse(oid in resolved_conflict_serial_dict)
self.assertEqual(object_stored_counter_dict[oid], {}) self.assertEqual(object_stored_counter_dict[oid], {})
# object was already accepted by another storage, raise # object was already accepted by another storage, raise
handler = self._getAnswerStoreObjectHandler({oid: {tid: set([1])}}, handler = self._getAnswerStoreObjectHandler({oid: {tid: {1}}},
{oid: set([tid, ])}, {}) {oid: {tid}}, {})
self.assertRaises(NEOStorageError, handler.answerStoreObject, self.assertRaises(NEOStorageError, handler.answerStoreObject,
conn, 1, oid, tid) conn, 1, oid, tid)
# detected conflict is different, don't raise # detected conflict is different, don't raise
self._getAnswerStoreObjectHandler({oid: {}}, {oid: set([tid, ])}, {}, self._getAnswerStoreObjectHandler({oid: {}}, {oid: {tid}}, {},
).answerStoreObject(conn, 1, oid, tid_2) ).answerStoreObject(conn, 1, oid, tid_2)
def test_answerStoreObject_3(self): def test_answerStoreObject_3(self):
...@@ -108,17 +108,16 @@ class StorageAnswerHandlerTests(NeoUnitTestBase): ...@@ -108,17 +108,16 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
# storage accepted the resolved object. # storage accepted the resolved object.
object_stored_counter_dict = {oid: {tid_2: 1}} object_stored_counter_dict = {oid: {tid_2: 1}}
conflict_serial_dict = {} conflict_serial_dict = {}
resolved_conflict_serial_dict = {oid: set([tid, ])} resolved_conflict_serial_dict = {oid: {tid}}
self._getAnswerStoreObjectHandler(object_stored_counter_dict, self._getAnswerStoreObjectHandler(object_stored_counter_dict,
conflict_serial_dict, resolved_conflict_serial_dict, conflict_serial_dict, resolved_conflict_serial_dict,
).answerStoreObject(conn, 1, oid, tid) ).answerStoreObject(conn, 1, oid, tid)
self.assertFalse(oid in conflict_serial_dict) self.assertFalse(oid in conflict_serial_dict)
self.assertEqual(resolved_conflict_serial_dict[oid], self.assertEqual(resolved_conflict_serial_dict[oid], {tid})
set([tid, ]))
self.assertEqual(object_stored_counter_dict[oid], {tid_2: 1}) self.assertEqual(object_stored_counter_dict[oid], {tid_2: 1})
# detected conflict is different, don't raise # detected conflict is different, don't raise
self._getAnswerStoreObjectHandler({oid: {tid: 1}}, {}, self._getAnswerStoreObjectHandler({oid: {tid: 1}}, {},
{oid: set([tid, ])}).answerStoreObject(conn, 1, oid, tid_2) {oid: {tid}}).answerStoreObject(conn, 1, oid, tid_2)
def test_answerStoreObject_4(self): def test_answerStoreObject_4(self):
uuid = self.getStorageUUID() uuid = self.getStorageUUID()
...@@ -135,7 +134,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase): ...@@ -135,7 +134,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
h.answerStoreObject(conn, 0, oid, tid) h.answerStoreObject(conn, 0, oid, tid)
self.assertFalse(oid in conflict_serial_dict) self.assertFalse(oid in conflict_serial_dict)
self.assertFalse(oid in resolved_conflict_serial_dict) self.assertFalse(oid in resolved_conflict_serial_dict)
self.assertEqual(object_stored_counter_dict[oid], {tid: set([uuid])}) self.assertEqual(object_stored_counter_dict[oid], {tid: {uuid}})
def test_answerTransactionInformation(self): def test_answerTransactionInformation(self):
conn = self.getFakeConnection() conn = self.getFakeConnection()
......
...@@ -67,7 +67,7 @@ class NotFound(Exception): ...@@ -67,7 +67,7 @@ class NotFound(Exception):
class PortAllocator(object): class PortAllocator(object):
lock = SocketLock('neo.PortAllocator') lock = SocketLock('neo.PortAllocator')
allocator_set = weakref.WeakKeyDictionary() # BBB: use WeakSet instead allocator_set = weakref.WeakSet()
def __init__(self): def __init__(self):
self.socket_list = [] self.socket_list = []
...@@ -81,7 +81,7 @@ class PortAllocator(object): ...@@ -81,7 +81,7 @@ class PortAllocator(object):
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if not self.lock.locked(): if not self.lock.locked():
self.lock.acquire() self.lock.acquire()
self.allocator_set[self] = None self.allocator_set.add(self)
self.socket_list.append(s) self.socket_list.append(s)
sock_port_set = self.sock_port_set sock_port_set = self.sock_port_set
while True: while True:
...@@ -109,7 +109,7 @@ class PortAllocator(object): ...@@ -109,7 +109,7 @@ class PortAllocator(object):
def reset(self): def reset(self):
if self.lock.locked(): if self.lock.locked():
self.allocator_set.pop(self, None) self.allocator_set.discard(self)
if not self.allocator_set: if not self.allocator_set:
self.lock.release() self.lock.release()
self.release() self.release()
...@@ -126,7 +126,7 @@ class NEOProcess(object): ...@@ -126,7 +126,7 @@ class NEOProcess(object):
except ImportError: except ImportError:
raise NotFound, '%s not found' % (command) raise NotFound, '%s not found' % (command)
self.command = command self.command = command
self.arg_dict = dict(('--' + k, v) for k, v in arg_dict.iteritems()) self.arg_dict = {'--' + k: v for k, v in arg_dict.iteritems()}
self.with_uuid = True self.with_uuid = True
self.setUUID(uuid) self.setUUID(uuid)
......
...@@ -186,7 +186,7 @@ class MasterClientHandlerTests(NeoUnitTestBase): ...@@ -186,7 +186,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
self.assertEqual(ptid, tid) self.assertEqual(ptid, tid)
self.assertTrue(self.app.packing[0] is conn) self.assertTrue(self.app.packing[0] is conn)
self.assertEqual(self.app.packing[1], peer_id) self.assertEqual(self.app.packing[1], peer_id)
self.assertEqual(self.app.packing[2], set([storage_uuid, ])) self.assertEqual(self.app.packing[2], {storage_uuid})
# Asking again to pack will cause an immediate error # Asking again to pack will cause an immediate error
storage_uuid = self.identifyToMasterNode(port=10022) storage_uuid = self.identifyToMasterNode(port=10022)
storage_conn = self.getFakeConnection(storage_uuid, storage_conn = self.getFakeConnection(storage_uuid,
......
...@@ -220,7 +220,7 @@ class MasterPartitionTableTests(NeoUnitTestBase): ...@@ -220,7 +220,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
if cell.isOutOfDate(): if cell.isOutOfDate():
pt.setUpToDate(cell.getNode(), offset) pt.setUpToDate(cell.getNode(), offset)
else: else:
node_dict = dict((x.getUUID(), x) for x in pt.count_dict) node_dict = {x.getUUID(): x for x in pt.count_dict}
for offset, uuid, state in change_list: for offset, uuid, state in change_list:
if state is CellStates.OUT_OF_DATE: if state is CellStates.OUT_OF_DATE:
pt.setUpToDate(node_dict[uuid], offset) pt.setUpToDate(node_dict[uuid], offset)
......
...@@ -242,11 +242,11 @@ class MasterStorageHandlerTests(NeoUnitTestBase): ...@@ -242,11 +242,11 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
'getPeerId': 512, 'getPeerId': 512,
}) })
client_peer_id = 42 client_peer_id = 42
self.app.packing = (client_conn, client_peer_id, set([conn1.getUUID(), self.app.packing = (client_conn, client_peer_id,
conn2.getUUID()])) {conn1.getUUID(), conn2.getUUID()})
self.service.answerPack(conn1, False) self.service.answerPack(conn1, False)
self.checkNoPacketSent(client_conn) self.checkNoPacketSent(client_conn)
self.assertEqual(self.app.packing[2], set([conn2.getUUID(), ])) self.assertEqual(self.app.packing[2], {conn2.getUUID()})
self.service.answerPack(conn2, False) self.service.answerPack(conn2, False)
status = self.checkAnswerPacket(client_conn, Packets.AnswerPack, status = self.checkAnswerPacket(client_conn, Packets.AnswerPack,
decode=True)[0] decode=True)[0]
......
...@@ -99,7 +99,7 @@ class StorageDBTests(NeoUnitTestBase): ...@@ -99,7 +99,7 @@ class StorageDBTests(NeoUnitTestBase):
cell2 = (1, uuid1, CellStates.UP_TO_DATE) cell2 = (1, uuid1, CellStates.UP_TO_DATE)
db.setPartitionTable(ptid, [cell1, cell2]) db.setPartitionTable(ptid, [cell1, cell2])
result = db.getPartitionTable() result = db.getPartitionTable()
self.assertEqual(set(result), set([cell1, cell2])) self.assertEqual(set(result), {cell1, cell2})
def getOIDs(self, count): def getOIDs(self, count):
return map(self.getOID, xrange(count)) return map(self.getOID, xrange(count))
...@@ -253,7 +253,7 @@ class StorageDBTests(NeoUnitTestBase): ...@@ -253,7 +253,7 @@ class StorageDBTests(NeoUnitTestBase):
# add more entries # add more entries
db.changePartitionTable(ptid, [cell2]) db.changePartitionTable(ptid, [cell2])
result = db.getPartitionTable() result = db.getPartitionTable()
self.assertEqual(set(result), set([cell1, cell2])) self.assertEqual(set(result), {cell1, cell2})
# drop discarded cells # drop discarded cells
db.changePartitionTable(ptid, [cell2, cell3]) db.changePartitionTable(ptid, [cell2, cell3])
result = db.getPartitionTable() result = db.getPartitionTable()
......
...@@ -263,7 +263,7 @@ class PartitionTableTests(NeoUnitTestBase): ...@@ -263,7 +263,7 @@ class PartitionTableTests(NeoUnitTestBase):
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
# must get only two node as feeding and discarded not taken # must get only two node as feeding and discarded not taken
# into account # into account
self.assertEqual(pt.getNodeSet(True), set((sn1, sn3))) self.assertEqual(pt.getNodeSet(True), {sn1, sn3})
self.assertEqual(len(pt.getNodeSet()), 3) self.assertEqual(len(pt.getNodeSet()), 3)
def test_08_filled(self): def test_08_filled(self):
......
...@@ -166,9 +166,9 @@ class Node(object): ...@@ -166,9 +166,9 @@ class Node(object):
def getConnectionList(self, *peers): def getConnectionList(self, *peers):
addr = lambda c: c and (c.accepted_from or c.getAddress()) addr = lambda c: c and (c.accepted_from or c.getAddress())
addr_set = set(addr(c.connector) for peer in peers addr_set = {addr(c.connector) for peer in peers
for c in peer.em.connection_dict.itervalues() for c in peer.em.connection_dict.itervalues()
if isinstance(c, Connection)) if isinstance(c, Connection)}
addr_set.discard(None) addr_set.discard(None)
return (c for c in self.em.connection_dict.itervalues() return (c for c in self.em.connection_dict.itervalues()
if isinstance(c, Connection) and addr(c.connector) in addr_set) if isinstance(c, Connection) and addr(c.connector) in addr_set)
...@@ -311,7 +311,7 @@ class StorageApplication(ServerNode, neo.storage.app.Application): ...@@ -311,7 +311,7 @@ class StorageApplication(ServerNode, neo.storage.app.Application):
checksum_dict = dict(dm.query("SELECT id, hash FROM data")) checksum_dict = dict(dm.query("SELECT id, hash FROM data"))
assert set(dm._uncommitted_data).issubset(checksum_dict) assert set(dm._uncommitted_data).issubset(checksum_dict)
get = dm._uncommitted_data.get get = dm._uncommitted_data.get
return dict((str(v), get(k, 0)) for k, v in checksum_dict.iteritems()) return {str(v): get(k, 0) for k, v in checksum_dict.iteritems()}
class ClientApplication(Node, neo.client.app.Application): class ClientApplication(Node, neo.client.app.Application):
......
...@@ -66,7 +66,7 @@ class ReplicationTests(NEOThreadedTest): ...@@ -66,7 +66,7 @@ class ReplicationTests(NEOThreadedTest):
np = pt.getPartitions() np = pt.getPartitions()
self.assertEqual(np, upstream_pt.getPartitions()) self.assertEqual(np, upstream_pt.getPartitions())
checked = 0 checked = 0
source_dict = dict((x.uuid, x) for x in cluster.upstream.storage_list) source_dict = {x.uuid: x for x in cluster.upstream.storage_list}
for storage in cluster.storage_list: for storage in cluster.storage_list:
self.assertEqual(np, storage.pt.getPartitions()) self.assertEqual(np, storage.pt.getPartitions())
for partition in pt.getAssignedPartitionList(storage.uuid): for partition in pt.getAssignedPartitionList(storage.uuid):
...@@ -349,7 +349,7 @@ class ReplicationTests(NEOThreadedTest): ...@@ -349,7 +349,7 @@ class ReplicationTests(NEOThreadedTest):
cluster.start() cluster.start()
cluster.populate([range(np*2)] * tid_count) cluster.populate([range(np*2)] * tid_count)
cluster.client.setPoll(0) cluster.client.setPoll(0)
storage_dict = dict((x.uuid, x) for x in cluster.storage_list) storage_dict = {x.uuid: x for x in cluster.storage_list}
cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None) cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None)
cluster.tic() cluster.tic()
check(ClusterStates.RUNNING, 0) check(ClusterStates.RUNNING, 0)
......
...@@ -9,7 +9,6 @@ Framework :: ZODB ...@@ -9,7 +9,6 @@ Framework :: ZODB
Intended Audience :: Developers Intended Audience :: Developers
License :: OSI Approved :: GNU General Public License (GPL) License :: OSI Approved :: GNU General Public License (GPL)
Operating System :: POSIX :: Linux Operating System :: POSIX :: Linux
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7 Programming Language :: Python :: 2.7
Topic :: Database Topic :: Database
Topic :: Software Development :: Libraries :: Python Modules Topic :: Software Development :: Libraries :: Python Modules
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment