Reduce size of UUIDs to 4-bytes

CHANGES

@@ -31,6 +31,8 @@ Other changes are:
 - Adding and removing master nodes is now easier: unknown incoming master nodes
   are now accepted instead of rejected, and nodes can be given a path to a file
   that maintains a list of known master nodes.
+- Node UUIDs have been shortened from 16 to 4 bytes, for better performance and
+  easier debugging.
 
 Also contains code clean-ups and bugfixes.
 

TODO

@@ -30,11 +30,6 @@ RC  - Review output of pylint (CODE)
       Consider the need to implement a keep-alive system (packets sent
       automatically when there is no activity on the connection for a period
       of time).
-    - Factorise packet data when sending partition table cells (BANDWITH)
-      Currently, each cell in a partition table update contains UUIDs of all
-      involved nodes.
-      It must be changed to a correspondance table using shorter keys (sent
-      in the packet) to avoid repeating the same UUIDs many times.
     - Consider using multicast for cluster-wide notifications. (BANDWITH)
       Currently, multi-receivers notifications are sent in unicast to each
       receiver. Multicast should be used.

UPGRADE

@@ -6,6 +6,7 @@ compatibility or transparent migration, so you will have to use the following
 SQL commands to migrate each storage from NEO 0.10.x::
 
   -- make sure 'tobj' & 'ttrans' are empty first
+  -        and all storages have up-to-date partition tables
   CREATE TABLE new_data (id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, hash BINARY(20) NOT NULL UNIQUE, compression TINYINT UNSIGNED NULL, value LONGBLOB NULL) ENGINE = InnoDB SELECT DISTINCT obj.hash as hash, compression, value FROM obj, data WHERE obj.hash=data.hash ORDER BY serial;
   DROP TABLE data;
   RENAME TABLE new_data TO data;
@@ -17,6 +18,12 @@ SQL commands to migrate each storage from NEO 0.10.x::
   UPDATE trans SET ttid=tid;
   ALTER TABLE ttrans ADD COLUMN ttid BIGINT UNSIGNED NOT NULL;
 
+  CREATE TEMPORARY TABLE uuid (new INT NOT NULL AUTO_INCREMENT PRIMARY KEY, old CHAR(32) NOT NULL, KEY (old)) ENGINE = InnoDB SELECT DISTINCT uuid as old FROM pt ORDER BY uuid;
+  ALTER TABLE pt DROP PRIMARY KEY, CHANGE uuid old CHAR(32) NOT NULL, ADD uuid INT NOT NULL after rid;
+  UPDATE pt, uuid SET pt.uuid=uuid.new WHERE pt.old=uuid.old;
+  ALTER TABLE pt DROP old, ADD PRIMARY KEY (rid, uuid);
+  UPDATE config, uuid SET config.value=uuid.new WHERE config.name='uuid' AND uuid.old=config.value;
+
 NEO 0.10
 ========
 

neo/admin/handler.py

@@ -16,9 +16,8 @@
 
 from neo.lib import logging, protocol
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import Packets, Errors
+from neo.lib.protocol import uuid_str, Packets, Errors
 from neo.lib.exception import PrimaryFailure
-from neo.lib.util import dump
 
 def check_primary_master(func):
     def wrapper(self, *args, **kw):
@@ -38,7 +37,7 @@ class AdminEventHandler(EventHandler):
     @check_primary_master
     def askPartitionList(self, conn, min_offset, max_offset, uuid):
         logging.info("ask partition list from %s to %s for %s",
-                     min_offset, max_offset, dump(uuid))
+                     min_offset, max_offset, uuid_str(uuid))
         self.app.sendPartitionTable(conn, min_offset, max_offset, uuid)
 
     @check_primary_master
@@ -56,7 +55,7 @@ class AdminEventHandler(EventHandler):
 
     @check_primary_master
     def setNodeState(self, conn, uuid, state, modify_partition_table):
-        logging.info("set node state for %s-%s", dump(uuid), state)
+        logging.info("set node state for %s-%s", uuid_str(uuid), state)
         node = self.app.nm.getByUUID(uuid)
         if node is None:
             raise protocol.ProtocolError('invalid uuid')

neo/lib/bootstrap.py

@@ -18,8 +18,7 @@ from time import sleep
 
 from . import logging
 from .handler import EventHandler
-from .protocol import Packets
-from .util import dump
+from .protocol import uuid_str, Packets
 from .connection import ClientConnection
 
 NO_SERVER = ('0.0.0.0', 0)
@@ -115,7 +114,7 @@ class BootstrapManager(EventHandler):
         if self.uuid != your_uuid:
             # got an uuid from the primary master
             self.uuid = your_uuid
-            logging.info('Got a new UUID: %s', dump(self.uuid))
+            logging.info('Got a new UUID: %s', uuid_str(self.uuid))
         self.accepted = True
 
     def getPrimaryConnection(self, connector_handler):

neo/lib/config.py

@@ -84,7 +84,9 @@ class ConfigurationManager(object):
 
     def getUUID(self):
         # only from command line
-        return util.bin(self.argument_list.get('uuid', None))
+        uuid = self.argument_list.get('uuid', None)
+        if uuid:
+            return int(uuid)
 
     def getUpstreamCluster(self):
         return self.__get('upstream_cluster', True)

neo/lib/connection.py

@@ -23,8 +23,9 @@ from .connector import ConnectorException, ConnectorTryAgainException, \
         ConnectorConnectionClosedException
 from .locking import RLock
 from .profiling import profiler_decorator
-from .protocol import Errors, PacketMalformedError, Packets, ParserState
-from .util import dump, ReadBuffer
+from .protocol import uuid_str, Errors, \
+        PacketMalformedError, Packets, ParserState
+from .util import ReadBuffer
 
 CRITICAL_TIMEOUT = 30
 
@@ -297,7 +298,7 @@ class BaseConnection(object):
         address = self.addr and '%s:%d' % self.addr or '?'
         return '<%s(uuid=%s, address=%s, closed=%s, handler=%s) at %x>' % (
             self.__class__.__name__,
-            dump(self.getUUID()),
+            uuid_str(self.getUUID()),
             address,
             int(self.isClosed()),
             self.getHandler(),

neo/lib/logger.py

@@ -21,7 +21,6 @@
 #          Fortunately, SQLite allow multiple process to access the same DB,
 #          so an external tool should be able to dump and empty tables.
 
-from binascii import b2a_hex
 from collections import deque
 from functools import wraps
 from logging import getLogger, Formatter, Logger, LogRecord, StreamHandler, \
@@ -117,6 +116,9 @@ class NEOLogger(Logger):
     def setup(self, filename=None, reset=False):
         self._acquire()
         try:
+            from . import protocol as p
+            global uuid_str
+            uuid_str = p.uuid_str
             if self.db is not None:
                 self.db.close()
                 if not filename:
@@ -153,7 +155,6 @@ class NEOLogger(Logger):
                         date REAL PRIMARY KEY NOT NULL,
                         text BLOB NOT NULL)
                   """)
-                from . import protocol as p
                 with open(inspect.getsourcefile(p)) as p:
                     p = buffer(bz2.compress(p.read()))
                 for t, in q("SELECT text FROM protocol ORDER BY date DESC"):
@@ -172,7 +173,7 @@ class NEOLogger(Logger):
         if type(r) is PacketRecord:
             ip, port = r.addr
             peer = '%s %s (%s:%u)' % ('>' if r.outgoing else '<',
-                                      r.uuid and b2a_hex(r.uuid), ip, port)
+                                      uuid_str(r.uuid), ip, port)
             self.db.execute("INSERT INTO packet VALUES (?,?,?,?,?,?)",
                 (r.created, r._name, r.msg_id, r.code, peer, buffer(r.msg)))
         else:

neo/lib/node.py

@@ -19,8 +19,7 @@ from os.path import exists, getsize
 import json
 
 from . import attributeTracker, logging
-from .util import dump
-from .protocol import NodeTypes, NodeStates, ProtocolError
+from .protocol import uuid_str, NodeTypes, NodeStates, ProtocolError
 
 
 class Node(object):
@@ -166,7 +165,7 @@ class Node(object):
     def __repr__(self):
         return '<%s(uuid=%s, address=%s, state=%s, connection=%r) at %x>' % (
             self.__class__.__name__,
-            dump(self._uuid),
+            uuid_str(self._uuid),
             self._address,
             self._state,
             self._connection,
@@ -511,16 +510,16 @@ class NodeManager(object):
             elif by_address is None:
                 node = by_uuid
             else:
-                raise ValueError('Got different nodes for uuid %r: %r and '
-                    'address %r: %r.' % (dump(uuid), by_uuid, address,
+                raise ValueError('Got different nodes for uuid %s: %r and '
+                    'address %r: %r.' % (uuid_str(uuid), by_uuid, address,
                     by_address))
             if uuid is not None:
                 node_uuid = node.getUUID()
                 if node_uuid is None:
                     node.setUUID(uuid)
                 elif node_uuid != uuid:
-                    raise ValueError('Expected uuid %r on node %r' % (
-                        dump(uuid), node))
+                    raise ValueError('Expected uuid %s on node %r' % (
+                        uuid_str(uuid), node))
             if address is not None:
                 node_address = node.getAddress()
                 if node_address is None:
@@ -574,7 +573,7 @@ class NodeManager(object):
             node_by_addr = self.getByAddress(addr)
             node = node_by_uuid or node_by_addr
 
-            log_args = (node_type, dump(uuid), addr, state)
+            log_args = node_type, uuid_str(uuid), addr, state
             if node is None:
                 if state == NodeStates.DOWN:
                     logging.debug('NOT creating node %s %s %s %s', *log_args)
@@ -606,10 +605,12 @@ class NodeManager(object):
 
     def log(self):
         logging.info('Node manager : %u nodes', len(self._node_set))
-        for node in sorted(list(self._node_set)):
-            uuid = dump(node.getUUID()) or '-' * 32
+        node_list = [(node, uuid_str(node.getUUID()))
+                     for node in sorted(self._node_set)]
+        max_len = max(len(x[1]) for x in node_list)
+        for node, uuid in node_list:
             address = node.getAddress() or ''
             if address:
                 address = '%s:%d' % address
-            logging.info(' * %32s | %8s | %22s | %s',
-                uuid, node.getType(), address, node.getState())
+            logging.info(' * %*s | %8s | %22s | %s',
+                max_len, uuid, node.getType(), address, node.getState())

neo/lib/protocol.py

@@ -26,7 +26,7 @@ except ImportError:
     pass
 
 # The protocol version (major, minor).
-PROTOCOL_VERSION = (9, 1)
+PROTOCOL_VERSION = (10, 1)
 
 # Size restrictions.
 MIN_PACKET_SIZE = 10
@@ -154,7 +154,7 @@ cell_state_prefix_dict = {
 }
 
 # Other constants.
-INVALID_UUID = '\0' * 16
+INVALID_UUID = 0
 INVALID_TID = '\xff' * 8
 INVALID_OID = '\xff' * 8
 INVALID_PARTITION = 0xffffffff
@@ -166,12 +166,24 @@ OID_LEN = len(INVALID_OID)
 TID_LEN = len(INVALID_TID)
 MAX_TID = '\x7f' + '\xff' * 7 # SQLite does not accept numbers above 2^63-1
 
+# High-order byte:
+# 7 6 5 4 3 2 1 0
+# | | | | +-+-+-+-- reserved (0)
+# | +-+-+---------- node type
+# +---------------- temporary if negative
+# UUID namespaces are required to prevent conflicts when the master generate
+# new uuid before it knows uuid of existing storage nodes. So only the high
+# order bit is really important and the 31 other bits could be random.
+# Extra namespace information and non-randomness of 3 LOB help to read logs.
 UUID_NAMESPACES = {
-    NodeTypes.STORAGE: 'S',
-    NodeTypes.MASTER: 'M',
-    NodeTypes.CLIENT: 'C',
-    NodeTypes.ADMIN: 'A',
+    NodeTypes.STORAGE: 0x00,
+    NodeTypes.MASTER: -0x10,
+    NodeTypes.CLIENT: -0x20,
+    NodeTypes.ADMIN: -0x30,
 }
+uuid_str = (lambda ns: lambda uuid:
+    ns[uuid >> 24] + str(uuid & 0xffffff) if uuid else str(uuid)
+    )(dict((v, str(k)[0]) for k, v in UUID_NAMESPACES.iteritems()))
 
 class ProtocolError(Exception):
     """ Base class for protocol errors, close the connection """
@@ -586,21 +598,18 @@ class PChecksum(PItem):
     def _decode(self, reader):
         return reader(20)
 
-class PUUID(PItem):
+class PUUID(PStructItem):
     """
-        An UUID (node identifier)
+        An UUID (node identifier, 4-bytes signed integer)
     """
+    def __init__(self, name):
+        PStructItem.__init__(self, name, '!l')
+
     def _encode(self, writer, uuid):
-        if uuid is None:
-            uuid = INVALID_UUID
-        assert len(uuid) == 16, (len(uuid), uuid)
-        writer(uuid)
+        writer(self.pack(uuid or 0))
 
     def _decode(self, reader):
-        uuid = reader(16)
-        if uuid == INVALID_UUID:
-            uuid = None
-        return uuid
+        return self.unpack(reader(self.size))[0] or None
 
 class PTID(PItem):
     """

neo/lib/pt.py

@@ -18,8 +18,8 @@ import math
 from functools import wraps
 
 from . import logging, protocol
-from .protocol import CellStates
-from .util import dump, u64
+from .protocol import uuid_str, CellStates
+from .util import u64
 from .locking import RLock
 
 class PartitionTableException(Exception):
@@ -36,7 +36,7 @@ class Cell(object):
 
     def __repr__(self):
         return "<Cell(uuid=%s, address=%s, state=%s)>" % (
-            dump(self.getUUID()),
+            uuid_str(self.getUUID()),
             self.getAddress(),
             self.getState(),
         )
@@ -226,7 +226,7 @@ class PartitionTable(object):
         self._id = ptid
         for offset, uuid, state in cell_list:
             node = nm.getByUUID(uuid)
-            assert node is not None, 'No node found for uuid %r' % (dump(uuid), )
+            assert node is not None, 'No node found for uuid ' + uuid_str(uuid)
             self.setCell(offset, node, state)
         logging.debug('partition table updated (ptid=%s)', ptid)
         self.log()
@@ -260,7 +260,7 @@ class PartitionTable(object):
         width under 80 column).
         """
         node_list = sorted(self.count_dict)
-        result = ['pt: node %u: %s, %s' % (i, dump(node.getUUID()),
+        result = ['pt: node %u: %s, %s' % (i, uuid_str(node.getUUID()),
                      protocol.node_state_prefix_dict[node.getState()])
                   for i, node in enumerate(node_list)]
         append = result.append

neo/master/app.py

@@ -20,7 +20,7 @@ from time import time
 from neo.lib import logging
 from neo.lib.connector import getConnectorHandler
 from neo.lib.debug import register as registerLiveDebugger
-from neo.lib.protocol import UUID_NAMESPACES, ZERO_TID, NotReadyError
+from neo.lib.protocol import uuid_str, UUID_NAMESPACES, ZERO_TID, NotReadyError
 from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets
 from neo.lib.node import NodeManager
 from neo.lib.event import EventManager
@@ -335,7 +335,7 @@ class Application(object):
 
         if self.uuid is None:
             self.uuid = self.getNewUUID(None, self.server, NodeTypes.MASTER)
-            logging.info('My UUID: ' + dump(self.uuid))
+            logging.info('My UUID: ' + uuid_str(self.uuid))
         else:
             in_conflict = self.nm.getByUUID(self.uuid)
             if in_conflict is not None:
@@ -443,14 +443,16 @@ class Application(object):
         self.cluster_state = state
 
     def getNewUUID(self, uuid, address, node_type):
+        getByUUID = self.nm.getByUUID
         if None != uuid != self.uuid:
-            node = self.nm.getByUUID(uuid)
+            node = getByUUID(uuid)
             if node is None or node.getAddress() == address:
                 return uuid
-        while True:
-            uuid = UUID_NAMESPACES[node_type] + os.urandom(15)
-            if uuid != self.uuid and self.nm.getByUUID(uuid) is None:
+        hob = UUID_NAMESPACES[node_type]
+        for uuid in xrange((hob << 24) + 1, hob + 0x10 << 24):
+            if uuid != self.uuid and getByUUID(uuid) is None:
                 return uuid
+        raise RuntimeError
 
     def getClusterState(self):
         return self.cluster_state

neo/master/backup_app.py

@@ -22,7 +22,7 @@ from neo.lib.connector import getConnectorHandler
 from neo.lib.exception import PrimaryFailure
 from neo.lib.node import NodeManager
 from neo.lib.protocol import CellStates, ClusterStates, NodeTypes, Packets
-from neo.lib.protocol import INVALID_TID, ZERO_TID
+from neo.lib.protocol import uuid_str, INVALID_TID, ZERO_TID
 from neo.lib.util import add64, dump
 from .app import StateChangedException
 from .pt import PartitionTable
@@ -154,9 +154,9 @@ class BackupApplication(object):
                 cell.replicating = tid
                 if cell.backup_tid < tid:
                     logging.debug(
-                        "ask %s to replicate partition %u up to %s from %r",
-                        dump(cell.getUUID()), offset,  dump(tid),
-                        dump(primary_node.getUUID()))
+                        "ask %s to replicate partition %u up to %s from %s",
+                        uuid_str(cell.getUUID()), offset,  dump(tid),
+                        uuid_str(primary_node.getUUID()))
                     cell.getNode().getConnection().notify(p)
             trigger_set.add(primary_node)
         for node in trigger_set:
@@ -238,7 +238,7 @@ class BackupApplication(object):
                 address_set.add(addr)
             source_dict[offset] = addr
             logging.debug("ask %s to replicate partition %u up to %s from %r",
-                dump(node.getUUID()), offset,  dump(tid), addr)
+                uuid_str(node.getUUID()), offset,  dump(tid), addr)
         node.getConnection().notify(Packets.Replicate(
             tid, self.name, source_dict))
 
@@ -272,9 +272,9 @@ class BackupApplication(object):
                 if tid < max_tid:
                     cell.replicating = max_tid
                     logging.debug(
-                        "ask %s to replicate partition %u up to %s from %r",
-                        dump(node.getUUID()), offset,  dump(max_tid),
-                        dump(primary_node.getUUID()))
+                        "ask %s to replicate partition %u up to %s from %s",
+                        uuid_str(node.getUUID()), offset,  dump(max_tid),
+                        uuid_str(primary_node.getUUID()))
                     node.getConnection().notify(Packets.Replicate(max_tid,
                         '', {offset: primary_node.getAddress()}))
             else:
@@ -288,7 +288,7 @@ class BackupApplication(object):
                             cell.replicating = tid
                             logging.debug(
                                 "ask %s to replicate partition %u up to %s from"
-                                " %r", dump(cell.getUUID()), offset,  dump(tid),
-                                dump(node.getUUID()))
+                                " %s", uuid_str(cell.getUUID()), offset,
+                                dump(tid), uuid_str(node.getUUID()))
                             cell.getNode().getConnection().notify(p)
         return result

neo/master/handlers/__init__.py

@@ -16,10 +16,9 @@
 
 from neo.lib import logging
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import (NodeTypes, NodeStates, Packets,
+from neo.lib.protocol import (uuid_str, NodeTypes, NodeStates, Packets,
     BrokenNodeDisallowedError,
 )
-from neo.lib.util import dump
 
 class MasterHandler(EventHandler):
     """This class implements a generic part of the event handlers."""
@@ -100,7 +99,7 @@ class BaseServiceHandler(MasterHandler):
         assert new_state in (NodeStates.TEMPORARILY_DOWN, NodeStates.DOWN,
             NodeStates.BROKEN), new_state
         assert node.getState() not in (NodeStates.TEMPORARILY_DOWN,
-            NodeStates.DOWN, NodeStates.BROKEN), (dump(self.app.uuid),
+            NodeStates.DOWN, NodeStates.BROKEN), (uuid_str(self.app.uuid),
             node.whoSetState(), new_state)
         was_pending = node.isPending()
         node.setState(new_state)

neo/master/handlers/administration.py

@@ -20,7 +20,7 @@ from . import MasterHandler
 from ..app import StateChangedException
 from neo.lib import logging
 from neo.lib.protocol import ClusterStates, NodeStates, Packets, ProtocolError
-from neo.lib.protocol import Errors
+from neo.lib.protocol import Errors, uuid_str
 from neo.lib.util import dump
 
 CLUSTER_STATE_WORKFLOW = {
@@ -73,7 +73,7 @@ class AdministrationHandler(MasterHandler):
 
     def setNodeState(self, conn, uuid, state, modify_partition_table):
         logging.info("set node state for %s-%s : %s",
-                     dump(uuid), state, modify_partition_table)
+                     uuid_str(uuid), state, modify_partition_table)
         app = self.app
         node = app.nm.getByUUID(uuid)
         if node is None:
@@ -127,7 +127,7 @@ class AdministrationHandler(MasterHandler):
         app.broadcastNodesInformation([node])
 
     def addPendingNodes(self, conn, uuid_list):
-        uuids = ', '.join(map(dump, uuid_list))
+        uuids = ', '.join(map(uuid_str, uuid_list))
         logging.debug('Add nodes %s', uuids)
         app = self.app
         nm = app.nm
@@ -148,7 +148,7 @@ class AdministrationHandler(MasterHandler):
             logging.warning('No nodes added')
             conn.answer(Errors.Ack('No nodes added'))
             return
-        uuids = ', '.join(map(dump, uuid_set))
+        uuids = ', '.join(map(uuid_str, uuid_set))
         logging.info('Adding nodes %s', uuids)
         # switch nodes to running state
         node_list = map(nm.getByUUID, uuid_set)

neo/master/handlers/election.py

@@ -15,12 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from neo.lib import logging
-from neo.lib.protocol import NodeTypes, NodeStates, Packets
+from neo.lib.protocol import uuid_str, NodeTypes, NodeStates, Packets
 from neo.lib.protocol import NotReadyError, ProtocolError, \
                               UnexpectedPacketError
 from neo.lib.exception import ElectionFailure
 from neo.lib.handler import EventHandler
-from neo.lib.util import dump
 from . import MasterHandler
 
 class BaseElectionHandler(EventHandler):
@@ -58,8 +57,8 @@ class ClientElectionHandler(BaseElectionHandler):
     def connectionFailed(self, conn):
         addr = conn.getAddress()
         node = self.app.nm.getByAddress(addr)
-        assert node is not None, (dump(self.app.uuid), addr)
-        assert node.isUnknown(), (dump(self.app.uuid), node.whoSetState(),
+        assert node is not None, (uuid_str(self.app.uuid), addr)
+        assert node.isUnknown(), (uuid_str(self.app.uuid), node.whoSetState(),
           node)
         # connection never success, node is still in unknown state
         self.app.negotiating_master_node_set.discard(addr)
@@ -92,7 +91,7 @@ class ClientElectionHandler(BaseElectionHandler):
             if app.server == address:
                 # This is self.
                 assert node.getAddress() != primary or uuid == your_uuid, (
-                    dump(uuid), dump(your_uuid))
+                    uuid_str(uuid), uuid_str(your_uuid))
                 continue
             n = app.nm.getByAddress(address)
             if n is None:

neo/master/handlers/identification.py

@@ -16,9 +16,8 @@
 
 import neo
 from neo.lib import logging
-from neo.lib.util import dump
 from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, Packets, \
-    NotReadyError, ProtocolError
+    NotReadyError, ProtocolError, uuid_str
 from . import MasterHandler
 
 class IdentificationHandler(MasterHandler):
@@ -68,7 +67,7 @@ class IdentificationHandler(MasterHandler):
             raise NotImplementedError(node_type)
 
         uuid = app.getNewUUID(uuid, address, node_type)
-        logging.info('Accept a' + human_readable_node_type + dump(uuid))
+        logging.info('Accept a' + human_readable_node_type + uuid_str(uuid))
         if node is None:
             node = node_ctor(uuid=uuid, address=address)
         node.setUUID(uuid)

neo/master/handlers/secondary.py

@@ -17,9 +17,8 @@
 from . import MasterHandler
 from neo.lib.handler import EventHandler
 from neo.lib.exception import ElectionFailure, PrimaryFailure
-from neo.lib.protocol import NodeTypes, Packets
+from neo.lib.protocol import NodeTypes, Packets, uuid_str
 from neo.lib import logging
-from neo.lib.util import dump
 
 class SecondaryMasterHandler(MasterHandler):
     """ Handler used by primary to handle secondary masters"""
@@ -97,7 +96,7 @@ class PrimaryHandler(EventHandler):
 
         if your_uuid != app.uuid:
             app.uuid = your_uuid
-            logging.info('My UUID: ' + dump(your_uuid))
+            logging.info('My UUID: ' + uuid_str(your_uuid))
 
         node.setUUID(uuid)
 

neo/master/transactions.py

@@ -19,6 +19,7 @@ from struct import pack, unpack
 from neo.lib.protocol import ZERO_TID
 from datetime import timedelta, datetime
 from neo.lib import logging
+from neo.lib.protocol import uuid_str
 from neo.lib.util import dump, u64, p64
 
 TID_LOW_OVERFLOW = 2**32
@@ -118,7 +119,7 @@ class Transaction(object):
                 self._node,
                 dump(self._tid),
                 map(dump, self._oid_list or ()),
-                map(dump, self._uuid_set or ()),
+                map(uuid_str, self._uuid_set or ()),
                 time() - self._birth,
                 id(self),
         )
@@ -417,7 +418,7 @@ class TransactionManager(object):
             If transaction is completely locked, calls function given at
             instanciation time.
         """
-        logging.debug('Lock TXN %s for %s', dump(ttid), dump(uuid))
+        logging.debug('Lock TXN %s for %s', dump(ttid), uuid_str(uuid))
         assert ttid in self._ttid_dict, "Transaction not started"
         txn = self._ttid_dict[ttid]
         if txn.lock(uuid) and self._queue[0][1] == ttid:

neo/neoctl/app.py

@@ -15,8 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from .neoctl import NeoCTL, NotReadyException
-from neo.lib.util import bin, dump, p64
-from neo.lib.protocol import ClusterStates, NodeStates, NodeTypes, ZERO_TID
+from neo.lib.util import bin, p64
+from neo.lib.protocol import uuid_str, ClusterStates, NodeStates, NodeTypes, \
+    ZERO_TID
 
 action_dict = {
     'print': {
@@ -57,7 +58,7 @@ class TerminalNeoCTL(object):
 
     def formatRowList(self, row_list):
         return '\n'.join('%03d | %s' % (offset,
-            ''.join('%s - %s |' % (dump(uuid), state)
+            ''.join('%s - %s |' % (uuid_str(uuid), state)
             for (uuid, state) in cell_list))
             for (offset, cell_list) in row_list)
 
@@ -69,13 +70,10 @@ class TerminalNeoCTL(object):
             if address is None:
                 address = (None, None)
             ip, port = address
-            result.append('%s - %s - %s:%s - %s' % (node_type, dump(uuid), ip,
-                                                    port, state))
+            result.append('%s - %s - %s:%s - %s' % (node_type, uuid_str(uuid),
+                                                    ip, port, state))
         return '\n'.join(result)
 
-    def formatUUID(self, uuid):
-        return dump(uuid)
-
     # Actual actions
     def getPartitionRowList(self, params):
         """
@@ -185,7 +183,7 @@ class TerminalNeoCTL(object):
         """
           Get primary master node.
         """
-        return self.formatUUID(self.neoctl.getPrimary())
+        return uuid_str(self.neoctl.getPrimary())
 
     def checkReplicas(self, params):
         """

neo/scripts/neolog.py

@@ -50,8 +50,8 @@ class main(object):
         g = {}
         exec bz2.decompress(*q("SELECT text FROM protocol WHERE date<?"
                                " ORDER BY date DESC", (date,)).next()) in g
-        self.Packets = g['Packets']
-        self.PacketMalformedError = g['PacketMalformedError']
+        for x in 'uuid_str', 'Packets', 'PacketMalformedError':
+            setattr(self, x, g[x])
         try:
             self._next_protocol, = q("SELECT date FROM protocol WHERE date>=?",
                                      (date,)).next()
@@ -96,11 +96,17 @@ class main(object):
         return "%s (%s)" % (code, message),
 
     def notifyNodeInformation(self, node_list):
-        for node_type, address, uuid, state in node_list:
-            address = '%s:%u' % address if address else '?'
-            if uuid is not None:
-                uuid = b2a_hex(uuid)
-            yield ' ! %s | %8s | %22s | %s' % (uuid, node_type, address, state)
+        node_list.sort(key=lambda x: x[2])
+        node_list = [(self.uuid_str(uuid), str(node_type),
+                      '%s:%u' % address if address else '?', state)
+                     for node_type, address, uuid, state in node_list]
+        if node_list:
+            t = ' ! %%%us | %%%us | %%%us | %%s' % (
+                max(len(x[0]) for x in node_list),
+                max(len(x[1]) for x in node_list),
+                max(len(x[2]) for x in node_list))
+            return map(t.__mod__, node_list)
+        return ()
 
 
 if __name__ == "__main__":

neo/storage/app.py

@@ -18,7 +18,7 @@ import sys
 from collections import deque
 
 from neo.lib import logging
-from neo.lib.protocol import NodeTypes, CellStates, Packets
+from neo.lib.protocol import uuid_str, CellStates, NodeTypes, Packets
 from neo.lib.node import NodeManager
 from neo.lib.event import EventManager
 from neo.lib.connection import ListeningConnection
@@ -138,7 +138,7 @@ class Application(object):
             self.pt = PartitionTable(num_partitions, num_replicas)
 
         logging.info('Configuration loaded:')
-        logging.info('UUID      : %s', dump(self.uuid))
+        logging.info('UUID      : %s', uuid_str(self.uuid))
         logging.info('PTID      : %s', dump(ptid))
         logging.info('Name      : %s', self.name)
         logging.info('Partitions: %s', num_partitions)
@@ -233,7 +233,7 @@ class Application(object):
         (node, conn, uuid, num_partitions, num_replicas) = data
         self.master_node = node
         self.master_conn = conn
-        logging.info('I am %s', dump(uuid))
+        logging.info('I am %s', uuid_str(uuid))
         self.uuid = uuid
         self.dm.setUUID(uuid)
 

neo/storage/database/manager.py

@@ -104,13 +104,15 @@ class DatabaseManager(object):
         """
             Load an UUID from a database.
         """
-        return util.bin(self.getConfiguration('uuid'))
+        uuid = self.getConfiguration('uuid')
+        if uuid is not None:
+            return int(uuid)
 
     def setUUID(self, uuid):
         """
             Store an UUID into a database.
         """
-        self.setConfiguration('uuid', util.dump(uuid))
+        self.setConfiguration('uuid', str(uuid))
 
     def getNumPartitions(self):
         """
@@ -188,7 +190,7 @@ class DatabaseManager(object):
         return self.setConfiguration('backup_tid', util.dump(backup_tid))
 
     def getPartitionTable(self):
-        """Return a whole partition table as a tuple of rows. Each row
+        """Return a whole partition table as a sequence of rows. Each row
         is again a tuple of an offset (row ID), an UUID of a storage
         node, and a cell state."""
         raise NotImplementedError

neo/storage/database/mysqldb.py

@@ -165,7 +165,7 @@ class MySQLDatabaseManager(DatabaseManager):
         # The table "pt" stores a partition table.
         q("""CREATE TABLE IF NOT EXISTS pt (
                  rid INT UNSIGNED NOT NULL,
-                 uuid CHAR(32) NOT NULL,
+                 uuid INT NOT NULL,
                  state TINYINT UNSIGNED NOT NULL,
                  PRIMARY KEY (rid, uuid)
              ) ENGINE = InnoDB""")
@@ -267,13 +267,7 @@ class MySQLDatabaseManager(DatabaseManager):
             return -1
 
     def getPartitionTable(self):
-        q = self.query
-        cell_list = q("""SELECT rid, uuid, state FROM pt""")
-        pt = []
-        for offset, uuid, state in cell_list:
-            uuid = util.bin(uuid)
-            pt.append((offset, uuid, state))
-        return pt
+        return self.query("SELECT * FROM pt")
 
     def _getLastTIDs(self, all=True):
         p64 = util.p64
@@ -340,21 +334,19 @@ class MySQLDatabaseManager(DatabaseManager):
         return serial, next_serial, compression, checksum, data, value_serial
 
     def doSetPartitionTable(self, ptid, cell_list, reset):
-        e = self.escape
         offset_list = []
         with self as q:
             if reset:
                 q("""TRUNCATE pt""")
             for offset, uuid, state in cell_list:
-                uuid = e(util.dump(uuid))
                 # TODO: this logic should move out of database manager
                 # add 'dropCells(cell_list)' to API and use one query
                 if state == CellStates.DISCARDED:
-                    q("""DELETE FROM pt WHERE rid = %d AND uuid = '%s'""" \
+                    q("""DELETE FROM pt WHERE rid = %d AND uuid = %d"""
                             % (offset, uuid))
                 else:
                     offset_list.append(offset)
-                    q("""INSERT INTO pt VALUES (%d, '%s', %d)
+                    q("""INSERT INTO pt VALUES (%d, %d, %d)
                             ON DUPLICATE KEY UPDATE state = %d""" \
                                     % (offset, uuid, state, state))
             self.setPTID(ptid)

neo/storage/database/sqlite.py

@@ -106,7 +106,7 @@ class SQLiteDatabaseManager(DatabaseManager):
         # The table "pt" stores a partition table.
         q("""CREATE TABLE IF NOT EXISTS pt (
                  rid INTEGER NOT NULL,
-                 uuid BLOB NOT NULL,
+                 uuid INTEGER NOT NULL,
                  state INTEGER NOT NULL,
                  PRIMARY KEY (rid, uuid))
           """)
@@ -203,9 +203,7 @@ class SQLiteDatabaseManager(DatabaseManager):
             return -1
 
     def getPartitionTable(self):
-        return [(offset, util.bin(uuid), state)
-            for offset, uuid, state in self.query(
-                "SELECT rid, uuid, state FROM pt")]
+        return self.query("SELECT * FROM pt")
 
     def _getLastTIDs(self, all=True):
         p64 = util.p64
@@ -275,7 +273,6 @@ class SQLiteDatabaseManager(DatabaseManager):
             if reset:
                 q("DELETE FROM pt")
             for offset, uuid, state in cell_list:
-                uuid = buffer(util.dump(uuid))
                 # TODO: this logic should move out of database manager
                 # add 'dropCells(cell_list)' to API and use one query
                 # WKRD: Why does SQLite need a statement journal file

neo/storage/handlers/__init__.py

@@ -16,9 +16,8 @@
 
 from neo.lib import logging
 from neo.lib.handler import EventHandler
-from neo.lib.util import dump
 from neo.lib.exception import PrimaryFailure, OperationFailure
-from neo.lib.protocol import NodeStates, NodeTypes
+from neo.lib.protocol import uuid_str, NodeStates, NodeTypes
 
 class BaseMasterHandler(EventHandler):
 
@@ -55,8 +54,8 @@ class BaseMasterHandler(EventHandler):
                 elif state == NodeStates.HIDDEN:
                     raise OperationFailure
             elif node_type == NodeTypes.CLIENT and state != NodeStates.RUNNING:
-                logging.info('Notified of non-running client, abort (%r)',
-                        dump(uuid))
+                logging.info('Notified of non-running client, abort (%s)',
+                        uuid_str(uuid))
                 self.app.tm.abortFor(uuid)
 
     def answerUnfinishedTransactions(self, conn, *args, **kw):

neo/storage/handlers/identification.py

@@ -16,9 +16,8 @@
 
 from neo.lib import logging
 from neo.lib.handler import EventHandler
-from neo.lib.protocol import NodeTypes, Packets, NotReadyError
+from neo.lib.protocol import uuid_str, NodeTypes, NotReadyError, Packets
 from neo.lib.protocol import ProtocolError, BrokenNodeDisallowedError
-from neo.lib.util import dump
 from .storage import StorageOperationHandler
 from .client import ClientOperationHandler
 
@@ -60,7 +59,7 @@ class IdentificationHandler(EventHandler):
             elif node_type == NodeTypes.STORAGE:
                 if node is None:
                     logging.error('reject an unknown storage node %s',
-                        dump(uuid))
+                        uuid_str(uuid))
                     raise NotReadyError
                 handler = StorageOperationHandler
             else:

neo/storage/transactions.py

@@ -17,7 +17,7 @@
 from time import time
 from neo.lib import logging
 from neo.lib.util import dump
-from neo.lib.protocol import ZERO_TID
+from neo.lib.protocol import uuid_str, ZERO_TID
 
 class ConflictError(Exception):
     """
@@ -55,15 +55,14 @@ class Transaction(object):
         self._checked_set = set()
 
     def __repr__(self):
-        return "<%s(ttid=%r, tid=%r, uuid=%r, locked=%r, age=%.2fs)> at %x" % (
-            self.__class__.__name__,
+        return "<%s(ttid=%r, tid=%r, uuid=%r, locked=%r, age=%.2fs) at 0x%x>" \
+         % (self.__class__.__name__,
             dump(self._ttid),
             dump(self._tid),
-            dump(self._uuid),
+            uuid_str(self._uuid),
             self.isLocked(),
             time() - self._birth,
-            id(self),
-        )
+            id(self))
 
     def addCheckedObject(self, oid):
         assert oid not in self._object_dict, dump(oid)
@@ -148,7 +147,7 @@ class TransactionManager(object):
         """
             Register a transaction, it may be already registered
         """
-        logging.debug('Register TXN %s for %s', dump(ttid), dump(uuid))
+        logging.debug('Register TXN %s for %s', dump(ttid), uuid_str(uuid))
         transaction = self._transaction_dict.get(ttid, None)
         if transaction is None:
             transaction = Transaction(uuid, ttid)
@@ -353,7 +352,7 @@ class TransactionManager(object):
         """
             Abort any non-locked transaction of a node
         """
-        logging.debug('Abort for %s', dump(uuid))
+        logging.debug('Abort for %s', uuid_str(uuid))
         # abort any non-locked transaction of this node
         for ttid in [x.getTTID() for x in self._uuid_dict.get(uuid, [])]:
             self.abort(ttid)

neo/tests/__init__.py

@@ -27,7 +27,7 @@ import transaction
 
 from mock import Mock
 from neo.lib import debug, logging, protocol
-from neo.lib.protocol import Packets
+from neo.lib.protocol import NodeTypes, Packets, UUID_NAMESPACES
 from neo.lib.util import getAddressType
 from time import time
 from struct import pack, unpack
@@ -159,6 +159,10 @@ class NeoUnitTestBase(NeoTestBase):
 
     local_ip = IP_VERSION_FORMAT_DICT[ADDRESS_TYPE]
 
+    def setUp(self):
+        self.uuid_dict = {}
+        NeoTestBase.setUp(self)
+
     def prepareDatabase(self, number, prefix='test_neo'):
         """ create empties databases """
         setupMySQLdb(['%s%u' % (prefix, i) for i in xrange(number)])
@@ -196,30 +200,26 @@ class NeoUnitTestBase(NeoTestBase):
                 'getAdapter': 'MySQL',
         })
 
-    def _makeUUID(self, prefix):
+    def getNewUUID(self, node_type):
         """
             Retuns a 16-bytes UUID according to namespace 'prefix'
         """
-        assert len(prefix) == 1
-        uuid = protocol.INVALID_UUID
-        while uuid[1:] == protocol.INVALID_UUID[1:]:
-            uuid = prefix + os.urandom(15)
-        return uuid
-
-    def getNewUUID(self):
-        return self._makeUUID('\0')
+        if node_type is None:
+            node_type = random.choice(NodeTypes)
+        self.uuid_dict[node_type] = uuid = 1 + self.uuid_dict.get(node_type, 0)
+        return uuid + (UUID_NAMESPACES[node_type] << 24)
 
     def getClientUUID(self):
-        return self._makeUUID('C')
+        return self.getNewUUID(NodeTypes.CLIENT)
 
     def getMasterUUID(self):
-        return self._makeUUID('M')
+        return self.getNewUUID(NodeTypes.MASTER)
 
     def getStorageUUID(self):
-        return self._makeUUID('S')
+        return self.getNewUUID(NodeTypes.STORAGE)
 
     def getAdminUUID(self):
-        return self._makeUUID('A')
+        return self.getNewUUID(NodeTypes.ADMIN)
 
     def getNextTID(self, ltid=None):
         return newTid(ltid)
@@ -236,12 +236,6 @@ class NeoUnitTestBase(NeoTestBase):
             return os.urandom(8)
         return pack('!Q', i)
 
-    def getTwoIDs(self):
-        """ Return a tuple of two sorted UUIDs """
-        # generate two ptid, first is lower
-        uuids = self.getNewUUID(), self.getNewUUID()
-        return min(uuids), max(uuids)
-
     def getFakeConnector(self, descriptor=None):
         return Mock({
             '__repr__': 'FakeConnector',

neo/tests/client/testClientApp.py

@@ -22,8 +22,8 @@ from .. import NeoUnitTestBase, buildUrlFromString, ADDRESS_TYPE
 from neo.client.app import Application
 from neo.client.exception import NEOStorageError, NEOStorageNotFoundError
 from neo.client.exception import NEOStorageDoesNotExistError
-from neo.lib.protocol import Packet, Packets, Errors, INVALID_TID, \
-    INVALID_PARTITION
+from neo.lib.protocol import NodeTypes, Packet, Packets, Errors, INVALID_TID, \
+    INVALID_PARTITION, UUID_NAMESPACES
 from neo.lib.util import makeChecksum, SOCKET_CONNECTORS_DICT
 import time
 
@@ -37,7 +37,7 @@ class Dispatcher(object):
 
 def _getMasterConnection(self):
     if self.master_conn is None:
-        self.uuid = 'C' * 16
+        self.uuid = 1 + (UUID_NAMESPACES[NodeTypes.CLIENT] << 24)
         self.num_partitions = 10
         self.num_replicas = 1
         self.pt = Mock({'getCellList': ()})
@@ -323,7 +323,7 @@ class ClientApplicationTests(NeoUnitTestBase):
 
     def test_store3(self):
         app = self.getApp()
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         oid = self.makeOID(11)
         tid = self.makeTID()
         txn = self.makeTransactionObject()
@@ -429,10 +429,9 @@ class ClientApplicationTests(NeoUnitTestBase):
         oid1 = self.makeOID(num_partitions + 1) # on partition 1, conflicting
         oid2 = self.makeOID(num_partitions + 2) # on partition 2
         # storage nodes
-        uuid1, uuid2, uuid3 = [self.getNewUUID() for _ in range(3)]
-        address1 = ('127.0.0.1', 10000)
-        address2 = ('127.0.0.1', 10001)
-        address3 = ('127.0.0.1', 10002)
+        address1 = ('127.0.0.1', 10000); uuid1 = self.getMasterUUID()
+        address2 = ('127.0.0.1', 10001); uuid2 = self.getStorageUUID()
+        address3 = ('127.0.0.1', 10002); uuid3 = self.getStorageUUID()
         app.nm.createMaster(address=address1, uuid=uuid1)
         app.nm.createStorage(address=address2, uuid=uuid2)
         app.nm.createStorage(address=address3, uuid=uuid3)
@@ -699,7 +698,7 @@ class ClientApplicationTests(NeoUnitTestBase):
     def test_undoLog(self):
         app = self.getApp()
         app.num_partitions = 2
-        uuid1, uuid2 = '\x00' * 15 + '\x01', '\x00' * 15 + '\x02'
+        uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
         # two nodes, two partition, two transaction, two objects :
         tid1, tid2 = self.makeTID(1), self.makeTID(2)
         oid1, oid2 = self.makeOID(1), self.makeOID(2)
@@ -778,7 +777,7 @@ class ClientApplicationTests(NeoUnitTestBase):
         # fifth packet : request node identification succeeded
         def _ask6(conn):
             app.master_conn = conn
-            app.uuid = 'C' * 16
+            app.uuid = 1 + (UUID_NAMESPACES[NodeTypes.CLIENT] << 24)
             app.trying_master_node = app.primary_master_node = Mock({
                 'getAddress': ('127.0.0.1', 10011),
                 '__str__': 'Fake master node',

neo/tests/client/testConnectionPool.py

@@ -26,10 +26,8 @@ class ConnectionPoolTests(NeoUnitTestBase):
     def test_removeConnection(self):
         app = None
         pool = ConnectionPool(app)
-        test_node_uuid = self.getNewUUID()
-        other_node_uuid = test_node_uuid
-        while other_node_uuid == test_node_uuid:
-            other_node_uuid = self.getNewUUID()
+        test_node_uuid = self.getStorageUUID()
+        other_node_uuid = self.getStorageUUID()
         test_node = Mock({'getUUID': test_node_uuid})
         other_node = Mock({'getUUID': other_node_uuid})
         # Test sanity check
@@ -50,9 +48,9 @@ class ConnectionPoolTests(NeoUnitTestBase):
 
     def test_CellSortKey(self):
         pool = ConnectionPool(None)
-        node_uuid_1 = self.getNewUUID()
-        node_uuid_2 = self.getNewUUID()
-        node_uuid_3 = self.getNewUUID()
+        node_uuid_1 = self.getStorageUUID()
+        node_uuid_2 = self.getStorageUUID()
+        node_uuid_3 = self.getStorageUUID()
         # We are connected to node 1
         pool.connection_dict[node_uuid_1] = None
         # A connection to node 3 failed, will be forgotten at 5

neo/tests/client/testMasterHandler.py

@@ -69,7 +69,7 @@ class MasterBootstrapHandlerTests(MasterHandlerTests):
     def test_acceptIdentification2(self):
         """ No UUID supplied """
         node, conn = self.getKnownMaster()
-        uuid = self.getNewUUID()
+        uuid = self.getMasterUUID()
         addr = conn.getAddress()
         self.checkProtocolErrorRaised(self.handler.acceptIdentification,
             conn, NodeTypes.MASTER, uuid, 100, 0, None,
@@ -79,9 +79,9 @@ class MasterBootstrapHandlerTests(MasterHandlerTests):
     def test_acceptIdentification3(self):
         """ identification accepted  """
         node, conn = self.getKnownMaster()
-        uuid = self.getNewUUID()
+        uuid = self.getMasterUUID()
         addr = conn.getAddress()
-        your_uuid = self.getNewUUID()
+        your_uuid = self.getClientUUID()
         self.handler.acceptIdentification(conn, NodeTypes.MASTER, uuid,
             100, 2, your_uuid, addr, [(addr, uuid)])
         self.assertEqual(self.app.uuid, your_uuid)

neo/tests/client/testStorageHandler.py

@@ -38,14 +38,14 @@ class StorageBootstrapHandlerTests(NeoUnitTestBase):
         self.handler = StorageBootstrapHandler(self.app)
         self.app.primary_master_node = node = Mock({
             'getConnection': self.getFakeConnection(),
-            'getUUID': self.getNewUUID(),
+            'getUUID': self.getMasterUUID(),
             'getAddress': (self.local_ip, 2999)
         })
         self._next_port = 3000
 
     def getKnownStorage(self):
         node = self.app.nm.createStorage(
-            uuid=self.getNewUUID(),
+            uuid=self.getStorageUUID(),
             address=(self.local_ip, self._next_port),
         )
         self._next_port += 1
@@ -172,7 +172,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
             {oid: set([tid, ])}).answerStoreObject(conn, 1, oid, tid_2)
 
     def test_answerStoreObject_4(self):
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         conn = self.getFakeConnection(uuid=uuid)
         oid = self.getOID(0)
         tid = self.getNextTID()
@@ -223,7 +223,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
             conn, 'message')
 
     def test_answerTIDs(self):
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         tid1 = self.getNextTID()
         tid2 = self.getNextTID(tid1)
         tid_list = [tid1, tid2]
@@ -233,7 +233,7 @@ class StorageAnswerHandlerTests(NeoUnitTestBase):
         self.assertEqual(tid_set, set(tid_list))
 
     def test_answerObjectUndoSerial(self):
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         conn = self.getFakeConnection(uuid=uuid)
         oid1 = self.getOID(1)
         oid2 = self.getOID(2)

neo/tests/functional/__init__.py

@@ -34,16 +34,19 @@ import psutil
 import neo.scripts
 from neo.neoctl.neoctl import NeoCTL, NotReadyException
 from neo.lib import logging
-from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates
+from neo.lib.protocol import ClusterStates, NodeTypes, CellStates, NodeStates, \
+    UUID_NAMESPACES
 from neo.lib.util import dump
 from .. import DB_USER, setupMySQLdb, NeoTestBase, buildUrlFromString, \
         ADDRESS_TYPE, IP_VERSION_FORMAT_DICT, getTempDirectory
 from ..cluster import SocketLock
 from neo.client.Storage import Storage
 
-NEO_MASTER = 'neomaster'
-NEO_STORAGE = 'neostorage'
-NEO_ADMIN = 'neoadmin'
+command_dict = {
+    NodeTypes.MASTER: 'neomaster',
+    NodeTypes.STORAGE: 'neostorage',
+    NodeTypes.ADMIN: 'neoadmin',
+}
 
 DELAY_SAFETY_MARGIN = 10
 MAX_START_TIME = 30
@@ -231,7 +234,7 @@ class NEOProcess(object):
           Note: for this change to take effect, the node must be restarted.
         """
         self.uuid = uuid
-        self.arg_dict['--uuid'] = dump(uuid)
+        self.arg_dict['--uuid'] = str(uuid)
 
     def isAlive(self):
         try:
@@ -252,7 +255,7 @@ class NEOCluster(object):
         self.adapter = adapter
         self.zodb_storage_list = []
         self.cleanup_on_delete = cleanup_on_delete
-        self.uuid_set = set()
+        self.uuid_dict = {}
         self.db_list = db_list
         if temp_dir is None:
             temp_dir = tempfile.mkdtemp(prefix='neo_')
@@ -280,7 +283,7 @@ class NEOCluster(object):
                 for x in master_node_list)
 
         # create admin node
-        self.__newProcess(NEO_ADMIN, {
+        self.__newProcess(NodeTypes.ADMIN, {
             '--cluster': self.cluster_name,
             '--logfile': os.path.join(self.temp_dir, 'admin.log'),
             '--bind': '%s:%d' % (buildUrlFromString(
@@ -289,7 +292,7 @@ class NEOCluster(object):
         })
         # create master nodes
         for i, port in enumerate(master_node_list):
-            self.__newProcess(NEO_MASTER, {
+            self.__newProcess(NodeTypes.MASTER, {
                 '--cluster': self.cluster_name,
                 '--logfile': os.path.join(self.temp_dir, 'master_%u.log' % i),
                 '--bind': '%s:%d' % (buildUrlFromString(
@@ -300,7 +303,7 @@ class NEOCluster(object):
             })
         # create storage nodes
         for i, db in enumerate(db_list):
-            self.__newProcess(NEO_STORAGE, {
+            self.__newProcess(NodeTypes.STORAGE, {
                 '--cluster': self.cluster_name,
                 '--logfile': os.path.join(self.temp_dir, 'storage_%u.log' % i),
                 '--bind': '%s:%d' % (buildUrlFromString(
@@ -313,11 +316,12 @@ class NEOCluster(object):
         # create neoctl
         self.neoctl = NeoCTL((self.local_ip, admin_port))
 
-    def __newProcess(self, command, arguments):
-        uuid = self.__allocateUUID()
+    def __newProcess(self, node_type, arguments):
+        self.uuid_dict[node_type] = uuid = 1 + self.uuid_dict.get(node_type, 0)
+        uuid += UUID_NAMESPACES[node_type] << 24
         arguments['--uuid'] = uuid
-        self.process_dict.setdefault(command, []).append(
-            NEOProcess(command, uuid, arguments))
+        self.process_dict.setdefault(node_type, []).append(
+            NEOProcess(command_dict[node_type], uuid, arguments))
 
     def __allocateUUID(self):
         uuid = ('%032x' % random.getrandbits(128)).decode('hex')
@@ -433,17 +437,14 @@ class NEOCluster(object):
             conn = sqlite3.connect(self.db_template % db, isolation_level=None)
         return conn
 
-    def _getProcessList(self, type):
-        return self.process_dict.get(type)
-
     def getMasterProcessList(self):
-        return self._getProcessList(NEO_MASTER)
+        return self.process_dict.get(NodeTypes.MASTER)
 
     def getStorageProcessList(self):
-        return self._getProcessList(NEO_STORAGE)
+        return self.process_dict.get(NodeTypes.STORAGE)
 
     def getAdminProcessList(self):
-        return self._getProcessList(NEO_ADMIN)
+        return self.process_dict.get(NodeTypes.ADMIN)
 
     def _killMaster(self, primary=False, all=False):
         killed_uuid_list = []

neo/tests/master/testClientHandler.py

@@ -42,7 +42,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
         self.master_address = ('127.0.0.1', self.master_port)
         self.client_address = ('127.0.0.1', self.client_port)
         self.storage_address = ('127.0.0.1', self.storage_port)
-        self.storage_uuid = self.getNewUUID()
+        self.storage_uuid = self.getStorageUUID()
         # register the storage
         self.app.nm.createStorage(
             uuid=self.storage_uuid,
@@ -53,7 +53,7 @@ class MasterClientHandlerTests(NeoUnitTestBase):
                              port=10021):
         """Do first step of identification to MN """
         # register the master itself
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(node_type)
         self.app.nm.createFromNodeType(
             node_type,
             address=(ip, port),

neo/tests/master/testElectionHandler.py

@@ -37,10 +37,8 @@ class MasterClientElectionTestBase(NeoUnitTestBase):
         super(MasterClientElectionTestBase, self).setUp()
         self._master_port = 3001
 
-    def identifyToMasterNode(self, uuid=True):
-        if uuid is True:
-            uuid = self.getNewUUID()
-        node = self.app.nm.createMaster(uuid=uuid)
+    def identifyToMasterNode(self):
+        node = self.app.nm.createMaster(uuid=self.getMasterUUID())
         node.setAddress((self.local_ip, self._master_port))
         self._master_port += 1
         conn = self.getFakeConnection(
@@ -59,7 +57,7 @@ class MasterClientElectionTests(MasterClientElectionTestBase):
         self.app = Application(config)
         self.app.pt.clear()
         self.app.em = Mock()
-        self.app.uuid = self._makeUUID('M')
+        self.app.uuid = self.getMasterUUID()
         self.app.server = (self.local_ip, 10000)
         self.app.name = 'NEOCLUSTER'
         self.election = ClientElectionHandler(self.app)
@@ -264,13 +262,13 @@ class MasterServerElectionTests(MasterClientElectionTestBase):
         return [x.asTuple() for x in self.app.nm.getList()]
 
     def __getClient(self):
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         conn = self.getFakeConnection(uuid=uuid, address=self.client_address)
         self.app.nm.createClient(uuid=uuid, address=self.client_address)
         return conn
 
     def __getMaster(self, port=1000, register=True):
-        uuid = self.getNewUUID()
+        uuid = self.getMasterUUID()
         address = ('127.0.0.1', port)
         conn = self.getFakeConnection(uuid=uuid, address=address)
         if register:
@@ -291,7 +289,7 @@ class MasterServerElectionTests(MasterClientElectionTestBase):
 
     def _requestIdentification(self):
         conn = self.getFakeConnection()
-        peer_uuid = self.getNewUUID()
+        peer_uuid = self.getMasterUUID()
         address = (self.local_ip, 2001)
         self.election.requestIdentification(
             conn,

neo/tests/master/testMasterApp.py

@@ -31,11 +31,11 @@ class MasterAppTests(NeoUnitTestBase):
 
     def test_06_broadcastNodeInformation(self):
         # defined some nodes to which data will be send
-        master_uuid = self.getNewUUID()
+        master_uuid = self.getMasterUUID()
         master = self.app.nm.createMaster(uuid=master_uuid)
-        storage_uuid = self.getNewUUID()
+        storage_uuid = self.getStorageUUID()
         storage = self.app.nm.createStorage(uuid=storage_uuid)
-        client_uuid = self.getNewUUID()
+        client_uuid = self.getClientUUID()
         client = self.app.nm.createClient(uuid=client_uuid)
         # create conn and patch em
         master_conn = self.getFakeConnection()
@@ -51,7 +51,7 @@ class MasterAppTests(NeoUnitTestBase):
         self.app.nm.add(client)
 
         # no address defined, not send to client node
-        c_node = self.app.nm.createClient(uuid = self.getNewUUID())
+        c_node = self.app.nm.createClient(uuid=self.getClientUUID())
         self.app.broadcastNodesInformation([c_node])
         # check conn
         self.checkNoPacketSent(client_conn)
@@ -60,7 +60,7 @@ class MasterAppTests(NeoUnitTestBase):
 
         # address defined and client type
         s_node = self.app.nm.createClient(
-            uuid = self.getNewUUID(),
+            uuid=self.getClientUUID(),
             address=("127.1.0.1", 3361)
         )
         self.app.broadcastNodesInformation([c_node])
@@ -71,7 +71,7 @@ class MasterAppTests(NeoUnitTestBase):
 
         # address defined and storage type
         s_node = self.app.nm.createStorage(
-            uuid=self.getNewUUID(),
+            uuid=self.getStorageUUID(),
             address=("127.0.0.1", 1351)
         )
 
@@ -91,8 +91,8 @@ class MasterAppTests(NeoUnitTestBase):
         self.checkNotifyNodeInformation(storage_conn)
 
     def test_storageReadinessAPI(self):
-        uuid_1 = self.getNewUUID()
-        uuid_2 = self.getNewUUID()
+        uuid_1 = self.getStorageUUID()
+        uuid_2 = self.getStorageUUID()
         self.assertFalse(self.app.isStorageReady(uuid_1))
         self.assertFalse(self.app.isStorageReady(uuid_2))
         # Must not raise, nor change readiness

neo/tests/master/testMasterPT.py

@@ -54,18 +54,18 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         pt.setCell(1, sn1, CellStates.UP_TO_DATE)
         pt.setCell(2, sn1, CellStates.UP_TO_DATE)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
         pt.setCell(0, sn2, CellStates.UP_TO_DATE)
         pt.setCell(1, sn2, CellStates.UP_TO_DATE)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
         pt.setCell(0, sn3, CellStates.UP_TO_DATE)
@@ -79,19 +79,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
 
     def test_13_outdate(self):
         # create nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19002)
         sn2 = StorageNode(Mock(), server2, uuid2)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19003)
         sn3 = StorageNode(Mock(), server3, uuid3)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19004)
         sn4 = StorageNode(Mock(), server4, uuid4)
-        uuid5 = self.getNewUUID()
+        uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 19005)
         sn5 = StorageNode(Mock(), server5, uuid5)
         # create partition table
@@ -147,7 +147,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         # add it to an empty pt
@@ -168,7 +168,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
             self.assertEqual(pt.getCellList(x)[0].getNode(), sn1)
         self.assertEqual(pt.count_dict[sn1], 5)
         # add a second node to fill the partition table
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19002)
         sn2 = StorageNode(Mock(), server2, uuid2)
         # add it
@@ -179,13 +179,13 @@ class MasterPartitionTableTests(NeoUnitTestBase):
             self.assertEqual(pt.getCellList(x)[0].getState(), CellStates.OUT_OF_DATE)
             self.assertTrue(pt.getCellList(x)[0].getNode() in (sn1, sn2))
         # test the most used node is remove from some partition
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
         sn4 = StorageNode(Mock(), server4, uuid4)
-        uuid5 = self.getNewUUID()
+        uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 1900)
         sn5 = StorageNode(Mock(), server5, uuid5)
         # partition looks like:
@@ -205,7 +205,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         pt.setCell(2, sn4, CellStates.UP_TO_DATE)
         pt.setCell(3, sn1, CellStates.OUT_OF_DATE)
         pt.setCell(3, sn5, CellStates.UP_TO_DATE)
-        uuid6 = self.getNewUUID()
+        uuid6 = self.getStorageUUID()
         server6 = ("127.0.0.6", 19006)
         sn6 = StorageNode(Mock(), server6, uuid6)
         cell_list = pt.addNode(sn6)
@@ -283,16 +283,16 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19002)
         sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
         sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
         # partition looks like:
@@ -350,11 +350,11 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         num_replicas = 1
         pt = PartitionTable(num_partitions, num_replicas)
         # add nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
         # add not running node
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2)
         sn2.setState(NodeStates.TEMPORARILY_DOWN)
@@ -362,10 +362,10 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, None, NodeStates.RUNNING)
         # add clear node
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
         sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
-        uuid5 = self.getNewUUID()
+        uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 1900)
         sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
         # make the table
@@ -404,19 +404,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         # if not enought cell, add least used node
 
         # create nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19002)
         sn2 = StorageNode(Mock(), server2, uuid2, NodeStates.RUNNING)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19003)
         sn3 = StorageNode(Mock(), server3, uuid3, NodeStates.RUNNING)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19004)
         sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
-        uuid5 = self.getNewUUID()
+        uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 19005)
         sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
         # create partition table

neo/tests/master/testRecovery.py

@@ -49,7 +49,7 @@ class MasterRecoveryTests(NeoUnitTestBase):
         """Do first step of identification to MN
         """
         address = (ip, port)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(node_type)
         self.app.nm.createFromNodeType(node_type, address=address, uuid=uuid,
             state=NodeStates.RUNNING)
         return uuid

neo/tests/master/testStorageHandler.py

@@ -60,7 +60,7 @@ class MasterStorageHandlerTests(NeoUnitTestBase):
         """Do first step of identification to MN
         """
         nm = self.app.nm
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(node_type)
         node = nm.createFromNodeType(node_type, address=(ip, port),
                 uuid=uuid)
         conn = self.getFakeConnection(node.getUUID(),node.getAddress())

neo/tests/master/testTransactions.py

@@ -18,7 +18,7 @@ import unittest
 from mock import Mock
 from struct import pack, unpack
 from .. import NeoUnitTestBase
-from neo.lib.protocol import ZERO_TID
+from neo.lib.protocol import NodeTypes, ZERO_TID
 
 from neo.master.transactions import Transaction, TransactionManager
 from neo.master.transactions import packTID, unpackTID, addTID, DelayedError
@@ -31,12 +31,9 @@ class testTransactionManager(NeoUnitTestBase):
     def makeOID(self, i):
         return pack('!Q', i)
 
-    def makeUUID(self, i):
-        return '\0' * 12 + pack('!Q', i)
-
-    def makeNode(self, i):
-        uuid = self.makeUUID(i)
-        node = Mock({'getUUID': uuid, '__hash__': i, '__repr__': 'FakeNode'})
+    def makeNode(self, node_type):
+        uuid = self.getNewUUID(node_type)
+        node = Mock({'getUUID': uuid, '__hash__': uuid, '__repr__': 'FakeNode'})
         return uuid, node
 
     def testTransaction(self):
@@ -45,7 +42,8 @@ class testTransactionManager(NeoUnitTestBase):
         tid = self.makeTID(1)
         ttid = self.makeTID(2)
         oid_list = (oid1, oid2) = [self.makeOID(1), self.makeOID(2)]
-        uuid_list = (uuid1, uuid2) = [self.makeUUID(1), self.makeUUID(2)]
+        uuid_list = (uuid1, uuid2) = [self.getStorageUUID(),
+                                      self.getStorageUUID()]
         msg_id = 1
         # create transaction object
         txn = Transaction(node, ttid)
@@ -63,8 +61,8 @@ class testTransactionManager(NeoUnitTestBase):
         node = Mock({'__hash__': 1})
         msg_id = 1
         oid_list = (oid1, oid2) = self.makeOID(1), self.makeOID(2)
-        uuid_list = (uuid1, uuid2) = self.makeUUID(1), self.makeUUID(2)
-        client_uuid = self.makeUUID(3)
+        uuid_list = uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
+        client_uuid = self.getClientUUID()
         # create transaction manager
         callback = Mock()
         txnman = TransactionManager(on_commit=callback)
@@ -94,9 +92,9 @@ class testTransactionManager(NeoUnitTestBase):
 
     def testAbortFor(self):
         oid_list = [self.makeOID(1), ]
-        storage_1_uuid, node1 = self.makeNode(1)
-        storage_2_uuid, node2 = self.makeNode(2)
-        client_uuid, client = self.makeNode(3)
+        storage_1_uuid, node1 = self.makeNode(NodeTypes.STORAGE)
+        storage_2_uuid, node2 = self.makeNode(NodeTypes.STORAGE)
+        client_uuid, client = self.makeNode(NodeTypes.CLIENT)
         txnman = TransactionManager(lambda tid, txn: None)
         # register 4 transactions made by two nodes
         self.assertEqual(txnman.registerForNotification(storage_1_uuid), [])
@@ -132,10 +130,10 @@ class testTransactionManager(NeoUnitTestBase):
         client1 = Mock({'__hash__': 1})
         client2 = Mock({'__hash__': 2})
         client3 = Mock({'__hash__': 3})
-        storage_1_uuid = self.makeUUID(1)
-        storage_2_uuid = self.makeUUID(2)
+        storage_1_uuid = self.getStorageUUID()
+        storage_2_uuid = self.getStorageUUID()
         oid_list = [self.makeOID(1), ]
-        client_uuid = self.makeUUID(3)
+        client_uuid = self.getClientUUID()
 
         tm = TransactionManager(lambda tid, txn: None)
         # Transaction 1: 2 storage nodes involved, one will die and the other
@@ -213,7 +211,7 @@ class testTransactionManager(NeoUnitTestBase):
         strictly increasing order.
         Note: this implementation might change later, to allow more paralelism.
         """
-        client_uuid, client = self.makeNode(1)
+        client_uuid, client = self.makeNode(NodeTypes.CLIENT)
         tm = TransactionManager(lambda tid, txn: None)
         # With a requested TID, lock spans from begin to remove
         ttid1 = self.getNextTID()
@@ -230,7 +228,7 @@ class testTransactionManager(NeoUnitTestBase):
         tm.prepare(ttid3, 1, [], [], 0)
 
     def testClientDisconectsAfterBegin(self):
-        client_uuid1, node1 = self.makeNode(1)
+        client_uuid1, node1 = self.makeNode(NodeTypes.CLIENT)
         tm = TransactionManager(lambda tid, txn: None)
         tid1 = self.getNextTID()
         tid2 = self.getNextTID()
@@ -240,9 +238,9 @@ class testTransactionManager(NeoUnitTestBase):
 
     def testUnlockPending(self):
         callback = Mock()
-        uuid1, node1 = self.makeNode(1)
-        uuid2, node2 = self.makeNode(2)
-        storage_uuid = self.makeUUID(3)
+        uuid1, node1 = self.makeNode(NodeTypes.CLIENT)
+        uuid2, node2 = self.makeNode(NodeTypes.CLIENT)
+        storage_uuid = self.getStorageUUID()
         tm = TransactionManager(callback)
         ttid1 = tm.begin(node1)
         ttid2 = tm.begin(node2)

neo/tests/master/testVerification.py

@@ -49,7 +49,7 @@ class MasterVerificationTests(NeoUnitTestBase):
                              port=10021):
         """Do first step of identification to MN
         """
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(node_type)
         self.app.nm.createFromNodeType(
             node_type,
             address=(ip, port),

neo/tests/storage/testClientHandler.py

@@ -49,7 +49,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         # handler
         self.operation = ClientOperationHandler(self.app)
         # set pmn
-        self.master_uuid = self.getNewUUID()
+        self.master_uuid = self.getMasterUUID()
         pmn = self.app.nm.getMasterList()[0]
         pmn.setUUID(self.master_uuid)
         self.app.primary_master_node = pmn
@@ -69,7 +69,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         calls[0].checkArgs(uuid)
 
     def test_connectionLost(self):
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         self.app.nm.createClient(uuid=uuid)
         conn = self._getConnection(uuid=uuid)
         self.operation.connectionClosed(conn)
@@ -192,8 +192,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         self.checkAnswerObjectHistory(conn)
 
     def test_askStoreTransaction(self):
-        uuid = self.getNewUUID()
-        conn = self._getConnection(uuid=uuid)
+        conn = self._getConnection(uuid=self.getClientUUID())
         tid = self.getNextTID()
         user = 'USER'
         desc = 'DESC'
@@ -217,8 +216,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
 
     def test_askStoreObject1(self):
         # no conflict => answer
-        uuid = self.getNewUUID()
-        conn = self._getConnection(uuid=uuid)
+        conn = self._getConnection(uuid=self.getClientUUID())
         tid = self.getNextTID()
         oid, serial, comp, checksum, data = self._getObject()
         self.operation.askStoreObject(conn, oid, serial, comp, checksum,
@@ -233,8 +231,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
 
     def test_askStoreObjectWithDataTID(self):
         # same as test_askStoreObject1, but with a non-None data_tid value
-        uuid = self.getNewUUID()
-        conn = self._getConnection(uuid=uuid)
+        conn = self._getConnection(uuid=self.getClientUUID())
         tid = self.getNextTID()
         oid, serial, comp, checksum, data = self._getObject()
         data_tid = self.getNextTID()
@@ -250,8 +247,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
 
     def test_askStoreObject2(self):
         # conflict error
-        uuid = self.getNewUUID()
-        conn = self._getConnection(uuid=uuid)
+        conn = self._getConnection(uuid=self.getClientUUID())
         tid = self.getNextTID()
         locking_tid = self.getNextTID(tid)
         def fakeStoreObject(*args):
@@ -275,8 +271,7 @@ class StorageClientHandlerTests(NeoUnitTestBase):
         calls[0].checkArgs(tid)
 
     def test_askObjectUndoSerial(self):
-        uuid = self.getNewUUID()
-        conn = self._getConnection(uuid=uuid)
+        conn = self._getConnection(uuid=self.getClientUUID())
         tid = self.getNextTID()
         ltid = self.getNextTID()
         undone_tid = self.getNextTID()

neo/tests/storage/testIdentificationHandler.py

@@ -47,7 +47,7 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
                 self.identification.requestIdentification,
                 self.getFakeConnection(),
                 NodeTypes.CLIENT,
-                self.getNewUUID(),
+                self.getClientUUID(),
                 None,
                 self.app.name,
         )
@@ -57,14 +57,14 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
                 self.identification.requestIdentification,
                 self.getFakeConnection(),
                 NodeTypes.STORAGE,
-                self.getNewUUID(),
+                self.getStorageUUID(),
                 None,
                 self.app.name,
         )
 
     def test_requestIdentification3(self):
         """ broken nodes must be rejected """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         conn = self.getFakeConnection(uuid=uuid)
         node = self.app.nm.createClient(uuid=uuid)
         node.setBroken()
@@ -79,7 +79,7 @@ class StorageIdentificationHandlerTests(NeoUnitTestBase):
 
     def test_requestIdentification2(self):
         """ accepted client must be connected and running """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         conn = self.getFakeConnection(uuid=uuid)
         node = self.app.nm.createClient(uuid=uuid)
         master = (self.local_ip, 3000)

neo/tests/storage/testInitializationHandler.py

@@ -49,7 +49,8 @@ class StorageInitializationHandlerTests(NeoUnitTestBase):
 
     def getClientConnection(self):
         address = ("127.0.0.1", self.client_port)
-        return self.getFakeConnection(uuid=self.getNewUUID(), address=address)
+        return self.getFakeConnection(uuid=self.getClientUUID(),
+                                      address=address)
 
     def test_03_connectionClosed(self):
         conn = self.getClientConnection()
@@ -62,15 +63,15 @@ class StorageInitializationHandlerTests(NeoUnitTestBase):
         # send a table
         conn = self.getClientConnection()
         self.app.pt = PartitionTable(3, 2)
-        node_1 = self.getNewUUID()
-        node_2 = self.getNewUUID()
-        node_3 = self.getNewUUID()
+        node_1 = self.getStorageUUID()
+        node_2 = self.getStorageUUID()
+        node_3 = self.getStorageUUID()
         self.app.uuid = node_1
         # SN already know all nodes
         self.app.nm.createStorage(uuid=node_1)
         self.app.nm.createStorage(uuid=node_2)
         self.app.nm.createStorage(uuid=node_3)
-        self.assertEqual(self.app.dm.getPartitionTable(), [])
+        self.assertEqual(self.app.dm.getPartitionTable(), ())
         row_list = [(0, ((node_1, CellStates.UP_TO_DATE), (node_2, CellStates.UP_TO_DATE))),
                     (1, ((node_3, CellStates.UP_TO_DATE), (node_1, CellStates.UP_TO_DATE))),
                     (2, ((node_2, CellStates.UP_TO_DATE), (node_3, CellStates.UP_TO_DATE)))]

neo/tests/storage/testMasterHandler.py

@@ -40,7 +40,7 @@ class StorageMasterHandlerTests(NeoUnitTestBase):
         # handler
         self.operation = MasterOperationHandler(self.app)
         # set pmn
-        self.master_uuid = self.getNewUUID()
+        self.master_uuid = self.getMasterUUID()
         pmn = self.app.nm.getMasterList()[0]
         pmn.setUUID(self.master_uuid)
         self.app.primary_master_node = pmn
@@ -79,7 +79,7 @@ class StorageMasterHandlerTests(NeoUnitTestBase):
 
     def test_14_notifyPartitionChanges2(self):
         # cases :
-        uuid1, uuid2, uuid3 = [self.getNewUUID() for i in range(3)]
+        uuid1, uuid2, uuid3 = [self.getStorageUUID() for i in range(3)]
         cells = (
             (0, uuid1, CellStates.UP_TO_DATE),
             (1, uuid2, CellStates.DISCARDED),

neo/tests/storage/testStorageApp.py

@@ -61,11 +61,11 @@ class StorageAppTests(NeoUnitTestBase):
             self.assertFalse(self.app.pt.hasOffset(x))
 
         # add some node, will be remove when loading table
-        master_uuid = self.getNewUUID()
+        master_uuid = self.getMasterUUID()
         master = self.app.nm.createMaster(uuid=master_uuid)
-        storage_uuid = self.getNewUUID()
+        storage_uuid = self.getStorageUUID()
         storage = self.app.nm.createStorage(uuid=storage_uuid)
-        client_uuid = self.getNewUUID()
+        client_uuid = self.getClientUUID()
 
         self.app.pt.setCell(0, master, CellStates.UP_TO_DATE)
         self.app.pt.setCell(0, storage, CellStates.UP_TO_DATE)

neo/tests/storage/testStorageDBTests.py

@@ -67,7 +67,7 @@ class StorageDBTests(NeoUnitTestBase):
                     db.dropPartitions(n)
         db.setNumPartitions(num_partitions)
         self.assertEqual(num_partitions, db.getNumPartitions())
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         db.setUUID(uuid)
         self.assertEqual(uuid, db.getUUID())
         db.setPartitionTable(1,
@@ -83,7 +83,7 @@ class StorageDBTests(NeoUnitTestBase):
 
     def test_UUID(self):
         db = self.getDB()
-        self.checkConfigEntry(db.getUUID, db.setUUID, 'TEST_VALUE')
+        self.checkConfigEntry(db.getUUID, db.setUUID, 123)
 
     def test_Name(self):
         db = self.getDB()
@@ -119,7 +119,7 @@ class StorageDBTests(NeoUnitTestBase):
     def test_getPartitionTable(self):
         db = self.getDB()
         ptid = self.getPTID(1)
-        uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
+        uuid1, uuid2 = self.getStorageUUID(), self.getStorageUUID()
         cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
         cell2 = (1, uuid1, CellStates.UP_TO_DATE)
         db.setPartitionTable(ptid, [cell1, cell2])
@@ -249,38 +249,38 @@ class StorageDBTests(NeoUnitTestBase):
     def test_setPartitionTable(self):
         db = self.getDB()
         ptid = self.getPTID(1)
-        uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
-        cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
-        cell2 = (1, uuid1, CellStates.UP_TO_DATE)
-        cell3 = (1, uuid1, CellStates.DISCARDED)
+        uuid = self.getStorageUUID()
+        cell1 = 0, uuid, CellStates.OUT_OF_DATE
+        cell2 = 1, uuid, CellStates.UP_TO_DATE
+        cell3 = 1, uuid, CellStates.DISCARDED
         # no partition table
-        self.assertEqual(db.getPartitionTable(), [])
+        self.assertEqual(list(db.getPartitionTable()), [])
         # set one
         db.setPartitionTable(ptid, [cell1])
         result = db.getPartitionTable()
-        self.assertEqual(result, [cell1])
+        self.assertEqual(list(result), [cell1])
         # then another
         db.setPartitionTable(ptid, [cell2])
         result = db.getPartitionTable()
-        self.assertEqual(result, [cell2])
+        self.assertEqual(list(result), [cell2])
         # drop discarded cells
         db.setPartitionTable(ptid, [cell2, cell3])
         result = db.getPartitionTable()
-        self.assertEqual(result, [])
+        self.assertEqual(list(result), [])
 
     def test_changePartitionTable(self):
         db = self.getDB()
         ptid = self.getPTID(1)
-        uuid1, uuid2 = self.getNewUUID(), self.getNewUUID()
-        cell1 = (0, uuid1, CellStates.OUT_OF_DATE)
-        cell2 = (1, uuid1, CellStates.UP_TO_DATE)
-        cell3 = (1, uuid1, CellStates.DISCARDED)
+        uuid = self.getStorageUUID()
+        cell1 = 0, uuid, CellStates.OUT_OF_DATE
+        cell2 = 1, uuid, CellStates.UP_TO_DATE
+        cell3 = 1, uuid, CellStates.DISCARDED
         # no partition table
-        self.assertEqual(db.getPartitionTable(), [])
+        self.assertEqual(list(db.getPartitionTable()), [])
         # set one
         db.changePartitionTable(ptid, [cell1])
         result = db.getPartitionTable()
-        self.assertEqual(result, [cell1])
+        self.assertEqual(list(result), [cell1])
         # add more entries
         db.changePartitionTable(ptid, [cell2])
         result = db.getPartitionTable()
@@ -288,7 +288,7 @@ class StorageDBTests(NeoUnitTestBase):
         # drop discarded cells
         db.changePartitionTable(ptid, [cell2, cell3])
         result = db.getPartitionTable()
-        self.assertEqual(result, [cell1])
+        self.assertEqual(list(result), [cell1])
 
     def test_dropUnfinishedData(self):
         oid1, oid2 = self.getOIDs(2)

neo/tests/storage/testTransactions.py

@@ -25,7 +25,7 @@ from neo.storage.transactions import ConflictError, DelayedError
 class TransactionTests(NeoUnitTestBase):
 
     def testInit(self):
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         ttid = self.getNextTID()
         tid = self.getNextTID()
         txn = Transaction(uuid, ttid)
@@ -37,15 +37,8 @@ class TransactionTests(NeoUnitTestBase):
         self.assertEqual(txn.getObjectList(), [])
         self.assertEqual(txn.getOIDList(), [])
 
-    def testRepr(self):
-        """ Just check if the __repr__ implementation will not raise """
-        uuid = self.getNewUUID()
-        tid = self.getNextTID()
-        txn = Transaction(uuid, tid)
-        repr(txn)
-
     def testLock(self):
-        txn = Transaction(self.getNewUUID(), self.getNextTID())
+        txn = Transaction(self.getClientUUID(), self.getNextTID())
         self.assertFalse(txn.isLocked())
         txn.lock()
         self.assertTrue(txn.isLocked())
@@ -53,7 +46,8 @@ class TransactionTests(NeoUnitTestBase):
         self.assertRaises(AssertionError, txn.lock)
 
     def testTransaction(self):
-        txn = Transaction(self.getNewUUID(), self.getNextTID())
+        txn = Transaction(self.getClientUUID(), self.getNextTID())
+        repr(txn) # check __repr__ does not raise
         oid_list = [self.getOID(1), self.getOID(2)]
         txn_info = (oid_list, 'USER', 'DESC', 'EXT', False)
         txn.prepare(*txn_info)
@@ -61,7 +55,7 @@ class TransactionTests(NeoUnitTestBase):
                          txn_info + (txn.getTTID(),))
 
     def testObjects(self):
-        txn = Transaction(self.getNewUUID(), self.getNextTID())
+        txn = Transaction(self.getClientUUID(), self.getNextTID())
         oid1, oid2 = self.getOID(1), self.getOID(2)
         object1 = oid1, "0" * 20, None
         object2 = oid2, "1" * 20, None
@@ -77,7 +71,7 @@ class TransactionTests(NeoUnitTestBase):
     def test_getObject(self):
         oid_1 = self.getOID(1)
         oid_2 = self.getOID(2)
-        txn = Transaction(self.getNewUUID(), self.getNextTID())
+        txn = Transaction(self.getClientUUID(), self.getNextTID())
         object_info = oid_1, None, None
         txn.addObject(*object_info)
         self.assertRaises(KeyError, txn.getObject, oid_2)
@@ -127,7 +121,7 @@ class TransactionManagerTests(NeoUnitTestBase):
         """ One node, one transaction, not abort """
         data_id_list = random.random(), random.random()
         self.app.dm.mockAddReturnValues(storeData=ReturnValues(*data_id_list))
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         ttid = self.getNextTID()
         tid, txn = self._getTransaction()
         serial1, object1 = self._getObject(1)
@@ -148,7 +142,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testDelayed(self):
         """ Two transactions, the first cause the second to be delayed """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         ttid1 = self.getNextTID()
         ttid2 = self.getNextTID()
         tid1, txn1 = self._getTransaction()
@@ -169,7 +163,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testUnresolvableConflict(self):
         """ A newer transaction has already modified an object """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         ttid1 = self.getNextTID()
         ttid2 = self.getNextTID()
         tid1, txn1 = self._getTransaction()
@@ -190,7 +184,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testResolvableConflict(self):
         """ Try to store an object with the lastest revision """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         tid, txn = self._getTransaction()
         serial, obj = self._getObject(1)
         next_serial = self.getNextTID(serial)
@@ -203,8 +197,8 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testLockDelayed(self):
         """ Check lock delay """
-        uuid1 = self.getNewUUID()
-        uuid2 = self.getNewUUID()
+        uuid1 = self.getClientUUID()
+        uuid2 = self.getClientUUID()
         self.assertNotEqual(uuid1, uuid2)
         ttid1 = self.getNextTID()
         ttid2 = self.getNextTID()
@@ -230,8 +224,8 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testLockConflict(self):
         """ Check lock conflict """
-        uuid1 = self.getNewUUID()
-        uuid2 = self.getNewUUID()
+        uuid1 = self.getClientUUID()
+        uuid2 = self.getClientUUID()
         self.assertNotEqual(uuid1, uuid2)
         ttid1 = self.getNextTID()
         ttid2 = self.getNextTID()
@@ -257,7 +251,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testAbortUnlocked(self):
         """ Abort a non-locked transaction """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         tid, txn = self._getTransaction()
         serial, obj = self._getObject(1)
         self.manager.register(uuid, tid)
@@ -272,7 +266,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testAbortLockedDoNothing(self):
         """ Try to abort a locked transaction """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         ttid = self.getNextTID()
         tid, txn = self._getTransaction()
         self.manager.register(uuid, ttid)
@@ -289,8 +283,8 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testAbortForNode(self):
         """ Abort transaction for a node """
-        uuid1 = self.getNewUUID()
-        uuid2 = self.getNewUUID()
+        uuid1 = self.getClientUUID()
+        uuid2 = self.getClientUUID()
         self.assertNotEqual(uuid1, uuid2)
         ttid1 = self.getNextTID()
         ttid2 = self.getNextTID()
@@ -319,7 +313,7 @@ class TransactionManagerTests(NeoUnitTestBase):
 
     def testReset(self):
         """ Reset the manager """
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         tid, txn = self._getTransaction()
         ttid = self.getNextTID()
         self.manager.register(uuid, ttid)
@@ -335,7 +329,7 @@ class TransactionManagerTests(NeoUnitTestBase):
     def test_getObjectFromTransaction(self):
         data_id = random.random()
         self.app.dm.mockAddReturnValues(storeData=ReturnValues(data_id))
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         tid1, txn1 = self._getTransaction()
         tid2, txn2 = self._getTransaction()
         serial1, obj1 = self._getObject(1)
@@ -350,7 +344,7 @@ class TransactionManagerTests(NeoUnitTestBase):
             (obj1[0], data_id, obj1[4]))
 
     def test_getLockingTID(self):
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         serial1, obj1 = self._getObject(1)
         oid1 = obj1[0]
         tid1, txn1 = self._getTransaction()
@@ -363,7 +357,7 @@ class TransactionManagerTests(NeoUnitTestBase):
         ram_serial = self.getNextTID()
         oid = self.getOID(1)
         orig_serial = self.getNextTID()
-        uuid = self.getNewUUID()
+        uuid = self.getClientUUID()
         locking_serial = self.getNextTID()
         other_serial = self.getNextTID()
         new_serial = self.getNextTID()

neo/tests/storage/testVerificationHandler.py

@@ -49,16 +49,12 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
         super(StorageVerificationHandlerTests, self)._tearDown(success)
 
     # Common methods
-    def getClientConnection(self):
-        address = ("127.0.0.1", self.client_port)
-        return self.getFakeConnection(uuid=self.getNewUUID(), address=address)
-
     def getMasterConnection(self):
         return self.getFakeConnection(address=("127.0.0.1", self.master_port))
 
     # Tests
     def test_03_connectionClosed(self):
-        conn = self.getClientConnection()
+        conn = self.getMasterConnection()
         self.app.listening_conn = object() # mark as running
         self.assertRaises(PrimaryFailure, self.verification.connectionClosed, conn,)
         # nothing happens
@@ -67,11 +63,11 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
     def test_08_askPartitionTable(self):
         node = self.app.nm.createStorage(
             address=("127.7.9.9", 1),
-            uuid=self.getNewUUID()
+            uuid=self.getStorageUUID()
         )
         self.app.pt.setCell(1, node, CellStates.UP_TO_DATE)
         self.assertTrue(self.app.pt.hasOffset(1))
-        conn = self.getClientConnection()
+        conn = self.getMasterConnection()
         self.verification.askPartitionTable(conn)
         ptid, row_list = self.checkAnswerPartitionTable(conn, decode=True)
         self.assertEqual(len(row_list), 1009)
@@ -85,12 +81,12 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
 
         # new node
         conn = self.getMasterConnection()
-        new_uuid = self.getNewUUID()
+        new_uuid = self.getStorageUUID()
         cell = (0, new_uuid, CellStates.UP_TO_DATE)
         self.app.nm.createStorage(uuid=new_uuid)
         self.app.pt = PartitionTable(1, 1)
         self.app.dm = Mock({ })
-        ptid, self.ptid = self.getTwoIDs()
+        ptid = self.getPTID()
         # pt updated
         self.verification.notifyPartitionChanges(conn, ptid, (cell, ))
         # check db update
@@ -150,7 +146,7 @@ class StorageVerificationHandlerTests(NeoUnitTestBase):
         self.app.dm = Mock({
             'getTransaction': ([p64(2)], 'u2', 'd2', 'e2', False),
         })
-        conn = self.getClientConnection()
+        conn = self.getMasterConnection()
         self.verification.askTransactionInformation(conn, p64(1))
         tid, user, desc, ext, packed, oid_list = self.checkAnswerTransactionInformation(conn, decode=True)
         self.assertEqual(u64(tid), 1)

neo/tests/testConnection.py

@@ -207,7 +207,7 @@ class ConnectionTests(NeoUnitTestBase):
         # test uuid
         self.assertEqual(bc.uuid, None)
         self.assertEqual(bc.getUUID(), None)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(None)
         bc.setUUID(uuid)
         self.assertEqual(bc.getUUID(), uuid)
         # test next id
@@ -423,7 +423,7 @@ class ConnectionTests(NeoUnitTestBase):
         self._checkPacketReceived(0)
         self._checkReadBuf(bc, '')
 
-        p = Packets.AnswerPrimary(self.getNewUUID())
+        p = Packets.AnswerPrimary(self.getNewUUID(None))
         p.setId(1)
         p_data = ''.join(p.encode())
         data_edge = len(p_data) - 1
@@ -450,10 +450,10 @@ class ConnectionTests(NeoUnitTestBase):
         # give multiple packet
         bc = self._makeConnection()
         bc._queue = Mock()
-        p1 = Packets.AnswerPrimary(self.getNewUUID())
+        p1 = Packets.AnswerPrimary(self.getNewUUID(None))
         p1.setId(1)
         self._appendPacketToReadBuf(bc, p1)
-        p2 = Packets.AnswerPrimary( self.getNewUUID())
+        p2 = Packets.AnswerPrimary( self.getNewUUID(None))
         p2.setId(2)
         self._appendPacketToReadBuf(bc, p2)
         self.assertEqual(len(bc.read_buf), len(p1) + len(p2))
@@ -488,7 +488,7 @@ class ConnectionTests(NeoUnitTestBase):
         # give an expected packet
         bc = self._makeConnection()
         bc._queue = Mock()
-        p = Packets.AnswerPrimary(self.getNewUUID())
+        p = Packets.AnswerPrimary(self.getNewUUID(None))
         p.setId(1)
         self._appendPacketToReadBuf(bc, p)
         bc.analyse()
@@ -587,7 +587,7 @@ class ConnectionTests(NeoUnitTestBase):
         # With aborted set to false
         # patch receive method to return data
         def receive(self):
-            p = Packets.AnswerPrimary(self.getNewUUID())
+            p = Packets.AnswerPrimary(self.getNewUUID(None))
             p.setId(1)
             return ''.join(p.encode())
         DoNothingConnector.receive = receive
@@ -757,7 +757,7 @@ class ConnectionTests(NeoUnitTestBase):
         # test uuid
         self.assertEqual(bc.uuid, None)
         self.assertEqual(bc.getUUID(), None)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(None)
         bc.setUUID(uuid)
         self.assertEqual(bc.getUUID(), uuid)
         # test next id

neo/tests/testNodes.py

@@ -44,7 +44,7 @@ class NodesTests(NeoUnitTestBase):
     def testInit(self):
         """ Check the node initialization """
         address = ('127.0.0.1', 10000)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(None)
         node = Node(self.manager, address=address, uuid=uuid)
         self.assertEqual(node.getState(), NodeStates.UNKNOWN)
         self.assertEqual(node.getAddress(), address)
@@ -74,7 +74,7 @@ class NodesTests(NeoUnitTestBase):
         """ As for Address but UUID """
         node = Node(self.manager)
         self.assertEqual(node.getAddress(), None)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(None)
         node.setUUID(uuid)
         self._updatedByUUID(node)
 
@@ -131,16 +131,16 @@ class NodeManagerTests(NeoUnitTestBase):
         self.manager = NodeManager()
 
     def _addStorage(self):
-        self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getNewUUID())
+        self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
 
     def _addMaster(self):
-        self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getNewUUID())
+        self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID())
 
     def _addClient(self):
-        self.client = ClientNode(self.manager, None, self.getNewUUID())
+        self.client = ClientNode(self.manager, None, self.getClientUUID())
 
     def _addAdmin(self):
-        self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getNewUUID())
+        self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID())
 
     def checkNodes(self, node_list):
         manager = self.manager
@@ -180,7 +180,7 @@ class NodeManagerTests(NeoUnitTestBase):
         address = ('127.0.0.1', 10000)
         self.assertEqual(manager.getByAddress(address), None)
         self.assertEqual(manager.getByAddress(None), None)
-        uuid = self.getNewUUID()
+        uuid = self.getNewUUID(None)
         self.assertEqual(manager.getByUUID(uuid), None)
         self.assertEqual(manager.getByUUID(None), None)
 
@@ -261,7 +261,7 @@ class NodeManagerTests(NeoUnitTestBase):
         old_address = self.master.getAddress()
         new_address = ('127.0.0.1', 2001)
         old_uuid = self.storage.getUUID()
-        new_uuid = self.getNewUUID()
+        new_uuid = self.getStorageUUID()
         node_list = (
             (NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN),
             (NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING),

neo/tests/testPT.py

@@ -24,7 +24,7 @@ from . import NeoUnitTestBase
 class PartitionTableTests(NeoUnitTestBase):
 
     def test_01_Cell(self):
-        uuid = self.getNewUUID()
+        uuid = self.getStorageUUID()
         server = ("127.0.0.1", 19001)
         sn = StorageNode(Mock(), server, uuid)
         cell = Cell(sn)
@@ -48,7 +48,7 @@ class PartitionTableTests(NeoUnitTestBase):
         num_partitions = 5
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -129,7 +129,7 @@ class PartitionTableTests(NeoUnitTestBase):
         num_partitions = 5
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -169,19 +169,19 @@ class PartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add two kind of node, usable and unsable
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2)
         pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3)
         pt.setCell(0, sn3, CellStates.FEEDING)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
         sn4 = StorageNode(Mock(), server4, uuid4)
         pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
@@ -215,15 +215,15 @@ class PartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add two kind of node, usable and unsable
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2)
         pt.setCell(1, sn2, CellStates.OUT_OF_DATE)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3)
         pt.setCell(2, sn3, CellStates.FEEDING)
@@ -245,19 +245,19 @@ class PartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add two kind of node, usable and unsable
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2)
         pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3)
         pt.setCell(0, sn3, CellStates.FEEDING)
-        uuid4 = self.getNewUUID()
+        uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
         sn4 = StorageNode(Mock(), server4, uuid4)
         pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
@@ -278,7 +278,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertEqual(pt.num_filled_rows, 0)
         self.assertFalse(pt.filled())
         # adding a node in all partition
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -291,7 +291,7 @@ class PartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add two kind of node, usable and unsable
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
@@ -308,7 +308,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -322,7 +322,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -337,7 +337,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         sn1.setState(NodeStates.TEMPORARILY_DOWN)
@@ -352,7 +352,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         for x in xrange(num_partitions):
@@ -366,18 +366,18 @@ class PartitionTableTests(NeoUnitTestBase):
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         # add nodes
-        uuid1 = self.getNewUUID()
+        uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
         sn1 = StorageNode(Mock(), server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         pt.setCell(1, sn1, CellStates.UP_TO_DATE)
         pt.setCell(2, sn1, CellStates.UP_TO_DATE)
-        uuid2 = self.getNewUUID()
+        uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
         sn2 = StorageNode(Mock(), server2, uuid2)
         pt.setCell(0, sn2, CellStates.UP_TO_DATE)
         pt.setCell(1, sn2, CellStates.UP_TO_DATE)
-        uuid3 = self.getNewUUID()
+        uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
         sn3 = StorageNode(Mock(), server3, uuid3)
         pt.setCell(0, sn3, CellStates.UP_TO_DATE)

neo/tests/threaded/__init__.py

@@ -32,8 +32,7 @@ from neo.lib.connection import BaseConnection, Connection
 from neo.lib.connector import SocketConnector, \
     ConnectorConnectionRefusedException, ConnectorTryAgainException
 from neo.lib.event import EventManager
-from neo.lib.protocol import CellStates, ClusterStates, NodeStates, NodeTypes, \
-    UUID_NAMESPACES, INVALID_UUID
+from neo.lib.protocol import CellStates, ClusterStates, NodeStates, NodeTypes
 from neo.lib.util import SOCKET_CONNECTORS_DICT, parseMasterList, p64
 from .. import NeoTestBase, getTempDirectory, setupMySQLdb, \
     ADDRESS_TYPE, IP_VERSION_FORMAT_DICT, DB_PREFIX, DB_USER
@@ -802,27 +801,10 @@ def predictable_random(seed=None):
             logging.info("using seed %r", s)
             r = random.Random(s)
             try:
-                def getNewUUID(self, uuid, address, node_type):
-                    if node_type == NodeTypes.CLIENT:
-                        return super(MasterApplication, self).getNewUUID(uuid,
-                            address, node_type)
-                    if None != uuid != self.uuid and \
-                            self.nm.getByAddress(address) is \
-                            self.nm.getByUUID(uuid):
-                        return uuid
-                    while True:
-                        uuid = UUID_NAMESPACES[node_type] + ''.join(
-                                chr(r.randrange(256)) for _ in xrange(15))
-                        if uuid != self.uuid and \
-                                self.nm.getByUUID(uuid) is None:
-                            return uuid
-                MasterApplication.getNewUUID = getNewUUID
-
                 administration.random = backup_app.random = replicator.random \
                     = r
                 return wrapped(*args, **kw)
             finally:
-                del MasterApplication.getNewUUID
                 administration.random = backup_app.random = replicator.random \
                     = random
         return wraps(wrapped)(wrapper)

neo/tests/threaded/test.py

@@ -495,13 +495,13 @@ class Test(NEOThreadedTest):
         cluster.reset() # reopen DB to check partition tables
         dm = cluster.storage_list[0].dm
         self.assertEqual(1, dm.getPTID())
-        pt = dm.getPartitionTable()
+        pt = list(dm.getPartitionTable())
         self.assertEqual(20, len(pt))
         for _, _, state in pt:
             self.assertEqual(state, CellStates.UP_TO_DATE)
         for s in cluster.storage_list[1:]:
             self.assertEqual(s.dm.getPTID(), 1)
-            self.assertEqual(s.dm.getPartitionTable(), pt)
+            self.assertEqual(list(s.dm.getPartitionTable()), pt)
 
 
 if __name__ == "__main__":

neo/tests/threaded/testReplication.py

@@ -26,8 +26,8 @@ from neo.storage.transactions import TransactionManager, \
     DelayedError, ConflictError
 from neo.lib.connection import MTClientConnection
 from neo.lib.protocol import CellStates, ClusterStates, NodeStates, Packets, \
-    ZERO_OID, ZERO_TID, MAX_TID
-from neo.lib.util import dump, p64
+    ZERO_OID, ZERO_TID, MAX_TID, uuid_str
+from neo.lib.util import p64
 from . import NEOCluster, NEOThreadedTest, Patch, predictable_random
 from neo.client.pool import CELL_CONNECTED, CELL_GOOD
 
@@ -239,7 +239,8 @@ class ReplicationTests(NEOThreadedTest):
         def corrupt(offset):
             s0, s1, s2 = (storage_dict[cell.getUUID()]
                 for cell in cluster.master.pt.getCellList(offset, True))
-            logging.info('corrupt partition %u of %s', offset, dump(s1.uuid))
+            logging.info('corrupt partition %u of %s',
+                         offset, uuid_str(s1.uuid))
             s1.dm.deleteObject(p64(np+offset), p64(corrupt_tid))
             return s0.uuid
         def check(expected_state, expected_count):