lib.node: code refactoring

neo/tests/__init__.py

@@ -32,6 +32,7 @@ from functools import wraps
 from mock import Mock
 from neo.lib import debug, logging, protocol
 from neo.lib.protocol import NodeTypes, Packets, UUID_NAMESPACES
+from neo.lib.util import cached_property
 from time import time
 from struct import pack, unpack
 from unittest.case import _ExpectedFailure, _UnexpectedSuccess
@@ -194,6 +195,15 @@ class NeoUnitTestBase(NeoTestBase):
         self.uuid_dict = {}
         NeoTestBase.setUp(self)
 
+    @cached_property
+    def nm(self):
+        from neo.lib import node
+        return node.NodeManager()
+
+    def createStorage(self, *args):
+        return self.nm.createStorage(**dict(zip(
+            ('address', 'uuid', 'state'), args)))
+
     def prepareDatabase(self, number, prefix=DB_PREFIX):
         """ create empty databases """
         adapter = os.getenv('NEO_TESTS_ADAPTER', 'MySQL')

neo/tests/master/testMasterPT.py

@@ -21,7 +21,6 @@ from .. import NeoUnitTestBase
 from neo.lib.protocol import NodeStates, CellStates
 from neo.lib.pt import PartitionTableException
 from neo.master.pt import PartitionTable
-from neo.lib.node import StorageNode
 
 class MasterPartitionTableTests(NeoUnitTestBase):
 
@@ -55,19 +54,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         # create nodes
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19002)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19003)
-        sn3 = StorageNode(Mock(), server3, uuid3)
+        sn3 = self.createStorage(server3, uuid3)
         uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19004)
-        sn4 = StorageNode(Mock(), server4, uuid4)
+        sn4 = self.createStorage(server4, uuid4)
         uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 19005)
-        sn5 = StorageNode(Mock(), server5, uuid5)
+        sn5 = self.createStorage(server5, uuid5)
         # create partition table
         num_partitions = 5
         num_replicas = 3
@@ -117,7 +116,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
 
     def test_15_dropNodeList(self):
-        sn = [StorageNode(Mock(), None, i + 1, NodeStates.RUNNING)
+        sn = [self.createStorage(None, i + 1, NodeStates.RUNNING)
               for i in xrange(3)]
         pt = PartitionTable(3, 0)
         pt.setCell(0, sn[0], CellStates.OUT_OF_DATE)
@@ -153,22 +152,22 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         # add nodes
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
+        sn1 = self.createStorage(server1, uuid1, NodeStates.RUNNING)
         # add not running node
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         sn2.setState(NodeStates.TEMPORARILY_DOWN)
         # add node without uuid
         server3 = ("127.0.0.3", 19001)
-        sn3 = StorageNode(Mock(), server3, None, NodeStates.RUNNING)
+        sn3 = self.createStorage(server3, None, NodeStates.RUNNING)
         # add clear node
         uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
-        sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
+        sn4 = self.createStorage(server4, uuid4, NodeStates.RUNNING)
         uuid5 = self.getStorageUUID()
         server5 = ("127.0.0.5", 1900)
-        sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
+        sn5 = self.createStorage(server5, uuid5, NodeStates.RUNNING)
         # make the table
         pt.make([sn1, sn2, sn3, sn4, sn5])
         # check it's ok, only running nodes and node with uuid
@@ -231,7 +230,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
         return change_list
 
     def test_17_tweak(self):
-        sn = [StorageNode(Mock(), None, i + 1, NodeStates.RUNNING)
+        sn = [self.createStorage(None, i + 1, NodeStates.RUNNING)
               for i in xrange(5)]
         pt = PartitionTable(5, 2)
         # part 0

neo/tests/testNodes.py

@@ -18,8 +18,7 @@ import unittest
 from mock import Mock
 from neo.lib import protocol
 from neo.lib.protocol import NodeTypes, NodeStates
-from neo.lib.node import Node, MasterNode, StorageNode, \
-    ClientNode, AdminNode, NodeManager, MasterDB
+from neo.lib.node import Node, NodeManager, MasterDB
 from . import NeoUnitTestBase, getTempDirectory
 from time import time
 from os import chmod, mkdir, rmdir, unlink
@@ -29,15 +28,15 @@ class NodesTests(NeoUnitTestBase):
 
     def setUp(self):
         NeoUnitTestBase.setUp(self)
-        self.manager = Mock()
+        self.nm = Mock()
 
     def _updatedByAddress(self, node, index=0):
-        calls = self.manager.mockGetNamedCalls('_updateAddress')
+        calls = self.nm.mockGetNamedCalls('_updateAddress')
         self.assertEqual(len(calls), index + 1)
         self.assertEqual(calls[index].getParam(0), node)
 
     def _updatedByUUID(self, node, index=0):
-        calls = self.manager.mockGetNamedCalls('_updateUUID')
+        calls = self.nm.mockGetNamedCalls('_updateUUID')
         self.assertEqual(len(calls), index + 1)
         self.assertEqual(calls[index].getParam(0), node)
 
@@ -45,7 +44,7 @@ class NodesTests(NeoUnitTestBase):
         """ Check the node initialization """
         address = ('127.0.0.1', 10000)
         uuid = self.getNewUUID(None)
-        node = Node(self.manager, address=address, uuid=uuid)
+        node = Node(self.nm, address=address, uuid=uuid)
         self.assertEqual(node.getState(), NodeStates.UNKNOWN)
         self.assertEqual(node.getAddress(), address)
         self.assertEqual(node.getUUID(), uuid)
@@ -53,7 +52,7 @@ class NodesTests(NeoUnitTestBase):
 
     def testState(self):
         """ Check if the last changed time is updated when state is changed """
-        node = Node(self.manager)
+        node = Node(self.nm)
         self.assertEqual(node.getState(), NodeStates.UNKNOWN)
         self.assertTrue(time() - 1 < node.getLastStateChange() < time())
         previous_time = node.getLastStateChange()
@@ -64,7 +63,7 @@ class NodesTests(NeoUnitTestBase):
 
     def testAddress(self):
         """ Check if the node is indexed by address """
-        node = Node(self.manager)
+        node = Node(self.nm)
         self.assertEqual(node.getAddress(), None)
         address = ('127.0.0.1', 10000)
         node.setAddress(address)
@@ -72,107 +71,55 @@ class NodesTests(NeoUnitTestBase):
 
     def testUUID(self):
         """ As for Address but UUID """
-        node = Node(self.manager)
+        node = Node(self.nm)
         self.assertEqual(node.getAddress(), None)
         uuid = self.getNewUUID(None)
         node.setUUID(uuid)
         self._updatedByUUID(node)
 
-    def testTypes(self):
-        """ Check that the abstract node has no type """
-        node = Node(self.manager)
-        self.assertRaises(NotImplementedError, node.getType)
-        self.assertFalse(node.isStorage())
-        self.assertFalse(node.isMaster())
-        self.assertFalse(node.isClient())
-        self.assertFalse(node.isAdmin())
-
-    def testMaster(self):
-        """ Check Master sub class """
-        node = MasterNode(self.manager)
-        self.assertEqual(node.getType(), protocol.NodeTypes.MASTER)
-        self.assertTrue(node.isMaster())
-        self.assertFalse(node.isStorage())
-        self.assertFalse(node.isClient())
-        self.assertFalse(node.isAdmin())
-
-    def testStorage(self):
-        """ Check Storage sub class """
-        node = StorageNode(self.manager)
-        self.assertEqual(node.getType(), protocol.NodeTypes.STORAGE)
-        self.assertTrue(node.isStorage())
-        self.assertFalse(node.isMaster())
-        self.assertFalse(node.isClient())
-        self.assertFalse(node.isAdmin())
-
-    def testClient(self):
-        """ Check Client sub class """
-        node = ClientNode(self.manager)
-        self.assertEqual(node.getType(), protocol.NodeTypes.CLIENT)
-        self.assertTrue(node.isClient())
-        self.assertFalse(node.isMaster())
-        self.assertFalse(node.isStorage())
-        self.assertFalse(node.isAdmin())
-
-    def testAdmin(self):
-        """ Check Admin sub class """
-        node = AdminNode(self.manager)
-        self.assertEqual(node.getType(), protocol.NodeTypes.ADMIN)
-        self.assertTrue(node.isAdmin())
-        self.assertFalse(node.isMaster())
-        self.assertFalse(node.isStorage())
-        self.assertFalse(node.isClient())
-
 
 class NodeManagerTests(NeoUnitTestBase):
 
-    def setUp(self):
-        NeoUnitTestBase.setUp(self)
-        self.manager = NodeManager()
-
     def _addStorage(self):
-        self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
+        self.storage = self.nm.createStorage(
+            address=('127.0.0.1', 1000), uuid=self.getStorageUUID())
 
     def _addMaster(self):
-        self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID())
+        self.master = self.nm.createMaster(
+            address=('127.0.0.1', 2000), uuid=self.getMasterUUID())
 
     def _addClient(self):
-        self.client = ClientNode(self.manager, None, self.getClientUUID())
+        self.client = self.nm.createClient(uuid=self.getClientUUID())
 
     def _addAdmin(self):
-        self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID())
+        self.admin = self.nm.createAdmin(
+            address=('127.0.0.1', 4000), uuid=self.getAdminUUID())
 
     def checkNodes(self, node_list):
-        manager = self.manager
-        self.assertEqual(sorted(manager.getList()), sorted(node_list))
+        self.assertEqual(sorted(self.nm.getList()), sorted(node_list))
 
     def checkMasters(self, master_list):
-        manager = self.manager
-        self.assertEqual(manager.getMasterList(), master_list)
+        self.assertEqual(self.nm.getMasterList(), master_list)
 
     def checkStorages(self, storage_list):
-        manager = self.manager
-        self.assertEqual(manager.getStorageList(), storage_list)
+        self.assertEqual(self.nm.getStorageList(), storage_list)
 
     def checkClients(self, client_list):
-        manager = self.manager
-        self.assertEqual(manager.getClientList(), client_list)
+        self.assertEqual(self.nm.getClientList(), client_list)
 
     def checkByServer(self, node):
-        node_found = self.manager.getByAddress(node.getAddress())
-        self.assertEqual(node_found, node)
+        self.assertEqual(node, self.nm.getByAddress(node.getAddress()))
 
     def checkByUUID(self, node):
-        node_found = self.manager.getByUUID(node.getUUID())
-        self.assertEqual(node_found, node)
+        self.assertEqual(node, self.nm.getByUUID(node.getUUID()))
 
     def checkIdentified(self, node_list, pool_set=None):
-        identified_node_list = self.manager.getIdentifiedList(pool_set)
+        identified_node_list = self.nm.getIdentifiedList(pool_set)
         self.assertEqual(set(identified_node_list), set(node_list))
 
     def testInit(self):
         """ Check the manager is empty when started """
-        manager = self.manager
+        manager = self.nm
         self.checkNodes([])
         self.checkMasters([])
         self.checkStorages([])
@@ -186,7 +133,7 @@ class NodeManagerTests(NeoUnitTestBase):
 
     def testAdd(self):
         """ Check if new nodes are registered in the manager """
-        manager = self.manager
+        manager = self.nm
         self.checkNodes([])
         # storage
         self._addStorage()
@@ -225,7 +172,7 @@ class NodeManagerTests(NeoUnitTestBase):
     def testUpdate(self):
         """ Check manager content update """
         # set up four nodes
-        manager = self.manager
+        manager = self.nm
         self._addMaster()
         self._addStorage()
         self._addClient()
@@ -268,7 +215,6 @@ class NodeManagerTests(NeoUnitTestBase):
 
     def testIdentified(self):
         # set up four nodes
-        manager = self.manager
         self._addMaster()
         self._addStorage()
         self._addClient()

neo/tests/testPT.py

@@ -18,7 +18,6 @@ import unittest
 from mock import Mock
 from neo.lib.protocol import NodeStates, CellStates
 from neo.lib.pt import Cell, PartitionTable, PartitionTableException
-from neo.lib.node import StorageNode
 from . import NeoUnitTestBase
 
 class PartitionTableTests(NeoUnitTestBase):
@@ -26,7 +25,7 @@ class PartitionTableTests(NeoUnitTestBase):
     def test_01_Cell(self):
         uuid = self.getStorageUUID()
         server = ("127.0.0.1", 19001)
-        sn = StorageNode(Mock(), server, uuid)
+        sn = self.createStorage(server, uuid)
         cell = Cell(sn)
         self.assertEqual(cell.node, sn)
         self.assertEqual(cell.state, CellStates.UP_TO_DATE)
@@ -50,7 +49,7 @@ class PartitionTableTests(NeoUnitTestBase):
         pt = PartitionTable(num_partitions, num_replicas)
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         for x in xrange(num_partitions):
             self.assertEqual(len(pt.partition_list[x]), 0)
         # add a cell to an empty row
@@ -131,7 +130,7 @@ class PartitionTableTests(NeoUnitTestBase):
         pt = PartitionTable(num_partitions, num_replicas)
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         for x in xrange(num_partitions):
             self.assertEqual(len(pt.partition_list[x]), 0)
         # add a cell to an empty row
@@ -171,19 +170,19 @@ class PartitionTableTests(NeoUnitTestBase):
         # add two kind of node, usable and unusable
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
         uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
-        sn3 = StorageNode(Mock(), server3, uuid3)
+        sn3 = self.createStorage(server3, uuid3)
         pt.setCell(0, sn3, CellStates.FEEDING)
         uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
-        sn4 = StorageNode(Mock(), server4, uuid4)
+        sn4 = self.createStorage(server4, uuid4)
         pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
         # now checks result
         self.assertEqual(len(pt.partition_list[0]), 3)
@@ -217,15 +216,15 @@ class PartitionTableTests(NeoUnitTestBase):
         # add two kind of node, usable and unusable
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         pt.setCell(1, sn2, CellStates.OUT_OF_DATE)
         uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
-        sn3 = StorageNode(Mock(), server3, uuid3)
+        sn3 = self.createStorage(server3, uuid3)
         pt.setCell(2, sn3, CellStates.FEEDING)
         # now checks result
         self.assertEqual(len(pt.partition_list[0]), 1)
@@ -247,19 +246,19 @@ class PartitionTableTests(NeoUnitTestBase):
         # add two kind of node, usable and unusable
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
         uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
-        sn3 = StorageNode(Mock(), server3, uuid3)
+        sn3 = self.createStorage(server3, uuid3)
         pt.setCell(0, sn3, CellStates.FEEDING)
         uuid4 = self.getStorageUUID()
         server4 = ("127.0.0.4", 19001)
-        sn4 = StorageNode(Mock(), server4, uuid4)
+        sn4 = self.createStorage(server4, uuid4)
         pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
         # must get only two node as feeding and discarded not taken
         # into account
@@ -276,7 +275,7 @@ class PartitionTableTests(NeoUnitTestBase):
         # adding a node in all partition
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         for x in xrange(num_partitions):
             pt.setCell(x, sn1, CellStates.UP_TO_DATE)
         self.assertEqual(pt.num_filled_rows, num_partitions)
@@ -289,7 +288,7 @@ class PartitionTableTests(NeoUnitTestBase):
         # add two kind of node, usable and unusable
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         # now test
         self.assertTrue(pt.hasOffset(0))
@@ -298,15 +297,16 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.hasOffset(50))
 
     def test_10_operational(self):
+        def createStorage():
+            uuid = self.getStorageUUID()
+            return self.createStorage(("127.0.0.1", uuid), uuid)
         num_partitions = 5
         num_replicas = 2
         pt = PartitionTable(num_partitions, num_replicas)
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getStorageUUID()
-        server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = createStorage()
         for x in xrange(num_partitions):
             pt.setCell(x, sn1, CellStates.UP_TO_DATE)
         self.assertTrue(pt.filled())
@@ -318,9 +318,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getStorageUUID()
-        server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = createStorage()
         for x in xrange(num_partitions):
             pt.setCell(x, sn1, CellStates.FEEDING)
         self.assertTrue(pt.filled())
@@ -333,9 +331,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getStorageUUID()
-        server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = createStorage()
         sn1.setState(NodeStates.TEMPORARILY_DOWN)
         for x in xrange(num_partitions):
             pt.setCell(x, sn1, CellStates.FEEDING)
@@ -348,9 +344,7 @@ class PartitionTableTests(NeoUnitTestBase):
         self.assertFalse(pt.filled())
         self.assertFalse(pt.operational())
         # adding a node in all partition
-        uuid1 = self.getStorageUUID()
-        server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = createStorage()
         for x in xrange(num_partitions):
             pt.setCell(x, sn1, CellStates.OUT_OF_DATE)
         self.assertTrue(pt.filled())
@@ -364,18 +358,18 @@ class PartitionTableTests(NeoUnitTestBase):
         # add nodes
         uuid1 = self.getStorageUUID()
         server1 = ("127.0.0.1", 19001)
-        sn1 = StorageNode(Mock(), server1, uuid1)
+        sn1 = self.createStorage(server1, uuid1)
         pt.setCell(0, sn1, CellStates.UP_TO_DATE)
         pt.setCell(1, sn1, CellStates.UP_TO_DATE)
         pt.setCell(2, sn1, CellStates.UP_TO_DATE)
         uuid2 = self.getStorageUUID()
         server2 = ("127.0.0.2", 19001)
-        sn2 = StorageNode(Mock(), server2, uuid2)
+        sn2 = self.createStorage(server2, uuid2)
         pt.setCell(0, sn2, CellStates.UP_TO_DATE)
         pt.setCell(1, sn2, CellStates.UP_TO_DATE)
         uuid3 = self.getStorageUUID()
         server3 = ("127.0.0.3", 19001)
-        sn3 = StorageNode(Mock(), server3, uuid3)
+        sn3 = self.createStorage(server3, uuid3)
         pt.setCell(0, sn3, CellStates.UP_TO_DATE)
         # test
         row_0 = pt.getRow(0)