Skip to content

N
neoppod
Commit 5941b27d authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

lib.node: code refactoring

parent c17f5f91
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 124 additions and 289 deletions
neo/lib/node.py
View file @ 5941b27d
This diff is collapsed.
neo/tests/__init__.py
View file @ 5941b27d
......@@ -32,6 +32,7 @@ from functools import wraps
from mock import Mock
from neo.lib import debug, logging, protocol
from neo.lib.protocol import NodeTypes, Packets, UUID_NAMESPACES
from neo.lib.util import cached_property
from time import time
from struct import pack, unpack
from unittest.case import _ExpectedFailure, _UnexpectedSuccess
......@@ -194,6 +195,15 @@ class NeoUnitTestBase(NeoTestBase):
self.uuid_dict = {}
NeoTestBase.setUp(self)
@cached_property
def nm(self):
from neo.lib import node
return node.NodeManager()
def createStorage(self, *args):
return self.nm.createStorage(**dict(zip(
('address', 'uuid', 'state'), args)))
def prepareDatabase(self, number, prefix=DB_PREFIX):
""" create empty databases """
adapter = os.getenv('NEO_TESTS_ADAPTER', 'MySQL')
......
neo/tests/master/testMasterPT.py
View file @ 5941b27d
......@@ -21,7 +21,6 @@ from .. import NeoUnitTestBase
from neo.lib.protocol import NodeStates, CellStates
from neo.lib.pt import PartitionTableException
from neo.master.pt import PartitionTable
from neo.lib.node import StorageNode
class MasterPartitionTableTests(NeoUnitTestBase):
......@@ -55,19 +54,19 @@ class MasterPartitionTableTests(NeoUnitTestBase):
# create nodes
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19002)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19003)
sn3 = StorageNode(Mock(), server3, uuid3)
sn3 = self.createStorage(server3, uuid3)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19004)
sn4 = StorageNode(Mock(), server4, uuid4)
sn4 = self.createStorage(server4, uuid4)
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 19005)
sn5 = StorageNode(Mock(), server5, uuid5)
sn5 = self.createStorage(server5, uuid5)
# create partition table
num_partitions = 5
num_replicas = 3
......@@ -117,7 +116,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
def test_15_dropNodeList(self):
sn = [StorageNode(Mock(), None, i + 1, NodeStates.RUNNING)
sn = [self.createStorage(None, i + 1, NodeStates.RUNNING)
for i in xrange(3)]
pt = PartitionTable(3, 0)
pt.setCell(0, sn[0], CellStates.OUT_OF_DATE)
......@@ -153,22 +152,22 @@ class MasterPartitionTableTests(NeoUnitTestBase):
# add nodes
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING)
sn1 = self.createStorage(server1, uuid1, NodeStates.RUNNING)
# add not running node
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
sn2.setState(NodeStates.TEMPORARILY_DOWN)
# add node without uuid
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, None, NodeStates.RUNNING)
sn3 = self.createStorage(server3, None, NodeStates.RUNNING)
# add clear node
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING)
sn4 = self.createStorage(server4, uuid4, NodeStates.RUNNING)
uuid5 = self.getStorageUUID()
server5 = ("127.0.0.5", 1900)
sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING)
sn5 = self.createStorage(server5, uuid5, NodeStates.RUNNING)
# make the table
pt.make([sn1, sn2, sn3, sn4, sn5])
# check it's ok, only running nodes and node with uuid
......@@ -231,7 +230,7 @@ class MasterPartitionTableTests(NeoUnitTestBase):
return change_list
def test_17_tweak(self):
sn = [StorageNode(Mock(), None, i + 1, NodeStates.RUNNING)
sn = [self.createStorage(None, i + 1, NodeStates.RUNNING)
for i in xrange(5)]
pt = PartitionTable(5, 2)
# part 0
......
neo/tests/testNodes.py
View file @ 5941b27d
......@@ -18,8 +18,7 @@ import unittest
from mock import Mock
from neo.lib import protocol
from neo.lib.protocol import NodeTypes, NodeStates
from neo.lib.node import Node, MasterNode, StorageNode, \
ClientNode, AdminNode, NodeManager, MasterDB
from neo.lib.node import Node, NodeManager, MasterDB
from . import NeoUnitTestBase, getTempDirectory
from time import time
from os import chmod, mkdir, rmdir, unlink
......@@ -29,15 +28,15 @@ class NodesTests(NeoUnitTestBase):
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = Mock()
self.nm = Mock()
def _updatedByAddress(self, node, index=0):
calls = self.manager.mockGetNamedCalls('_updateAddress')
calls = self.nm.mockGetNamedCalls('_updateAddress')
self.assertEqual(len(calls), index + 1)
self.assertEqual(calls[index].getParam(0), node)
def _updatedByUUID(self, node, index=0):
calls = self.manager.mockGetNamedCalls('_updateUUID')
calls = self.nm.mockGetNamedCalls('_updateUUID')
self.assertEqual(len(calls), index + 1)
self.assertEqual(calls[index].getParam(0), node)
......@@ -45,7 +44,7 @@ class NodesTests(NeoUnitTestBase):
""" Check the node initialization """
address = ('127.0.0.1', 10000)
uuid = self.getNewUUID(None)
node = Node(self.manager, address=address, uuid=uuid)
node = Node(self.nm, address=address, uuid=uuid)
self.assertEqual(node.getState(), NodeStates.UNKNOWN)
self.assertEqual(node.getAddress(), address)
self.assertEqual(node.getUUID(), uuid)
......@@ -53,7 +52,7 @@ class NodesTests(NeoUnitTestBase):
def testState(self):
""" Check if the last changed time is updated when state is changed """
node = Node(self.manager)
node = Node(self.nm)
self.assertEqual(node.getState(), NodeStates.UNKNOWN)
self.assertTrue(time() - 1 < node.getLastStateChange() < time())
previous_time = node.getLastStateChange()
......@@ -64,7 +63,7 @@ class NodesTests(NeoUnitTestBase):
def testAddress(self):
""" Check if the node is indexed by address """
node = Node(self.manager)
node = Node(self.nm)
self.assertEqual(node.getAddress(), None)
address = ('127.0.0.1', 10000)
node.setAddress(address)
......@@ -72,107 +71,55 @@ class NodesTests(NeoUnitTestBase):
def testUUID(self):
""" As for Address but UUID """
node = Node(self.manager)
node = Node(self.nm)
self.assertEqual(node.getAddress(), None)
uuid = self.getNewUUID(None)
node.setUUID(uuid)
self._updatedByUUID(node)
def testTypes(self):
""" Check that the abstract node has no type """
node = Node(self.manager)
self.assertRaises(NotImplementedError, node.getType)
self.assertFalse(node.isStorage())
self.assertFalse(node.isMaster())
self.assertFalse(node.isClient())
self.assertFalse(node.isAdmin())
def testMaster(self):
""" Check Master sub class """
node = MasterNode(self.manager)
self.assertEqual(node.getType(), protocol.NodeTypes.MASTER)
self.assertTrue(node.isMaster())
self.assertFalse(node.isStorage())
self.assertFalse(node.isClient())
self.assertFalse(node.isAdmin())
def testStorage(self):
""" Check Storage sub class """
node = StorageNode(self.manager)
self.assertEqual(node.getType(), protocol.NodeTypes.STORAGE)
self.assertTrue(node.isStorage())
self.assertFalse(node.isMaster())
self.assertFalse(node.isClient())
self.assertFalse(node.isAdmin())
def testClient(self):
""" Check Client sub class """
node = ClientNode(self.manager)
self.assertEqual(node.getType(), protocol.NodeTypes.CLIENT)
self.assertTrue(node.isClient())
self.assertFalse(node.isMaster())
self.assertFalse(node.isStorage())
self.assertFalse(node.isAdmin())
def testAdmin(self):
""" Check Admin sub class """
node = AdminNode(self.manager)
self.assertEqual(node.getType(), protocol.NodeTypes.ADMIN)
self.assertTrue(node.isAdmin())
self.assertFalse(node.isMaster())
self.assertFalse(node.isStorage())
self.assertFalse(node.isClient())
class NodeManagerTests(NeoUnitTestBase):
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = NodeManager()
def _addStorage(self):
self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
self.storage = self.nm.createStorage(
address=('127.0.0.1', 1000), uuid=self.getStorageUUID())
def _addMaster(self):
self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID())
self.master = self.nm.createMaster(
address=('127.0.0.1', 2000), uuid=self.getMasterUUID())
def _addClient(self):
self.client = ClientNode(self.manager, None, self.getClientUUID())
self.client = self.nm.createClient(uuid=self.getClientUUID())
def _addAdmin(self):
self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID())
self.admin = self.nm.createAdmin(
address=('127.0.0.1', 4000), uuid=self.getAdminUUID())
def checkNodes(self, node_list):
manager = self.manager
self.assertEqual(sorted(manager.getList()), sorted(node_list))
self.assertEqual(sorted(self.nm.getList()), sorted(node_list))
def checkMasters(self, master_list):
manager = self.manager
self.assertEqual(manager.getMasterList(), master_list)
self.assertEqual(self.nm.getMasterList(), master_list)
def checkStorages(self, storage_list):
manager = self.manager
self.assertEqual(manager.getStorageList(), storage_list)
self.assertEqual(self.nm.getStorageList(), storage_list)
def checkClients(self, client_list):
manager = self.manager
self.assertEqual(manager.getClientList(), client_list)
self.assertEqual(self.nm.getClientList(), client_list)
def checkByServer(self, node):
node_found = self.manager.getByAddress(node.getAddress())
self.assertEqual(node_found, node)
self.assertEqual(node, self.nm.getByAddress(node.getAddress()))
def checkByUUID(self, node):
node_found = self.manager.getByUUID(node.getUUID())
self.assertEqual(node_found, node)
self.assertEqual(node, self.nm.getByUUID(node.getUUID()))
def checkIdentified(self, node_list, pool_set=None):
identified_node_list = self.manager.getIdentifiedList(pool_set)
identified_node_list = self.nm.getIdentifiedList(pool_set)
self.assertEqual(set(identified_node_list), set(node_list))
def testInit(self):
""" Check the manager is empty when started """
manager = self.manager
manager = self.nm
self.checkNodes([])
self.checkMasters([])
self.checkStorages([])
......@@ -186,7 +133,7 @@ class NodeManagerTests(NeoUnitTestBase):
def testAdd(self):
""" Check if new nodes are registered in the manager """
manager = self.manager
manager = self.nm
self.checkNodes([])
# storage
self._addStorage()
......@@ -225,7 +172,7 @@ class NodeManagerTests(NeoUnitTestBase):
def testUpdate(self):
""" Check manager content update """
# set up four nodes
manager = self.manager
manager = self.nm
self._addMaster()
self._addStorage()
self._addClient()
......@@ -268,7 +215,6 @@ class NodeManagerTests(NeoUnitTestBase):
def testIdentified(self):
# set up four nodes
manager = self.manager
self._addMaster()
self._addStorage()
self._addClient()
......
neo/tests/testPT.py
View file @ 5941b27d
......@@ -18,7 +18,6 @@ import unittest
from mock import Mock
from neo.lib.protocol import NodeStates, CellStates
from neo.lib.pt import Cell, PartitionTable, PartitionTableException
from neo.lib.node import StorageNode
from . import NeoUnitTestBase
class PartitionTableTests(NeoUnitTestBase):
......@@ -26,7 +25,7 @@ class PartitionTableTests(NeoUnitTestBase):
def test_01_Cell(self):
uuid = self.getStorageUUID()
server = ("127.0.0.1", 19001)
sn = StorageNode(Mock(), server, uuid)
sn = self.createStorage(server, uuid)
cell = Cell(sn)
self.assertEqual(cell.node, sn)
self.assertEqual(cell.state, CellStates.UP_TO_DATE)
......@@ -50,7 +49,7 @@ class PartitionTableTests(NeoUnitTestBase):
pt = PartitionTable(num_partitions, num_replicas)
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
# add a cell to an empty row
......@@ -131,7 +130,7 @@ class PartitionTableTests(NeoUnitTestBase):
pt = PartitionTable(num_partitions, num_replicas)
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
for x in xrange(num_partitions):
self.assertEqual(len(pt.partition_list[x]), 0)
# add a cell to an empty row
......@@ -171,19 +170,19 @@ class PartitionTableTests(NeoUnitTestBase):
# add two kind of node, usable and unusable
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
sn4 = self.createStorage(server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
# now checks result
self.assertEqual(len(pt.partition_list[0]), 3)
......@@ -217,15 +216,15 @@ class PartitionTableTests(NeoUnitTestBase):
# add two kind of node, usable and unusable
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(1, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(2, sn3, CellStates.FEEDING)
# now checks result
self.assertEqual(len(pt.partition_list[0]), 1)
......@@ -247,19 +246,19 @@ class PartitionTableTests(NeoUnitTestBase):
# add two kind of node, usable and unusable
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.OUT_OF_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.FEEDING)
uuid4 = self.getStorageUUID()
server4 = ("127.0.0.4", 19001)
sn4 = StorageNode(Mock(), server4, uuid4)
sn4 = self.createStorage(server4, uuid4)
pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added
# must get only two node as feeding and discarded not taken
# into account
......@@ -276,7 +275,7 @@ class PartitionTableTests(NeoUnitTestBase):
# adding a node in all partition
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.UP_TO_DATE)
self.assertEqual(pt.num_filled_rows, num_partitions)
......@@ -289,7 +288,7 @@ class PartitionTableTests(NeoUnitTestBase):
# add two kind of node, usable and unusable
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
# now test
self.assertTrue(pt.hasOffset(0))
......@@ -298,15 +297,16 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.hasOffset(50))
def test_10_operational(self):
def createStorage():
uuid = self.getStorageUUID()
return self.createStorage(("127.0.0.1", uuid), uuid)
num_partitions = 5
num_replicas = 2
pt = PartitionTable(num_partitions, num_replicas)
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.UP_TO_DATE)
self.assertTrue(pt.filled())
......@@ -318,9 +318,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.FEEDING)
self.assertTrue(pt.filled())
......@@ -333,9 +331,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = createStorage()
sn1.setState(NodeStates.TEMPORARILY_DOWN)
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.FEEDING)
......@@ -348,9 +344,7 @@ class PartitionTableTests(NeoUnitTestBase):
self.assertFalse(pt.filled())
self.assertFalse(pt.operational())
# adding a node in all partition
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = createStorage()
for x in xrange(num_partitions):
pt.setCell(x, sn1, CellStates.OUT_OF_DATE)
self.assertTrue(pt.filled())
......@@ -364,18 +358,18 @@ class PartitionTableTests(NeoUnitTestBase):
# add nodes
uuid1 = self.getStorageUUID()
server1 = ("127.0.0.1", 19001)
sn1 = StorageNode(Mock(), server1, uuid1)
sn1 = self.createStorage(server1, uuid1)
pt.setCell(0, sn1, CellStates.UP_TO_DATE)
pt.setCell(1, sn1, CellStates.UP_TO_DATE)
pt.setCell(2, sn1, CellStates.UP_TO_DATE)
uuid2 = self.getStorageUUID()
server2 = ("127.0.0.2", 19001)
sn2 = StorageNode(Mock(), server2, uuid2)
sn2 = self.createStorage(server2, uuid2)
pt.setCell(0, sn2, CellStates.UP_TO_DATE)
pt.setCell(1, sn2, CellStates.UP_TO_DATE)
uuid3 = self.getStorageUUID()
server3 = ("127.0.0.3", 19001)
sn3 = StorageNode(Mock(), server3, uuid3)
sn3 = self.createStorage(server3, uuid3)
pt.setCell(0, sn3, CellStates.UP_TO_DATE)
# test
row_0 = pt.getRow(0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7