Commit f4d44526 authored by Vincent Pelletier's avatar Vincent Pelletier

Update testMaster:

- Fix an inversion in killSecondaryMaster making it kill only primary nodes.
- Add DELAY_SAFETY_MARGIN to make explicit which part of a sleep delay is just here for safety, WRT the "mandatory" part of the delay.
- A disconnected secondary master node must be removed from NM, so its state becomes None.
- Add tests comparing the behaviour of a 2-masters and 3-masters cluster.


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@1158 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent ca187888
...@@ -18,6 +18,8 @@ from neo import protocol ...@@ -18,6 +18,8 @@ from neo import protocol
neo = NEOCluster([], port_base=20000, master_node_count=3) neo = NEOCluster([], port_base=20000, master_node_count=3)
DELAY_SAFETY_MARGIN = 5
class MasterTests(unittest.TestCase): class MasterTests(unittest.TestCase):
def setUp(self): def setUp(self):
...@@ -48,10 +50,13 @@ class MasterTests(unittest.TestCase): ...@@ -48,10 +50,13 @@ class MasterTests(unittest.TestCase):
return self._killMaster(primary=True) return self._killMaster(primary=True)
def killSecondaryMaster(self, all=False): def killSecondaryMaster(self, all=False):
return self._killMaster(primary=True, all=all) return self._killMaster(primary=False, all=all)
def getMasterNodeList(self):
return self.neoctl.getNodeList(protocol.MASTER_NODE_TYPE)
def getMasterNodeState(self, uuid): def getMasterNodeState(self, uuid):
node_list = self.neoctl.getNodeList(protocol.MASTER_NODE_TYPE) node_list = self.getMasterNodeList()
for node_type, address, node_uuid, state in node_list: for node_type, address, node_uuid, state in node_list:
if node_uuid == uuid: if node_uuid == uuid:
break break
...@@ -65,23 +70,102 @@ class MasterTests(unittest.TestCase): ...@@ -65,23 +70,102 @@ class MasterTests(unittest.TestCase):
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0] uuid = killed_uuid_list[0]
# Leave some time for the primary master to notice the disconnection. # Leave some time for the primary master to notice the disconnection.
time.sleep(1) time.sleep(DELAY_SAFETY_MARGIN)
# Check node state has changed. # Check node state has changed.
self.assertEqual(self.getMasterNodeState(uuid), protocol.TEMPORARILY_DOWN_STATE) self.assertEqual(self.getMasterNodeState(uuid), None)
def testStoppingPrimaryMaster(self): def testStoppingPrimaryMasterWithTwoSecondaries(self):
killed_uuid_list = self.killPrimaryMaster() killed_uuid_list = self.killPrimaryMaster()
# Test sanity check. # Test sanity check.
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0] uuid = killed_uuid_list[0]
# Leave some time for the other master to elect a new primary. # Leave some time for the disconnection to be noticed by the admin
time.sleep(1) # node.
time.sleep(DELAY_SAFETY_MARGIN)
# Check the state of the primary we just killed
self.assertEqual(self.getMasterNodeState(uuid),
protocol.TEMPORARILY_DOWN_STATE)
# Leave some time for the other masters to elect a new primary.
time.sleep(10)
# Check that a primary master arised. # Check that a primary master arised.
# (raises if admin node is not connected to a primary master) # (raises if admin node is not connected to a primary master)
new_uuid = self.neoctl.getPrimaryMaster() new_uuid = self.neoctl.getPrimaryMaster()
# Check that the uuid really changed. # Check that the uuid really changed.
self.assertNotEqual(new_uuid, uuid) self.assertNotEqual(new_uuid, uuid)
def testStoppingPrimaryMasterWithOneSecondary(self):
# Kill one secondary master.
killed_uuid_list = self.killSecondaryMaster()
# Test sanity checks.
self.assertEqual(len(killed_uuid_list), 1)
time.sleep(DELAY_SAFETY_MARGIN)
self.assertEqual(self.getMasterNodeState(killed_uuid_list[0]), None)
self.assertEqual(len(self.getMasterNodeList()), 2)
killed_uuid_list = self.killPrimaryMaster()
# Test sanity check.
self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0]
# Leave some time for the disconnection to be noticed by the admin
# node.
time.sleep(DELAY_SAFETY_MARGIN)
# Check the state of the primary we just killed
self.assertEqual(self.getMasterNodeState(uuid),
protocol.TEMPORARILY_DOWN_STATE)
# Leave some time for the other masters to elect a new primary.
time.sleep(10)
# Check that a primary master arised.
# (raises if admin node is not connected to a primary master)
new_uuid = self.neoctl.getPrimaryMaster()
# Check that the uuid really changed.
self.assertNotEqual(new_uuid, uuid)
def testMasterSequentialStart(self):
# Stop the cluster (so we can start processes manually)
neo.stop()
master_list = neo.getMasterProcessList()
# Test sanity check.
self.assertEqual(len(master_list), 3)
# Start the admin node.
neo.getAdminProcessList()[0].start()
# Start the first master.
first_master = master_list[0]
first_master.start()
# Leave some time for the master to elect itself without any other
# master.
time.sleep(60 + DELAY_SAFETY_MARGIN)
# Check that the master node we started elected itself.
self.assertEqual(self.neoctl.getPrimaryMaster(),
first_master.getUUID())
# Check that no other master node is known as running.
self.assertEqual(len(self.neoctl.getNodeList(
protocol.MASTER_NODE_TYPE)), 1)
# Start a second master.
second_master = master_list[1]
second_master.start()
# Leave the second master some time to connect to the primary master.
time.sleep(DELAY_SAFETY_MARGIN)
# Check that the second master is running under his known UUID.
self.assertEqual(self.getMasterNodeState(second_master.getUUID()),
protocol.RUNNING_STATE)
# Check that the primary master didn't change.
self.assertEqual(self.neoctl.getPrimaryMaster(),
first_master.getUUID())
# Start a third master.
third_master = master_list[2]
third_master.start()
# Leave the third master some time to connect to the primary master.
time.sleep(DELAY_SAFETY_MARGIN)
# Check that the third master is running under his known UUID.
self.assertEqual(self.getMasterNodeState(third_master.getUUID()),
protocol.RUNNING_STATE)
# Check that the primary master didn't change.
self.assertEqual(self.neoctl.getPrimaryMaster(),
first_master.getUUID())
def test_suite(): def test_suite():
return unittest.makeSuite(MasterTests) return unittest.makeSuite(MasterTests)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment