Commit d4fc6235 authored by Grégory Wisniewski's avatar Grégory Wisniewski

Move expect* helpers from testMaster.


git-svn-id: https://svn.erp5.org/repos/neo/branches/prototype3@1198 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent d51bf457
...@@ -62,6 +62,8 @@ NEO_MASTER = 'neomaster' ...@@ -62,6 +62,8 @@ NEO_MASTER = 'neomaster'
NEO_STORAGE = 'neostorage' NEO_STORAGE = 'neostorage'
NEO_ADMIN = 'neoadmin' NEO_ADMIN = 'neoadmin'
DELAY_SAFETY_MARGIN = 5
class AlreadyRunning(Exception): class AlreadyRunning(Exception):
pass pass
...@@ -321,10 +323,16 @@ class NEOCluster(object): ...@@ -321,10 +323,16 @@ class NEOCluster(object):
primary_list = self.killPrimaryMaster() primary_list = self.killPrimaryMaster()
return secondary_list + primary_list return secondary_list + primary_list
def getMasterNodeList(self, state=None): def __getNodeList(self, node_type, state=None):
return [x for x in self.neoctl.getNodeList(protocol.MASTER_NODE_TYPE) return [x for x in self.neoctl.getNodeList(node_type)
if state is None or x[3] == state] if state is None or x[3] == state]
def getMasterNodeList(self, state=None):
return self.__getNodeList(protocol.MASTER_NODE_TYPE, state)
def getStorageNodeList(self, state=None):
return self.__getNodeList(protocol.STORAGE_NODE_TYPE, state)
def getMasterNodeState(self, uuid): def getMasterNodeState(self, uuid):
node_list = self.getMasterNodeList() node_list = self.getMasterNodeList()
for node_type, address, node_uuid, state in node_list: for node_type, address, node_uuid, state in node_list:
...@@ -341,6 +349,48 @@ class NEOCluster(object): ...@@ -341,6 +349,48 @@ class NEOCluster(object):
current_try = None current_try = None
return current_try return current_try
def expectCondition(self, condition, timeout, delay):
end = time.time() + timeout + DELAY_SAFETY_MARGIN
opaque = None
opaque_history = []
while time.time() < end:
reached, opaque = condition(opaque)
if reached:
break
else:
opaque_history.append(opaque)
time.sleep(delay)
else:
raise AssertionError, 'Timeout while expecting condition. ' \
'History: %s' % (opaque_history, )
def expectAllMasters(self, node_count, state=None, timeout=0, delay=1):
def callback(last_try):
current_try = len(self.getMasterNodeList(state=state))
if last_try is not None and current_try < last_try:
raise AssertionError, 'Regression: %s became %s' % \
(last_try, current_try)
return (current_try == node_count, current_try)
self.expectCondition(callback, timeout, delay)
def expectMasterState(self, uuid, state, timeout=0, delay=1):
if not isinstance(state, (tuple, list)):
state = (state, )
def callback(last_try):
current_try = self.getMasterNodeState(uuid)
return current_try in state, current_try
self.expectCondition(callback, timeout, delay)
def expectPrimaryMaster(self, uuid=None, timeout=0, delay=1):
def callback(last_try):
current_try = self.getPrimaryMaster()
if None not in (uuid, current_try) and uuid != current_try:
raise AssertionError, 'An unexpected primary arised: %r, ' \
'expected %r' % (dump(current_try), dump(uuid))
return uuid is None or uuid == current_try, current_try
self.expectCondition(callback, timeout, delay)
def __del__(self): def __del__(self):
if self.cleanup_on_delete: if self.cleanup_on_delete:
os.removedirs(self.temp_dir) os.removedirs(self.temp_dir)
......
...@@ -22,7 +22,6 @@ from neo.neoctl.neoctl import NotReadyException ...@@ -22,7 +22,6 @@ from neo.neoctl.neoctl import NotReadyException
from neo import protocol from neo import protocol
from neo.util import dump from neo.util import dump
DELAY_SAFETY_MARGIN = 5
MASTER_NODE_COUNT = 3 MASTER_NODE_COUNT = 3
neo = NEOCluster([], port_base=20000, master_node_count=MASTER_NODE_COUNT) neo = NEOCluster([], port_base=20000, master_node_count=MASTER_NODE_COUNT)
...@@ -38,50 +37,9 @@ class MasterTests(unittest.TestCase): ...@@ -38,50 +37,9 @@ class MasterTests(unittest.TestCase):
def tearDown(self): def tearDown(self):
neo.stop() neo.stop()
def expectCondition(self, condition, timeout, delay):
end = time() + timeout + DELAY_SAFETY_MARGIN
opaque = None
opaque_history = []
while time() < end:
reached, opaque = condition(opaque)
if reached:
break
else:
opaque_history.append(opaque)
sleep(delay)
else:
raise AssertionError, 'Timeout while expecting condition. ' \
'History: %s' % (opaque_history, )
def expectAllMasters(self, state=None, timeout=0, delay=1):
def callback(last_try):
current_try = len(neo.getMasterNodeList(state=state))
if last_try is not None and current_try < last_try:
raise AssertionError, 'Regression: %s became %s' % \
(last_try, current_try)
return (current_try == MASTER_NODE_COUNT, current_try)
self.expectCondition(callback, timeout, delay)
def expectMasterState(self, uuid, state, timeout=0, delay=1):
if not isinstance(state, (tuple, list)):
state = (state, )
def callback(last_try):
current_try = neo.getMasterNodeState(uuid)
return current_try in state, current_try
self.expectCondition(callback, timeout, delay)
def expectPrimaryMaster(self, uuid=None, timeout=0, delay=1):
def callback(last_try):
current_try = neo.getPrimaryMaster()
if None not in (uuid, current_try) and uuid != current_try:
raise AssertionError, 'An unexpected primary arised: %r, ' \
'expected %r' % (dump(current_try), dump(uuid))
return uuid is None or uuid == current_try, current_try
self.expectCondition(callback, timeout, delay)
def testStoppingSecondaryMaster(self): def testStoppingSecondaryMaster(self):
# Wait for masters to stabilize # Wait for masters to stabilize
self.expectAllMasters() neo.expectAllMasters(MASTER_NODE_COUNT)
# Kill # Kill
killed_uuid_list = neo.killSecondaryMaster() killed_uuid_list = neo.killSecondaryMaster()
...@@ -89,11 +47,11 @@ class MasterTests(unittest.TestCase): ...@@ -89,11 +47,11 @@ class MasterTests(unittest.TestCase):
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0] uuid = killed_uuid_list[0]
# Check node state has changed. # Check node state has changed.
self.expectMasterState(uuid, None) neo.expectMasterState(uuid, None)
def testStoppingPrimaryMasterWithTwoSecondaries(self): def testStoppingPrimaryMasterWithTwoSecondaries(self):
# Wait for masters to stabilize # Wait for masters to stabilize
self.expectAllMasters() neo.expectAllMasters(MASTER_NODE_COUNT)
# Kill # Kill
killed_uuid_list = neo.killPrimaryMaster() killed_uuid_list = neo.killPrimaryMaster()
...@@ -101,22 +59,22 @@ class MasterTests(unittest.TestCase): ...@@ -101,22 +59,22 @@ class MasterTests(unittest.TestCase):
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0] uuid = killed_uuid_list[0]
# Check the state of the primary we just killed # Check the state of the primary we just killed
self.expectMasterState(uuid, (None, protocol.UNKNOWN_STATE)) neo.expectMasterState(uuid, (None, protocol.UNKNOWN_STATE))
self.assertEqual(self.getPrimaryMaster(), None) self.assertEqual(neo.getPrimaryMaster(), None)
# Check that a primary master arised. # Check that a primary master arised.
self.expectPrimaryMaster(timeout=10) neo.expectPrimaryMaster(timeout=10)
# Check that the uuid really changed. # Check that the uuid really changed.
new_uuid = self.getPrimaryMaster() new_uuid = neo.getPrimaryMaster()
self.assertNotEqual(new_uuid, uuid) self.assertNotEqual(new_uuid, uuid)
def testStoppingPrimaryMasterWithOneSecondary(self): def testStoppingPrimaryMasterWithOneSecondary(self):
self.expectAllMasters(state=protocol.RUNNING_STATE) neo.expectAllMasters(MASTER_NODE_COUNT, state=protocol.RUNNING_STATE)
# Kill one secondary master. # Kill one secondary master.
killed_uuid_list = neo.killSecondaryMaster() killed_uuid_list = neo.killSecondaryMaster()
# Test sanity checks. # Test sanity checks.
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
self.expectMasterState(killed_uuid_list[0], None) neo.expectMasterState(killed_uuid_list[0], None)
self.assertEqual(len(neo.getMasterNodeList()), 2) self.assertEqual(len(neo.getMasterNodeList()), 2)
killed_uuid_list = neo.killPrimaryMaster() killed_uuid_list = neo.killPrimaryMaster()
...@@ -124,16 +82,16 @@ class MasterTests(unittest.TestCase): ...@@ -124,16 +82,16 @@ class MasterTests(unittest.TestCase):
self.assertEqual(len(killed_uuid_list), 1) self.assertEqual(len(killed_uuid_list), 1)
uuid = killed_uuid_list[0] uuid = killed_uuid_list[0]
# Check the state of the primary we just killed # Check the state of the primary we just killed
self.expectMasterState(uuid, (None, protocol.UNKNOWN_STATE)) neo.expectMasterState(uuid, (None, protocol.UNKNOWN_STATE))
self.assertEqual(self.getPrimaryMaster(), None) self.assertEqual(neo.getPrimaryMaster(), None)
# Check that a primary master arised. # Check that a primary master arised.
self.expectPrimaryMaster(timeout=10) neo.expectPrimaryMaster(timeout=10)
# Check that the uuid really changed. # Check that the uuid really changed.
new_uuid = self.getPrimaryMaster() new_uuid = neo.getPrimaryMaster()
self.assertNotEqual(new_uuid, uuid) self.assertNotEqual(new_uuid, uuid)
def testMasterSequentialStart(self): def testMasterSequentialStart(self):
self.expectAllMasters(state=protocol.RUNNING_STATE) neo.expectAllMasters(MASTER_NODE_COUNT, state=protocol.RUNNING_STATE)
master_list = neo.getMasterProcessList() master_list = neo.getMasterProcessList()
# Stop the cluster (so we can start processes manually) # Stop the cluster (so we can start processes manually)
...@@ -144,7 +102,7 @@ class MasterTests(unittest.TestCase): ...@@ -144,7 +102,7 @@ class MasterTests(unittest.TestCase):
first_master.start() first_master.start()
first_master_uuid = first_master.getUUID() first_master_uuid = first_master.getUUID()
# Check that the master node we started elected itself. # Check that the master node we started elected itself.
self.expectPrimaryMaster(first_master_uuid, timeout=60) neo.expectPrimaryMaster(first_master_uuid, timeout=60)
# Check that no other node is known as running. # Check that no other node is known as running.
self.assertEqual(len(neo.getMasterNodeList( self.assertEqual(len(neo.getMasterNodeList(
state=protocol.RUNNING_STATE)), 1) state=protocol.RUNNING_STATE)), 1)
...@@ -155,7 +113,7 @@ class MasterTests(unittest.TestCase): ...@@ -155,7 +113,7 @@ class MasterTests(unittest.TestCase):
self.assertEqual(neo.getMasterNodeState(second_master.getUUID()), None) self.assertEqual(neo.getMasterNodeState(second_master.getUUID()), None)
second_master.start() second_master.start()
# Check that the second master is running under his known UUID. # Check that the second master is running under his known UUID.
self.expectMasterState(second_master.getUUID(), protocol.RUNNING_STATE) neo.expectMasterState(second_master.getUUID(), protocol.RUNNING_STATE)
# Check that the primary master didn't change. # Check that the primary master didn't change.
self.assertEqual(neo.getPrimaryMaster(), first_master_uuid) self.assertEqual(neo.getPrimaryMaster(), first_master_uuid)
...@@ -165,7 +123,7 @@ class MasterTests(unittest.TestCase): ...@@ -165,7 +123,7 @@ class MasterTests(unittest.TestCase):
self.assertEqual(neo.getMasterNodeState(third_master.getUUID()), None) self.assertEqual(neo.getMasterNodeState(third_master.getUUID()), None)
third_master.start() third_master.start()
# Check that the third master is running under his known UUID. # Check that the third master is running under his known UUID.
self.expectMasterState(third_master.getUUID(), protocol.RUNNING_STATE) neo.expectMasterState(third_master.getUUID(), protocol.RUNNING_STATE)
# Check that the primary master didn't change. # Check that the primary master didn't change.
self.assertEqual(neo.getPrimaryMaster(), first_master_uuid) self.assertEqual(neo.getPrimaryMaster(), first_master_uuid)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment