Commit 3bfbf89b authored by Julien Muchembled's avatar Julien Muchembled

tests: make (i)pdb multiprocess-safe

- do not kill processes being debugged
- pause timeout while debugging

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2687 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent dfc1076f
...@@ -62,6 +62,15 @@ def debugHandler(sig, frame): ...@@ -62,6 +62,15 @@ def debugHandler(sig, frame):
neo.__path__) neo.__path__)
imp.load_module('neo.debug', file, filename, (suffix, mode, type)) imp.load_module('neo.debug', file, filename, (suffix, mode, type))
def getPdb():
try: # try ipython if available
import IPython
IPython.Shell.IPShell(argv=[])
return IPython.Debugger.Tracer().debugger
except ImportError:
import pdb
return pdb.Pdb()
_debugger = None _debugger = None
@decorate @decorate
...@@ -71,13 +80,7 @@ def pdbHandler(sig, frame): ...@@ -71,13 +80,7 @@ def pdbHandler(sig, frame):
except ImportError: except ImportError:
global _debugger global _debugger
if _debugger is None: if _debugger is None:
try: # try ipython if available _debugger = getPdb()
import IPython
IPython.Shell.IPShell(argv=[])
_debugger = IPython.Debugger.Tracer().debugger
except ImportError:
import pdb
_debugger = pdb.Pdb()
return debugger.set_trace(frame) return debugger.set_trace(frame)
# WKRD: rpdb2 take an integer (depth) instead of a frame as parameter, # WKRD: rpdb2 take an integer (depth) instead of a frame as parameter,
# so we must hardcode the value, taking the decorator into account # so we must hardcode the value, taking the decorator into account
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import __builtin__
import errno import errno
import os import os
import random import random
...@@ -31,6 +32,7 @@ from neo.lib.protocol import Packets ...@@ -31,6 +32,7 @@ from neo.lib.protocol import Packets
from neo.lib.util import getAddressType from neo.lib.util import getAddressType
from time import time, gmtime, sleep from time import time, gmtime, sleep
from struct import pack, unpack from struct import pack, unpack
from functools import wraps
DB_PREFIX = os.getenv('NEO_DB_PREFIX', 'test_neo_') DB_PREFIX = os.getenv('NEO_DB_PREFIX', 'test_neo_')
DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root') DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root')
...@@ -518,3 +520,51 @@ class SocketLock(object): ...@@ -518,3 +520,51 @@ class SocketLock(object):
s = self._socket s = self._socket
del self._socket del self._socket
s.close() s.close()
class ClusterPdb(object):
# TODO: monkey-patch normal code not to timeout
# if another node is being debugged
def __init__(self):
self._r, self._w = os.pipe()
self.release(0)
def __getattr__(self, attr):
try:
debugger = self.__dict__['_debugger']
except KeyError:
self._debugger = debugger = debug.getPdb()
def hook(name):
hook = getattr(self, name)
hooked = getattr(debugger, name)
def wrapper(*args, **kw):
return hook(hooked, *args, **kw)
setattr(debugger, name, wraps(hooked)(wrapper))
hook('interaction')
return getattr(debugger, attr)
def acquire(self):
return unpack('d', os.read(self._r, 8))[0]
def release(self, delay):
os.write(self._w, pack('d', delay))
def interaction(self, hooked, *args, **kw):
delay = self.acquire() - time()
try:
return hooked(*args, **kw)
finally:
self.release(delay + time())
def wait(self, test, timeout, period):
end_time = time() + timeout
while not test():
delay = self.acquire()
self.release(delay)
if time() > end_time + delay:
return False
sleep(period)
return True
__builtin__.pdb = ClusterPdb()
...@@ -153,10 +153,14 @@ class NEOProcess(object): ...@@ -153,10 +153,14 @@ class NEOProcess(object):
def kill(self, sig=signal.SIGTERM): def kill(self, sig=signal.SIGTERM):
if self.pid: if self.pid:
delay = pdb.acquire()
try:
try: try:
os.kill(self.pid, sig) os.kill(self.pid, sig)
except OSError: except OSError:
traceback.print_last() traceback.print_last()
finally:
pdb.release(delay)
else: else:
raise AlreadyStopped raise AlreadyStopped
...@@ -340,16 +344,14 @@ class NEOCluster(object): ...@@ -340,16 +344,14 @@ class NEOCluster(object):
if process not in except_storages: if process not in except_storages:
process.start() process.start()
# wait for the admin node availability # wait for the admin node availability
end_time = time.time() + MAX_START_TIME def test():
while True:
if time.time() > end_time:
raise AssertionError, 'Timeout when starting cluster'
try: try:
self.neoctl.getClusterState() self.neoctl.getClusterState()
except NotReadyException: except NotReadyException:
time.sleep(0.5) return False
else: return True
break if not pdb.wait(test, MAX_START_TIME, 0.5):
raise AssertionError('Timeout when starting cluster')
self.port_allocator.reset() self.port_allocator.reset()
def start(self, except_storages=()): def start(self, except_storages=()):
...@@ -358,18 +360,16 @@ class NEOCluster(object): ...@@ -358,18 +360,16 @@ class NEOCluster(object):
neoctl = self.neoctl neoctl = self.neoctl
neoctl.startCluster() neoctl.startCluster()
target_count = len(self.db_list) - len(except_storages) target_count = len(self.db_list) - len(except_storages)
end_time = time.time() + MAX_START_TIME storage_node_list = []
while True: def test():
storage_node_list = neoctl.getNodeList( storage_node_list[:] = neoctl.getNodeList(
node_type=NodeTypes.STORAGE) node_type=NodeTypes.STORAGE)
# wait at least number of started storages, admin node can know # wait at least number of started storages, admin node can know
# more nodes when the cluster restart with an existing partition # more nodes when the cluster restart with an existing partition
# table referencing non-running nodes # table referencing non-running nodes
if len(storage_node_list) >= target_count: return len(storage_node_list) >= target_count
break if not pdb.wait(test, MAX_START_TIME, 0.5):
time.sleep(0.5) raise AssertionError('Timeout when starting cluster')
if time.time() > end_time:
raise AssertionError, 'Timeout when starting cluster'
if storage_node_list: if storage_node_list:
self.expectClusterRunning() self.expectClusterRunning()
neoctl.enableStorageList([x[2] for x in storage_node_list]) neoctl.enableStorageList([x[2] for x in storage_node_list])
...@@ -497,20 +497,18 @@ class NEOCluster(object): ...@@ -497,20 +497,18 @@ class NEOCluster(object):
def expectCondition(self, condition, timeout=0, delay=.5, on_fail=None): def expectCondition(self, condition, timeout=0, delay=.5, on_fail=None):
end = time.time() + timeout + DELAY_SAFETY_MARGIN end = time.time() + timeout + DELAY_SAFETY_MARGIN
opaque = None opaque_history = [None]
opaque_history = [] def test():
while time.time() < end: reached, opaque = condition(opaque_history[-1])
reached, opaque = condition(opaque) if not reached:
if reached:
break
else:
opaque_history.append(opaque) opaque_history.append(opaque)
time.sleep(delay) return reached
else: if not pdb.wait(test, timeout + DELAY_SAFETY_MARGIN, delay):
del opaque_history[0]
if on_fail is not None: if on_fail is not None:
on_fail(opaque_history) on_fail(opaque_history)
raise AssertionError, 'Timeout while expecting condition. ' \ raise AssertionError('Timeout while expecting condition. '
'History: %s' % (opaque_history, ) 'History: %s' % opaque_history)
def expectAllMasters(self, node_count, state=None, *args, **kw): def expectAllMasters(self, node_count, state=None, *args, **kw):
def callback(last_try): def callback(last_try):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment