Commit 3bfbf89b authored by Julien Muchembled's avatar Julien Muchembled

tests: make (i)pdb multiprocess-safe

- do not kill processes being debugged
- pause timeout while debugging

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2687 71dcc9de-d417-0410-9af5-da40c76e7ee4
parent dfc1076f
......@@ -62,6 +62,15 @@ def debugHandler(sig, frame):
neo.__path__)
imp.load_module('neo.debug', file, filename, (suffix, mode, type))
def getPdb():
try: # try ipython if available
import IPython
IPython.Shell.IPShell(argv=[])
return IPython.Debugger.Tracer().debugger
except ImportError:
import pdb
return pdb.Pdb()
_debugger = None
@decorate
......@@ -71,13 +80,7 @@ def pdbHandler(sig, frame):
except ImportError:
global _debugger
if _debugger is None:
try: # try ipython if available
import IPython
IPython.Shell.IPShell(argv=[])
_debugger = IPython.Debugger.Tracer().debugger
except ImportError:
import pdb
_debugger = pdb.Pdb()
_debugger = getPdb()
return debugger.set_trace(frame)
# WKRD: rpdb2 take an integer (depth) instead of a frame as parameter,
# so we must hardcode the value, taking the decorator into account
......
......@@ -15,6 +15,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import __builtin__
import errno
import os
import random
......@@ -31,6 +32,7 @@ from neo.lib.protocol import Packets
from neo.lib.util import getAddressType
from time import time, gmtime, sleep
from struct import pack, unpack
from functools import wraps
DB_PREFIX = os.getenv('NEO_DB_PREFIX', 'test_neo_')
DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root')
......@@ -518,3 +520,51 @@ class SocketLock(object):
s = self._socket
del self._socket
s.close()
class ClusterPdb(object):
# TODO: monkey-patch normal code not to timeout
# if another node is being debugged
def __init__(self):
self._r, self._w = os.pipe()
self.release(0)
def __getattr__(self, attr):
try:
debugger = self.__dict__['_debugger']
except KeyError:
self._debugger = debugger = debug.getPdb()
def hook(name):
hook = getattr(self, name)
hooked = getattr(debugger, name)
def wrapper(*args, **kw):
return hook(hooked, *args, **kw)
setattr(debugger, name, wraps(hooked)(wrapper))
hook('interaction')
return getattr(debugger, attr)
def acquire(self):
return unpack('d', os.read(self._r, 8))[0]
def release(self, delay):
os.write(self._w, pack('d', delay))
def interaction(self, hooked, *args, **kw):
delay = self.acquire() - time()
try:
return hooked(*args, **kw)
finally:
self.release(delay + time())
def wait(self, test, timeout, period):
end_time = time() + timeout
while not test():
delay = self.acquire()
self.release(delay)
if time() > end_time + delay:
return False
sleep(period)
return True
__builtin__.pdb = ClusterPdb()
......@@ -153,10 +153,14 @@ class NEOProcess(object):
def kill(self, sig=signal.SIGTERM):
if self.pid:
delay = pdb.acquire()
try:
try:
os.kill(self.pid, sig)
except OSError:
traceback.print_last()
finally:
pdb.release(delay)
else:
raise AlreadyStopped
......@@ -340,16 +344,14 @@ class NEOCluster(object):
if process not in except_storages:
process.start()
# wait for the admin node availability
end_time = time.time() + MAX_START_TIME
while True:
if time.time() > end_time:
raise AssertionError, 'Timeout when starting cluster'
def test():
try:
self.neoctl.getClusterState()
except NotReadyException:
time.sleep(0.5)
else:
break
return False
return True
if not pdb.wait(test, MAX_START_TIME, 0.5):
raise AssertionError('Timeout when starting cluster')
self.port_allocator.reset()
def start(self, except_storages=()):
......@@ -358,18 +360,16 @@ class NEOCluster(object):
neoctl = self.neoctl
neoctl.startCluster()
target_count = len(self.db_list) - len(except_storages)
end_time = time.time() + MAX_START_TIME
while True:
storage_node_list = neoctl.getNodeList(
storage_node_list = []
def test():
storage_node_list[:] = neoctl.getNodeList(
node_type=NodeTypes.STORAGE)
# wait at least number of started storages, admin node can know
# more nodes when the cluster restart with an existing partition
# table referencing non-running nodes
if len(storage_node_list) >= target_count:
break
time.sleep(0.5)
if time.time() > end_time:
raise AssertionError, 'Timeout when starting cluster'
return len(storage_node_list) >= target_count
if not pdb.wait(test, MAX_START_TIME, 0.5):
raise AssertionError('Timeout when starting cluster')
if storage_node_list:
self.expectClusterRunning()
neoctl.enableStorageList([x[2] for x in storage_node_list])
......@@ -497,20 +497,18 @@ class NEOCluster(object):
def expectCondition(self, condition, timeout=0, delay=.5, on_fail=None):
end = time.time() + timeout + DELAY_SAFETY_MARGIN
opaque = None
opaque_history = []
while time.time() < end:
reached, opaque = condition(opaque)
if reached:
break
else:
opaque_history = [None]
def test():
reached, opaque = condition(opaque_history[-1])
if not reached:
opaque_history.append(opaque)
time.sleep(delay)
else:
return reached
if not pdb.wait(test, timeout + DELAY_SAFETY_MARGIN, delay):
del opaque_history[0]
if on_fail is not None:
on_fail(opaque_history)
raise AssertionError, 'Timeout while expecting condition. ' \
'History: %s' % (opaque_history, )
raise AssertionError('Timeout while expecting condition. '
'History: %s' % opaque_history)
def expectAllMasters(self, node_count, state=None, *args, **kw):
def callback(last_try):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment