tests: make (i)pdb multiprocess-safe

- do not kill processes being debugged
- pause timeout while debugging

git-svn-id: https://svn.erp5.org/repos/neo/trunk@2687 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo/lib/debug.py

@@ -62,6 +62,15 @@ def debugHandler(sig, frame):
         neo.__path__)
     imp.load_module('neo.debug', file, filename, (suffix, mode, type))
 
+def getPdb():
+    try: # try ipython if available
+        import IPython
+        IPython.Shell.IPShell(argv=[])
+        return IPython.Debugger.Tracer().debugger
+    except ImportError:
+        import pdb
+        return pdb.Pdb()
+
 _debugger = None
 
 @decorate
@@ -71,13 +80,7 @@ def pdbHandler(sig, frame):
     except ImportError:
         global _debugger
         if _debugger is None:
-            try: # try ipython if available
-                import IPython
-                IPython.Shell.IPShell(argv=[])
-                _debugger = IPython.Debugger.Tracer().debugger
-            except ImportError:
-                import pdb
-                _debugger = pdb.Pdb()
+            _debugger = getPdb()
         return debugger.set_trace(frame)
     # WKRD: rpdb2 take an integer (depth) instead of a frame as parameter,
     #       so we must hardcode the value, taking the decorator into account

neo/tests/__init__.py

@@ -15,6 +15,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
+import __builtin__
 import errno
 import os
 import random
@@ -31,6 +32,7 @@ from neo.lib.protocol import Packets
 from neo.lib.util import getAddressType
 from time import time, gmtime, sleep
 from struct import pack, unpack
+from functools import wraps
 
 DB_PREFIX = os.getenv('NEO_DB_PREFIX', 'test_neo_')
 DB_ADMIN = os.getenv('NEO_DB_ADMIN', 'root')
@@ -518,3 +520,51 @@ class SocketLock(object):
         s = self._socket
         del self._socket
         s.close()
+
+
+class ClusterPdb(object):
+    # TODO: monkey-patch normal code not to timeout
+    #       if another node is being debugged
+
+    def __init__(self):
+        self._r, self._w = os.pipe()
+        self.release(0)
+
+    def __getattr__(self, attr):
+        try:
+            debugger = self.__dict__['_debugger']
+        except KeyError:
+            self._debugger = debugger = debug.getPdb()
+            def hook(name):
+                hook = getattr(self, name)
+                hooked = getattr(debugger, name)
+                def wrapper(*args, **kw):
+                    return hook(hooked, *args, **kw)
+                setattr(debugger, name, wraps(hooked)(wrapper))
+            hook('interaction')
+        return getattr(debugger, attr)
+
+    def acquire(self):
+        return unpack('d', os.read(self._r, 8))[0]
+
+    def release(self, delay):
+        os.write(self._w, pack('d', delay))
+
+    def interaction(self, hooked, *args, **kw):
+        delay = self.acquire() - time()
+        try:
+            return hooked(*args, **kw)
+        finally:
+            self.release(delay + time())
+
+    def wait(self, test, timeout, period):
+        end_time = time() + timeout
+        while not test():
+            delay = self.acquire()
+            self.release(delay)
+            if time() > end_time + delay:
+                return False
+            sleep(period)
+        return True
+
+__builtin__.pdb = ClusterPdb()

neo/tests/functional/__init__.py

@@ -153,10 +153,14 @@ class NEOProcess(object):
 
     def kill(self, sig=signal.SIGTERM):
         if self.pid:
+            delay = pdb.acquire()
+            try:
                 try:
                     os.kill(self.pid, sig)
                 except OSError:
                     traceback.print_last()
+            finally:
+                pdb.release(delay)
         else:
             raise AlreadyStopped
 
@@ -340,16 +344,14 @@ class NEOCluster(object):
                 if process not in except_storages:
                     process.start()
         # wait for the admin node availability
-        end_time = time.time() + MAX_START_TIME
-        while True:
-            if time.time() > end_time:
-                raise AssertionError, 'Timeout when starting cluster'
+        def test():
             try:
                 self.neoctl.getClusterState()
             except NotReadyException:
-                time.sleep(0.5)
-            else:
-                break
+                return False
+            return True
+        if not pdb.wait(test, MAX_START_TIME, 0.5):
+            raise AssertionError('Timeout when starting cluster')
         self.port_allocator.reset()
 
     def start(self, except_storages=()):
@@ -358,18 +360,16 @@ class NEOCluster(object):
         neoctl = self.neoctl
         neoctl.startCluster()
         target_count = len(self.db_list) - len(except_storages)
-        end_time = time.time() + MAX_START_TIME
-        while True:
-            storage_node_list = neoctl.getNodeList(
+        storage_node_list = []
+        def test():
+            storage_node_list[:] = neoctl.getNodeList(
                 node_type=NodeTypes.STORAGE)
             # wait at least number of started storages, admin node can know
             # more nodes when the cluster restart with an existing partition
             # table referencing non-running nodes
-            if len(storage_node_list) >= target_count:
-                break
-            time.sleep(0.5)
-            if time.time() > end_time:
-                raise AssertionError, 'Timeout when starting cluster'
+            return len(storage_node_list) >= target_count
+        if not pdb.wait(test, MAX_START_TIME, 0.5):
+            raise AssertionError('Timeout when starting cluster')
         if storage_node_list:
             self.expectClusterRunning()
             neoctl.enableStorageList([x[2] for x in storage_node_list])
@@ -497,20 +497,18 @@ class NEOCluster(object):
 
     def expectCondition(self, condition, timeout=0, delay=.5, on_fail=None):
         end = time.time() + timeout + DELAY_SAFETY_MARGIN
-        opaque = None
-        opaque_history = []
-        while time.time() < end:
-            reached, opaque = condition(opaque)
-            if reached:
-                break
-            else:
+        opaque_history = [None]
+        def test():
+            reached, opaque = condition(opaque_history[-1])
+            if not reached:
                 opaque_history.append(opaque)
-                time.sleep(delay)
-        else:
+            return reached
+        if not pdb.wait(test, timeout + DELAY_SAFETY_MARGIN, delay):
+            del opaque_history[0]
             if on_fail is not None:
                 on_fail(opaque_history)
-            raise AssertionError, 'Timeout while expecting condition. ' \
-                                'History: %s' % (opaque_history, )
+            raise AssertionError('Timeout while expecting condition. '
+                                 'History: %s' % opaque_history)
 
     def expectAllMasters(self, node_count, state=None, *args, **kw):
         def callback(last_try):