erp5testnode: make sure to kill grandchild when we kill a process

erp5/tests/testERP5TestNode.py

@@ -466,7 +466,9 @@ branch = foo
       self.assertEqual(result['status_code'], expected_status)
     process_manager = ProcessManager(log=self.log, max_timeout=1)
     _checkCorrectStatus(0, *['sleep','0'])
-    _checkCorrectStatus(-15, *['sleep','2'])
+    # We must make sure that if the command is too long that
+    # it will be automatically killed
+    self.assertRaises(SubprocessError, process_manager.spawn, 'sleep','3')
 
   def test_13_SlaposControlerResetSoftware(self):
     test_node = self.getTestNode()

erp5/util/testnode/ProcessManager.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 import os
+import psutil
 import re
 import subprocess
 import threading
@@ -98,6 +99,25 @@ def subprocess_capture(p, log, log_prefix, get_output=True):
   return (p.stdout and ''.join(stdout),
           p.stderr and ''.join(stderr))
 
+def killCommand(pid):
+  """
+  To avoid letting orphaned childs, we stop the process and all it's
+  child (until childs does not change) and then we brutally kill
+  everyone at the same time
+  """
+  process = psutil.Process(pid)
+  child_set = set([x.pid for x in process.get_children(recursive=True)])
+  new_child_set = None
+  os.kill(pid, signal.SIGSTOP)
+  while new_child_set != child_set:
+    for child_pid in child_set:
+      os.kill(child_pid, signal.SIGSTOP)
+    time.sleep(1)
+    new_child_set = set([x.pid for x in process.get_children(recursive=True)])
+  for child_pid in child_set:
+    os.kill(child_pid, signal.SIGKILL)
+  os.kill(pid, signal.SIGKILL)
+
 class ProcessManager(object):
 
   stdin = file(os.devnull)
@@ -115,7 +135,7 @@ class ProcessManager(object):
     def timeoutExpired(p, log):
       if p.poll() is None:
         log('PROCESS TOO LONG OR DEAD, GOING TO BE TERMINATED')
-        p.terminate()
+        killCommand(p.pid)
 
     if self.under_cancellation:
       raise CancellationError("Test Result was cancelled")
@@ -147,7 +167,7 @@ class ProcessManager(object):
     self.process_pid_set.discard(p.pid)
     if self.under_cancellation:
       raise CancellationError("Test Result was cancelled")
-    if raise_error_if_fail and p.returncode != -15 and p.returncode:
+    if raise_error_if_fail and p.returncode:
       raise SubprocessError(result)
     return result
 
@@ -163,7 +183,7 @@ class ProcessManager(object):
       self.under_cancellation = True
     for pgpid in self.process_pid_set:
       try:
-        os.kill(pgpid, signal.SIGTERM)
+        killCommand(pgpid)
       except:
         pass
     try:

setup.py

@@ -49,9 +49,10 @@ setup(name=name,
       namespace_packages=['erp5', 'erp5.util'],
       install_requires=[
         'setuptools', # namespaces
+        'psutil >= 0.5.0',
       ],
       extras_require={
-        'testnode': ['slapos.core', 'xml_marshaller'],
+        'testnode': ['slapos.core', 'xml_marshaller', 'psutil >= 0.5.0'],
         'testbrowser': ['zope.testbrowser >= 3.11.1', 'z3c.etestbrowser'],
         'benchmark': benchmark_install_require_list,
         'benchmark-report': [name+'[benchmark]', 'matplotlib', 'numpy'],
@@ -76,6 +77,7 @@ setup(name=name,
       tests_require=[
         'slapos.core',
         'xml_marshaller',
+        'psutil >= 0.5.0',
       ],
     )