improve testnode for more reliability and better logging

- display the output of all commands launched by erp5testnode in
  erp5testnode log
- add a thread to upload ongoing logs to the master regularly
- if the software release is not built successfully after a
  few time, totally erase software. This help unblocking if
  buildout is unable to update software.
- check if the last test result was cancelled in order to
  allow relaunching test without restarting testnode
- improve logs to allow much better understanding of what is
  going on

erp5/util/testnode/ProcessManager.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+import re
+import subprocess
+import threading
+import signal
+import sys
+
+class SubprocessError(EnvironmentError):
+  def __init__(self, status_dict):
+    self.status_dict = status_dict
+  def __getattr__(self, name):
+    return self.status_dict[name]
+  def __str__(self):
+    return 'Error %i' % self.status_code
+
+_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
+_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
+
+def format_command(*args, **kw):
+  cmdline = []
+  for k, v in sorted(kw.items()):
+    if _format_command_search(v):
+      v = _format_command_escape(v)
+    cmdline.append('%s=%s' % (k, v))
+  for v in args:
+    if _format_command_search(v):
+      v = _format_command_escape(v)
+    cmdline.append(v)
+  return ' '.join(cmdline)
+
+def subprocess_capture(p, log, log_prefix, get_output=True):
+  def readerthread(input, output, buffer):
+    while True:
+      data = input.readline()
+      if not data:
+        break
+      if get_output:
+        buffer.append(data)
+      if log_prefix:
+        data = "%s : " % log_prefix +  data
+      data = data.rstrip('\n')
+      output(data)
+  if p.stdout:
+    stdout = []
+    stdout_thread = threading.Thread(target=readerthread,
+                                     args=(p.stdout, log, stdout))
+    stdout_thread.daemon = True
+    stdout_thread.start()
+  if p.stderr:
+    stderr = []
+    stderr_thread = threading.Thread(target=readerthread,
+                                     args=(p.stderr, log, stderr))
+    stderr_thread.daemon = True
+    stderr_thread.start()
+  p.wait()
+  if p.stdout:
+    stdout_thread.join()
+  if p.stderr:
+    stderr_thread.join()
+  return (p.stdout and ''.join(stdout),
+          p.stderr and ''.join(stderr))
+
+class ProcessManager(object):
+
+  stdin = file(os.devnull)
+
+  def __init__(self, log, *args, **kw):
+    self.log = log
+    self.process_pid_set = set()
+    signal.signal(signal.SIGTERM, self.sigterm_handler)
+
+  def spawn(self, *args, **kw):
+    get_output = kw.pop('get_output', True)
+    log_prefix = kw.pop('log_prefix', '')
+    new_session = kw.pop('new_session', False)
+    subprocess_kw = {}
+    cwd = kw.pop('cwd', None)
+    if cwd:
+      subprocess_kw['cwd'] = cwd
+    if new_session:
+      subprocess_kw['preexec_fn'] = os.setsid
+    raise_error_if_fail = kw.pop('raise_error_if_fail', True)
+    env = kw and dict(os.environ, **kw) or None
+    command = format_command(*args, **kw)
+    self.log('subprocess_kw : %r' % (subprocess_kw,))
+    self.log('$ ' + command)
+    sys.stdout.flush()
+    p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, env=env, **subprocess_kw)
+    self.process_pid_set.add(p.pid)
+    stdout, stderr = subprocess_capture(p, self.log, log_prefix,
+                                        get_output=get_output)
+    result = dict(status_code=p.returncode, command=command,
+                  stdout=stdout, stderr=stderr)
+    self.process_pid_set.remove(p.pid)
+    if raise_error_if_fail and p.returncode:
+      raise SubprocessError(result)
+    return result
+
+  def killPreviousRun(self):
+    self.log('ProcessManager killPreviousRun, going to kill %r' % (self.process_pid_set,))
+    for pgpid in self.process_pid_set:
+      try:
+        os.kill(pgpid, signal.SIGTERM)
+      except:
+        pass
+    try:
+      if os.path.exists(self.supervisord_pid_file):
+        supervisor_pid = int(open(self.supervisord_pid_file).read().strip())
+        self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r' % supervisor_pid)
+        os.kill(supervisor_pid, signal.SIGTERM)
+    except:
+      self.log('ProcessManager killPreviousRun, exception when killing supervisor')
+      pass
+    self.process_pid_set.clear()
+
+  def sigterm_handler(self, signal, frame):
+    self.log('SIGTERM_HANDLER')
+    sys.exit(1)

erp5/util/testnode/SlapOSControler.py

@@ -36,10 +36,12 @@ MAX_SR_RETRIES = 3
 
 class SlapOSControler(object):
 
-  def __init__(self, config, log, process_group_pid_set=None,
-      slapproxy_log=None):
+  def __init__(self, config, log,
+      slapproxy_log=None, process_manager=None, reset_software=False):
+    log('SlapOSControler, initialize, reset_software: %r' % reset_software)
     self.log = log
     self.config = config
+    self.process_manager = process_manager
     # By erasing everything, we make sure that we are able to "update"
     # existing profiles. This is quite dirty way to do updates...
     if os.path.exists(config['proxy_database']):
@@ -51,7 +53,7 @@ class SlapOSControler(object):
       kwargs['stderr'] = slapproxy_log_fp
     proxy = subprocess.Popen([config['slapproxy_binary'],
       config['slapos_config']], **kwargs)
-    process_group_pid_set.add(proxy.pid)
+    process_manager.process_pid_set.add(proxy.pid)
     # XXX: dirty, giving some time for proxy to being able to accept
     # connections
     time.sleep(10)
@@ -63,6 +65,14 @@ class SlapOSControler(object):
         self.software_profile,
         computer_guid=config['computer_id'])
     computer = slap.registerComputer(config['computer_id'])
+    # Reset all previously generated software if needed
+    if reset_software:
+      software_root = config['software_root']
+      log('SlapOSControler : GOING TO RESET ALL SOFTWARE')
+      if os.path.exists(software_root):
+        shutil.rmtree(software_root)
+      os.mkdir(software_root)
+      os.chmod(software_root, 0750)
     for i in range(0, MAX_PARTIONS):
       # create partition and configure computer
       # XXX: at the moment all partitions do share same virtual interface address
@@ -91,37 +101,23 @@ class SlapOSControler(object):
                                     'reference': config['computer_id'],
                                     'software_root': config['software_root']}))
 
-  def runSoftwareRelease(self, config, environment, process_group_pid_set=None,
-                         stdout=None, stderr=None):
+  def spawn(self, *args, **kw):
+    return self.process_manager.spawn(*args, **kw)
+
+  def runSoftwareRelease(self, config, environment):
     self.log("SlapOSControler.runSoftwareRelease")
     cpu_count = os.sysconf("SC_NPROCESSORS_ONLN")
     os.putenv('MAKEFLAGS', '-j%s' % cpu_count)
     os.environ['PATH'] = environment['PATH']
-    command = [config['slapgrid_software_binary'], '-v', '-c',
-      #'--buildout-parameter',"'-U -N' -o",
-      config['slapos_config']]
     # a SR may fail for number of reasons (incl. network failures)
     # so be tolerant and run it a few times before giving up
     for runs in range(0, MAX_SR_RETRIES):
-      slapgrid = subprocess.Popen(command,
-        stdout=stdout, stderr=stderr,
-        close_fds=True, preexec_fn=os.setsid)
-      process_group_pid_set.add(slapgrid.pid)
-      slapgrid.wait()
-
-    stdout.seek(0)
-    stderr.seek(0)
-    process_group_pid_set.remove(slapgrid.pid)
-    status_dict = {'status_code':slapgrid.returncode,
-                    'command': repr(command),
-                    'stdout':stdout.read(),
-                    'stderr':stderr.read()}
-    stdout.close()
-    stderr.close()
+      status_dict = self.spawn(config['slapgrid_software_binary'], '-v', '-c',
+                 config['slapos_config'], raise_error_if_fail=False,
+                 log_prefix='slapgrid_sr', get_output=False)
     return status_dict
 
   def runComputerPartition(self, config, environment,
-                           process_group_pid_set=None,
                            stdout=None, stderr=None):
     self.log("SlapOSControler.runComputerPartition")
     slap = slapos.slap.slap()
@@ -130,26 +126,12 @@ class SlapOSControler(object):
     slap.registerOpenOrder().request(self.software_profile,
         partition_reference='testing partition',
         partition_parameter_kw=config['instance_dict'])
-    command = [config['slapgrid_partition_binary'],
-      config['slapos_config'], '-c', '-v']
 
     # try to run for all partitions as one partition may in theory request another one 
     # this not always is required but curently no way to know how "tree" of partitions
     # may "expand"
     for runs in range(0, MAX_PARTIONS):
-      slapgrid = subprocess.Popen(command,
-        stdout=stdout, stderr=stderr,
-        close_fds=True, preexec_fn=os.setsid)
-      process_group_pid_set.add(slapgrid.pid)
-      slapgrid.wait()
-      process_group_pid_set.remove(slapgrid.pid)
-
-    stdout.seek(0)
-    stderr.seek(0)
-    status_dict = {'status_code':slapgrid.returncode,
-                    'command': repr(command),
-                    'stdout':stdout.read(),
-                    'stderr':stderr.read()}
-    stdout.close()
-    stderr.close()
+      status_dict = self.spawn(config['slapgrid_partition_binary'], '-v', '-c',
+                 config['slapos_config'], raise_error_if_fail=False,
+                 log_prefix='slapgrid_cp', get_output=False)
     return status_dict

erp5/util/testnode/Updater.py

@@ -32,69 +32,29 @@ import sys
 import threading
 
 from testnode import SubprocessError
+from ProcessManager import ProcessManager
 
 SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
 SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
 
-_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
-_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
-def format_command(*args, **kw):
-  cmdline = []
-  for k, v in sorted(kw.items()):
-    if _format_command_search(v):
-      v = _format_command_escape(v)
-    cmdline.append('%s=%s' % (k, v))
-  for v in args:
-    if _format_command_search(v):
-      v = _format_command_escape(v)
-    cmdline.append(v)
-  return ' '.join(cmdline)
-
-def subprocess_capture(p, quiet=False):
-  def readerthread(input, output, buffer):
-    while True:
-      data = input.readline()
-      if not data:
-        break
-      output(data)
-      buffer.append(data)
-  if p.stdout:
-    stdout = []
-    output = quiet and (lambda data: None) or sys.stdout.write
-    stdout_thread = threading.Thread(target=readerthread,
-                                     args=(p.stdout, output, stdout))
-    stdout_thread.setDaemon(True)
-    stdout_thread.start()
-  if p.stderr:
-    stderr = []
-    stderr_thread = threading.Thread(target=readerthread,
-                                     args=(p.stderr, sys.stderr.write, stderr))
-    stderr_thread.setDaemon(True)
-    stderr_thread.start()
-  if p.stdout:
-    stdout_thread.join()
-  if p.stderr:
-    stderr_thread.join()
-  p.wait()
-  return (p.stdout and ''.join(stdout),
-          p.stderr and ''.join(stderr))
 
 GIT_TYPE = 'git'
 SVN_TYPE = 'svn'
 
-class Updater(object):
+class Updater(ProcessManager):
 
   _git_cache = {}
   stdin = file(os.devnull)
 
   def __init__(self, repository_path, log, revision=None, git_binary=None,
-      realtime_output=True):
+      realtime_output=True, process_manager=None):
     self.log = log
     self.revision = revision
     self._path_list = []
     self.repository_path = repository_path
     self.git_binary = git_binary
     self.realtime_output = realtime_output
+    self.process_manager = process_manager
 
   def getRepositoryPath(self):
     return self.repository_path
@@ -128,25 +88,10 @@ class Updater(object):
               raise
 
   def spawn(self, *args, **kw):
-    quiet = kw.pop('quiet', False)
-    env = kw and dict(os.environ, **kw) or None
-    command = format_command(*args, **kw)
-    self.log('$ ' + command)
-    sys.stdout.flush()
-    p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, env=env,
-                         cwd=self.getRepositoryPath())
-    if self.realtime_output:
-      stdout, stderr = subprocess_capture(p, quiet)
-    else:
-      stdout, stderr = p.communicate()
-      self.log(stdout)
-      self.log(stderr)
-    result = dict(status_code=p.returncode, command=command,
-                  stdout=stdout, stderr=stderr)
-    if p.returncode:
-      raise SubprocessError(result)
-    return result
+    return self.process_manager.spawn(*args, 
+                                      log_prefix='git',
+                                      cwd=self.getRepositoryPath(),
+                                      **kw)
 
   def _git(self, *args, **kw):
     return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()

erp5/util/testnode/__init__.py

@@ -30,7 +30,7 @@ import logging
 import os
 import pkg_resources
 
-import testnode
+from testnode import TestNode
 
 CONFIG = {
   'computer_id': 'COMPUTER',
@@ -48,15 +48,21 @@ def main(*args):
     parsed_argument = parser.parse_args(list(args))
   else:
     parsed_argument = parser.parse_args()
+  logger_format = '%(asctime)s %(name)-13s: %(levelname)-8s %(message)s'
+  formatter = logging.Formatter(logger_format)
+  logging.basicConfig(level=logging.INFO,
+                     format=logger_format)
   logger = logging.getLogger('erp5testnode')
   if parsed_argument.console or parsed_argument.logfile:
-    logger.setLevel(logging.INFO)
     if parsed_argument.console:
       logger.addHandler(logging.StreamHandler())
       logger.info('Activated console output.')
     if parsed_argument.logfile:
-      logger.addHandler(logging.FileHandler(filename=parsed_argument.logfile))
+      file_handler = logging.FileHandler(filename=parsed_argument.logfile)
+      file_handler.setFormatter(formatter)
+      logger.addHandler(file_handler)
       logger.info('Activated logfile %r output' % parsed_argument.logfile)
+      CONFIG['log_file'] = parsed_argument.logfile
   else:
     logger.addHandler(logging.NullHandler())
   CONFIG['logger'] = logger.info
@@ -118,4 +124,5 @@ def main(*args):
   else:
     instance_dict = {}
   CONFIG['instance_dict'] = instance_dict
-  testnode.run(CONFIG)
+  testnode = TestNode(logger.info, CONFIG)
+  testnode.run()