Commit c791efc3 authored by Sebastien Robin's avatar Sebastien Robin

improve testnode for more reliability and better logging

- display the output of all commands launched by erp5testnode in
  erp5testnode log
- add a thread to upload ongoing logs to the master regularly
- if the software release is not built successfully after a
  few time, totally erase software. This help unblocking if
  buildout is unable to update software.
- check if the last test result was cancelled in order to
  allow relaunching test without restarting testnode
- improve logs to allow much better understanding of what is
  going on
parent b6921405
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import re
import subprocess
import threading
import signal
import sys
class SubprocessError(EnvironmentError):
def __init__(self, status_dict):
self.status_dict = status_dict
def __getattr__(self, name):
return self.status_dict[name]
def __str__(self):
return 'Error %i' % self.status_code
_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
def format_command(*args, **kw):
cmdline = []
for k, v in sorted(kw.items()):
if _format_command_search(v):
v = _format_command_escape(v)
cmdline.append('%s=%s' % (k, v))
for v in args:
if _format_command_search(v):
v = _format_command_escape(v)
cmdline.append(v)
return ' '.join(cmdline)
def subprocess_capture(p, log, log_prefix, get_output=True):
def readerthread(input, output, buffer):
while True:
data = input.readline()
if not data:
break
if get_output:
buffer.append(data)
if log_prefix:
data = "%s : " % log_prefix + data
data = data.rstrip('\n')
output(data)
if p.stdout:
stdout = []
stdout_thread = threading.Thread(target=readerthread,
args=(p.stdout, log, stdout))
stdout_thread.daemon = True
stdout_thread.start()
if p.stderr:
stderr = []
stderr_thread = threading.Thread(target=readerthread,
args=(p.stderr, log, stderr))
stderr_thread.daemon = True
stderr_thread.start()
p.wait()
if p.stdout:
stdout_thread.join()
if p.stderr:
stderr_thread.join()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
class ProcessManager(object):
stdin = file(os.devnull)
def __init__(self, log, *args, **kw):
self.log = log
self.process_pid_set = set()
signal.signal(signal.SIGTERM, self.sigterm_handler)
def spawn(self, *args, **kw):
get_output = kw.pop('get_output', True)
log_prefix = kw.pop('log_prefix', '')
new_session = kw.pop('new_session', False)
subprocess_kw = {}
cwd = kw.pop('cwd', None)
if cwd:
subprocess_kw['cwd'] = cwd
if new_session:
subprocess_kw['preexec_fn'] = os.setsid
raise_error_if_fail = kw.pop('raise_error_if_fail', True)
env = kw and dict(os.environ, **kw) or None
command = format_command(*args, **kw)
self.log('subprocess_kw : %r' % (subprocess_kw,))
self.log('$ ' + command)
sys.stdout.flush()
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env, **subprocess_kw)
self.process_pid_set.add(p.pid)
stdout, stderr = subprocess_capture(p, self.log, log_prefix,
get_output=get_output)
result = dict(status_code=p.returncode, command=command,
stdout=stdout, stderr=stderr)
self.process_pid_set.remove(p.pid)
if raise_error_if_fail and p.returncode:
raise SubprocessError(result)
return result
def killPreviousRun(self):
self.log('ProcessManager killPreviousRun, going to kill %r' % (self.process_pid_set,))
for pgpid in self.process_pid_set:
try:
os.kill(pgpid, signal.SIGTERM)
except:
pass
try:
if os.path.exists(self.supervisord_pid_file):
supervisor_pid = int(open(self.supervisord_pid_file).read().strip())
self.log('ProcessManager killPreviousRun, going to kill supervisor with pid %r' % supervisor_pid)
os.kill(supervisor_pid, signal.SIGTERM)
except:
self.log('ProcessManager killPreviousRun, exception when killing supervisor')
pass
self.process_pid_set.clear()
def sigterm_handler(self, signal, frame):
self.log('SIGTERM_HANDLER')
sys.exit(1)
...@@ -36,10 +36,12 @@ MAX_SR_RETRIES = 3 ...@@ -36,10 +36,12 @@ MAX_SR_RETRIES = 3
class SlapOSControler(object): class SlapOSControler(object):
def __init__(self, config, log, process_group_pid_set=None, def __init__(self, config, log,
slapproxy_log=None): slapproxy_log=None, process_manager=None, reset_software=False):
log('SlapOSControler, initialize, reset_software: %r' % reset_software)
self.log = log self.log = log
self.config = config self.config = config
self.process_manager = process_manager
# By erasing everything, we make sure that we are able to "update" # By erasing everything, we make sure that we are able to "update"
# existing profiles. This is quite dirty way to do updates... # existing profiles. This is quite dirty way to do updates...
if os.path.exists(config['proxy_database']): if os.path.exists(config['proxy_database']):
...@@ -51,7 +53,7 @@ class SlapOSControler(object): ...@@ -51,7 +53,7 @@ class SlapOSControler(object):
kwargs['stderr'] = slapproxy_log_fp kwargs['stderr'] = slapproxy_log_fp
proxy = subprocess.Popen([config['slapproxy_binary'], proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], **kwargs) config['slapos_config']], **kwargs)
process_group_pid_set.add(proxy.pid) process_manager.process_pid_set.add(proxy.pid)
# XXX: dirty, giving some time for proxy to being able to accept # XXX: dirty, giving some time for proxy to being able to accept
# connections # connections
time.sleep(10) time.sleep(10)
...@@ -63,6 +65,14 @@ class SlapOSControler(object): ...@@ -63,6 +65,14 @@ class SlapOSControler(object):
self.software_profile, self.software_profile,
computer_guid=config['computer_id']) computer_guid=config['computer_id'])
computer = slap.registerComputer(config['computer_id']) computer = slap.registerComputer(config['computer_id'])
# Reset all previously generated software if needed
if reset_software:
software_root = config['software_root']
log('SlapOSControler : GOING TO RESET ALL SOFTWARE')
if os.path.exists(software_root):
shutil.rmtree(software_root)
os.mkdir(software_root)
os.chmod(software_root, 0750)
for i in range(0, MAX_PARTIONS): for i in range(0, MAX_PARTIONS):
# create partition and configure computer # create partition and configure computer
# XXX: at the moment all partitions do share same virtual interface address # XXX: at the moment all partitions do share same virtual interface address
...@@ -91,37 +101,23 @@ class SlapOSControler(object): ...@@ -91,37 +101,23 @@ class SlapOSControler(object):
'reference': config['computer_id'], 'reference': config['computer_id'],
'software_root': config['software_root']})) 'software_root': config['software_root']}))
def runSoftwareRelease(self, config, environment, process_group_pid_set=None, def spawn(self, *args, **kw):
stdout=None, stderr=None): return self.process_manager.spawn(*args, **kw)
def runSoftwareRelease(self, config, environment):
self.log("SlapOSControler.runSoftwareRelease") self.log("SlapOSControler.runSoftwareRelease")
cpu_count = os.sysconf("SC_NPROCESSORS_ONLN") cpu_count = os.sysconf("SC_NPROCESSORS_ONLN")
os.putenv('MAKEFLAGS', '-j%s' % cpu_count) os.putenv('MAKEFLAGS', '-j%s' % cpu_count)
os.environ['PATH'] = environment['PATH'] os.environ['PATH'] = environment['PATH']
command = [config['slapgrid_software_binary'], '-v', '-c',
#'--buildout-parameter',"'-U -N' -o",
config['slapos_config']]
# a SR may fail for number of reasons (incl. network failures) # a SR may fail for number of reasons (incl. network failures)
# so be tolerant and run it a few times before giving up # so be tolerant and run it a few times before giving up
for runs in range(0, MAX_SR_RETRIES): for runs in range(0, MAX_SR_RETRIES):
slapgrid = subprocess.Popen(command, status_dict = self.spawn(config['slapgrid_software_binary'], '-v', '-c',
stdout=stdout, stderr=stderr, config['slapos_config'], raise_error_if_fail=False,
close_fds=True, preexec_fn=os.setsid) log_prefix='slapgrid_sr', get_output=False)
process_group_pid_set.add(slapgrid.pid)
slapgrid.wait()
stdout.seek(0)
stderr.seek(0)
process_group_pid_set.remove(slapgrid.pid)
status_dict = {'status_code':slapgrid.returncode,
'command': repr(command),
'stdout':stdout.read(),
'stderr':stderr.read()}
stdout.close()
stderr.close()
return status_dict return status_dict
def runComputerPartition(self, config, environment, def runComputerPartition(self, config, environment,
process_group_pid_set=None,
stdout=None, stderr=None): stdout=None, stderr=None):
self.log("SlapOSControler.runComputerPartition") self.log("SlapOSControler.runComputerPartition")
slap = slapos.slap.slap() slap = slapos.slap.slap()
...@@ -130,26 +126,12 @@ class SlapOSControler(object): ...@@ -130,26 +126,12 @@ class SlapOSControler(object):
slap.registerOpenOrder().request(self.software_profile, slap.registerOpenOrder().request(self.software_profile,
partition_reference='testing partition', partition_reference='testing partition',
partition_parameter_kw=config['instance_dict']) partition_parameter_kw=config['instance_dict'])
command = [config['slapgrid_partition_binary'],
config['slapos_config'], '-c', '-v']
# try to run for all partitions as one partition may in theory request another one # try to run for all partitions as one partition may in theory request another one
# this not always is required but curently no way to know how "tree" of partitions # this not always is required but curently no way to know how "tree" of partitions
# may "expand" # may "expand"
for runs in range(0, MAX_PARTIONS): for runs in range(0, MAX_PARTIONS):
slapgrid = subprocess.Popen(command, status_dict = self.spawn(config['slapgrid_partition_binary'], '-v', '-c',
stdout=stdout, stderr=stderr, config['slapos_config'], raise_error_if_fail=False,
close_fds=True, preexec_fn=os.setsid) log_prefix='slapgrid_cp', get_output=False)
process_group_pid_set.add(slapgrid.pid)
slapgrid.wait()
process_group_pid_set.remove(slapgrid.pid)
stdout.seek(0)
stderr.seek(0)
status_dict = {'status_code':slapgrid.returncode,
'command': repr(command),
'stdout':stdout.read(),
'stderr':stderr.read()}
stdout.close()
stderr.close()
return status_dict return status_dict
...@@ -32,69 +32,29 @@ import sys ...@@ -32,69 +32,29 @@ import sys
import threading import threading
from testnode import SubprocessError from testnode import SubprocessError
from ProcessManager import ProcessManager
SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$') SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE) SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
_format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
_format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
def format_command(*args, **kw):
cmdline = []
for k, v in sorted(kw.items()):
if _format_command_search(v):
v = _format_command_escape(v)
cmdline.append('%s=%s' % (k, v))
for v in args:
if _format_command_search(v):
v = _format_command_escape(v)
cmdline.append(v)
return ' '.join(cmdline)
def subprocess_capture(p, quiet=False):
def readerthread(input, output, buffer):
while True:
data = input.readline()
if not data:
break
output(data)
buffer.append(data)
if p.stdout:
stdout = []
output = quiet and (lambda data: None) or sys.stdout.write
stdout_thread = threading.Thread(target=readerthread,
args=(p.stdout, output, stdout))
stdout_thread.setDaemon(True)
stdout_thread.start()
if p.stderr:
stderr = []
stderr_thread = threading.Thread(target=readerthread,
args=(p.stderr, sys.stderr.write, stderr))
stderr_thread.setDaemon(True)
stderr_thread.start()
if p.stdout:
stdout_thread.join()
if p.stderr:
stderr_thread.join()
p.wait()
return (p.stdout and ''.join(stdout),
p.stderr and ''.join(stderr))
GIT_TYPE = 'git' GIT_TYPE = 'git'
SVN_TYPE = 'svn' SVN_TYPE = 'svn'
class Updater(object): class Updater(ProcessManager):
_git_cache = {} _git_cache = {}
stdin = file(os.devnull) stdin = file(os.devnull)
def __init__(self, repository_path, log, revision=None, git_binary=None, def __init__(self, repository_path, log, revision=None, git_binary=None,
realtime_output=True): realtime_output=True, process_manager=None):
self.log = log self.log = log
self.revision = revision self.revision = revision
self._path_list = [] self._path_list = []
self.repository_path = repository_path self.repository_path = repository_path
self.git_binary = git_binary self.git_binary = git_binary
self.realtime_output = realtime_output self.realtime_output = realtime_output
self.process_manager = process_manager
def getRepositoryPath(self): def getRepositoryPath(self):
return self.repository_path return self.repository_path
...@@ -128,25 +88,10 @@ class Updater(object): ...@@ -128,25 +88,10 @@ class Updater(object):
raise raise
def spawn(self, *args, **kw): def spawn(self, *args, **kw):
quiet = kw.pop('quiet', False) return self.process_manager.spawn(*args,
env = kw and dict(os.environ, **kw) or None log_prefix='git',
command = format_command(*args, **kw) cwd=self.getRepositoryPath(),
self.log('$ ' + command) **kw)
sys.stdout.flush()
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env,
cwd=self.getRepositoryPath())
if self.realtime_output:
stdout, stderr = subprocess_capture(p, quiet)
else:
stdout, stderr = p.communicate()
self.log(stdout)
self.log(stderr)
result = dict(status_code=p.returncode, command=command,
stdout=stdout, stderr=stderr)
if p.returncode:
raise SubprocessError(result)
return result
def _git(self, *args, **kw): def _git(self, *args, **kw):
return self.spawn(self.git_binary, *args, **kw)['stdout'].strip() return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
......
...@@ -30,7 +30,7 @@ import logging ...@@ -30,7 +30,7 @@ import logging
import os import os
import pkg_resources import pkg_resources
import testnode from testnode import TestNode
CONFIG = { CONFIG = {
'computer_id': 'COMPUTER', 'computer_id': 'COMPUTER',
...@@ -48,15 +48,21 @@ def main(*args): ...@@ -48,15 +48,21 @@ def main(*args):
parsed_argument = parser.parse_args(list(args)) parsed_argument = parser.parse_args(list(args))
else: else:
parsed_argument = parser.parse_args() parsed_argument = parser.parse_args()
logger_format = '%(asctime)s %(name)-13s: %(levelname)-8s %(message)s'
formatter = logging.Formatter(logger_format)
logging.basicConfig(level=logging.INFO,
format=logger_format)
logger = logging.getLogger('erp5testnode') logger = logging.getLogger('erp5testnode')
if parsed_argument.console or parsed_argument.logfile: if parsed_argument.console or parsed_argument.logfile:
logger.setLevel(logging.INFO)
if parsed_argument.console: if parsed_argument.console:
logger.addHandler(logging.StreamHandler()) logger.addHandler(logging.StreamHandler())
logger.info('Activated console output.') logger.info('Activated console output.')
if parsed_argument.logfile: if parsed_argument.logfile:
logger.addHandler(logging.FileHandler(filename=parsed_argument.logfile)) file_handler = logging.FileHandler(filename=parsed_argument.logfile)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.info('Activated logfile %r output' % parsed_argument.logfile) logger.info('Activated logfile %r output' % parsed_argument.logfile)
CONFIG['log_file'] = parsed_argument.logfile
else: else:
logger.addHandler(logging.NullHandler()) logger.addHandler(logging.NullHandler())
CONFIG['logger'] = logger.info CONFIG['logger'] = logger.info
...@@ -118,4 +124,5 @@ def main(*args): ...@@ -118,4 +124,5 @@ def main(*args):
else: else:
instance_dict = {} instance_dict = {}
CONFIG['instance_dict'] = instance_dict CONFIG['instance_dict'] = instance_dict
testnode.run(CONFIG) testnode = TestNode(logger.info, CONFIG)
testnode.run()
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment