Commit 3d7e4345 authored by Tatuya Kamada's avatar Tatuya Kamada Committed by Sebastien Robin

erp5testnode: [experimental] Add code for review.

parent 4feb8f4d
...@@ -496,3 +496,57 @@ branch = foo ...@@ -496,3 +496,57 @@ branch = foo
self.assertEquals(True, os.path.exists(to_drop_path)) self.assertEquals(True, os.path.exists(to_drop_path))
createFolder(folder, clean=True) createFolder(folder, clean=True)
self.assertEquals(False, os.path.exists(to_drop_path)) self.assertEquals(False, os.path.exists(to_drop_path))
def test_15_log_directory(self):
def doNothing(self, *args, **kw):
pass
test_self = self
test_result_path_root = os.path.join(test_self._temp_dir,'test/results')
os.makedirs(test_result_path_root)
global counter
counter = 0
def patch_startTestSuite(self,test_node_title):
global counter
config_list = []
def _checkExistingTestSuite(reference_set):
test_self.assertEquals(set(reference_set),
set(os.listdir(test_node.config["working_directory"])))
for x in reference_set:
test_self.assertTrue(os.path.exists(os.path.join(
test_node.config["working_directory"],x)),True)
if counter == 0:
config_list.append(test_self.getTestSuiteData(reference='foo')[0])
elif counter == 1:
_checkExistingTestSuite(set(['foo']))
raise StopIteration
counter += 1
return json.dumps(config_list)
def patch_createTestResult(self, revision, test_name_list, node_title,
allow_restart=False, test_title=None, project_title=None):
test_result_path = os.path.join(test_result_path_root, test_title)
result = TestResultProxy(self._proxy, self._retry_time,
self._logger, test_result_path, node_title, revision)
return result
original_sleep = time.sleep
time.sleep = doNothing
self.generateTestRepositoryList()
original_startTestSuite = TaskDistributor.startTestSuite
TaskDistributor.startTestSuite = patch_startTestSuite
original_createTestResult = TaskDistributionTool.createTestResult
TaskDistributionTool.createTestResult = patch_createTestResult
test_node = self.getTestNode()
original_prepareSlapOS = test_node._prepareSlapOS
test_node._prepareSlapOS = doNothing
original_runTestSuite = test_node.runTestSuite
test_node.runTestSuite = doNothing
SlapOSControler.initializeSlapOSControler = doNothing
try:
test_node.run()
except Exception as e:
self.assertEqual(type(e),StopIteration)
finally:
time.sleep = original_sleep
TaskDistributor.startTestSuite = original_startTestSuite
TaskDistributionTool.createTestResult = original_createTestResult
test_node._prepareSlapOS = original_prepareSlapOS
test_node.runTestSuite = original_runTestSuite
...@@ -142,6 +142,9 @@ class ProcessManager(object): ...@@ -142,6 +142,9 @@ class ProcessManager(object):
get_output = kw.pop('get_output', True) get_output = kw.pop('get_output', True)
log_prefix = kw.pop('log_prefix', '') log_prefix = kw.pop('log_prefix', '')
new_session = kw.pop('new_session', True) new_session = kw.pop('new_session', True)
log = kw.pop('log', None)
if log is None:
log = self.log
subprocess_kw = {} subprocess_kw = {}
cwd = kw.pop('cwd', None) cwd = kw.pop('cwd', None)
if cwd: if cwd:
...@@ -151,15 +154,15 @@ class ProcessManager(object): ...@@ -151,15 +154,15 @@ class ProcessManager(object):
raise_error_if_fail = kw.pop('raise_error_if_fail', True) raise_error_if_fail = kw.pop('raise_error_if_fail', True)
env = kw and dict(os.environ, **kw) or None env = kw and dict(os.environ, **kw) or None
command = format_command(*args, **kw) command = format_command(*args, **kw)
self.log('subprocess_kw : %r' % (subprocess_kw,)) log('subprocess_kw : %r' % (subprocess_kw,))
self.log('$ ' + command) log('$ ' + command)
sys.stdout.flush() sys.stdout.flush()
p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE, p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env, **subprocess_kw) stderr=subprocess.PIPE, env=env, **subprocess_kw)
self.process_pid_set.add(p.pid) self.process_pid_set.add(p.pid)
timer = threading.Timer(self.max_timeout, timeoutExpired, args=(p, self.log)) timer = threading.Timer(self.max_timeout, timeoutExpired, args=(p, log))
timer.start() timer.start()
stdout, stderr = subprocess_capture(p, self.log, log_prefix, stdout, stderr = subprocess_capture(p, log, log_prefix,
get_output=get_output) get_output=get_output)
timer.cancel() timer.cancel()
result = dict(status_code=p.returncode, command=command, result = dict(status_code=p.returncode, command=command,
......
...@@ -42,6 +42,10 @@ def createFolder(folder, clean=False): ...@@ -42,6 +42,10 @@ def createFolder(folder, clean=False):
if not(os.path.exists(folder)): if not(os.path.exists(folder)):
os.mkdir(folder) os.mkdir(folder)
def createFolders(folder):
if not(os.path.exists(folder)):
os.makedirs(folder)
class SlapOSControler(object): class SlapOSControler(object):
def __init__(self, working_directory, config, log): def __init__(self, working_directory, config, log):
......
...@@ -34,6 +34,7 @@ import SlapOSControler ...@@ -34,6 +34,7 @@ import SlapOSControler
import json import json
import time import time
import shutil import shutil
import logging
from ProcessManager import SubprocessError, ProcessManager, CancellationError from ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError from subprocess import CalledProcessError
from Updater import Updater from Updater import Updater
...@@ -106,6 +107,40 @@ class NodeTestSuite(SlapOSInstance): ...@@ -106,6 +107,40 @@ class NodeTestSuite(SlapOSInstance):
vcs_repository['repository_id'] = repository_id vcs_repository['repository_id'] = repository_id
vcs_repository['repository_path'] = repository_path vcs_repository['repository_path'] = repository_path
def createSuiteLog(self):
# /srv/slapgrid/slappartXX/srv/var/log/suite/az/mlksjfmlk234Sljssdflkj23KSdfslj/suite.log
# /srv/slapgrid/slappartXX/srv/testnode is working directory
if getattr(self, "log_directory", None) is not None:
if getattr(self, "suite_log_path", None) is None:
suite_log_directory = os.path.join(self.log_directory,
'suite', self.reference)
SlapOSControler.createFolders(suite_log_directory)
self.suite_log_path = os.path.join(suite_log_directory,
'suite.log')
self._initializeSuiteLog()
return self.getSuiteLogPath()
def getSuiteLogPath(self):
if getattr(self, "suite_log_path", None) is None:
return None
return self.suite_log_path
def getSuiteLog(self):
if getattr(self, "suite_log", None) is None:
return None
return self.suite_log
def _initializeSuiteLog(self):
logger_format = '%(asctime)s %(name)-13s: %(levelname)-8s %(message)s'
formatter = logging.Formatter(logger_format)
logging.basicConfig(level=logging.INFO, format=logger_format)
logger = logging.getLogger('erp5testsuite')
file_handler = logging.FileHandler(filename=self.suite_log_path)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.info('Activated logfile %r output' % self.suite_log_path)
self.suite_log = logger.info
class TestNode(object): class TestNode(object):
def __init__(self, log, config): def __init__(self, log, config):
...@@ -206,10 +241,11 @@ branch = %(branch)s ...@@ -206,10 +241,11 @@ branch = %(branch)s
node_test_suite.revision = ','.join(full_revision_list) node_test_suite.revision = ','.join(full_revision_list)
return full_revision_list return full_revision_list
def addWatcher(self,test_result): def addWatcher(self,test_result, node_test_suite):
config = self.config log_file_name = node_test_suite.createSuiteLog()
if config.get('log_file'): if log_file_name is None and config.get('log_file'):
log_file_name = config['log_file'] log_file_name = config['log_file']
if log_file_name is not None:
log_file = open(log_file_name) log_file = open(log_file_name)
log_file.seek(0, 2) log_file.seek(0, 2)
log_file.seek(-min(5000, log_file.tell()), 2) log_file.seek(-min(5000, log_file.tell()), 2)
...@@ -218,6 +254,8 @@ branch = %(branch)s ...@@ -218,6 +254,8 @@ branch = %(branch)s
def checkRevision(self, test_result, node_test_suite): def checkRevision(self, test_result, node_test_suite):
config = self.config config = self.config
log = node_test_suite.getSuiteLog()
if log is None:
log = self.log log = self.log
if node_test_suite.revision != test_result.revision: if node_test_suite.revision != test_result.revision:
log('Disagreement on tested revision, checking out: %r' % ( log('Disagreement on tested revision, checking out: %r' % (
...@@ -235,19 +273,20 @@ branch = %(branch)s ...@@ -235,19 +273,20 @@ branch = %(branch)s
updater.checkout() updater.checkout()
node_test_suite.revision = test_result.revision node_test_suite.revision = test_result.revision
def _prepareSlapOS(self, working_directory, slapos_instance, def _prepareSlapOS(self, working_directory, slapos_instance, log,
create_partition=1, software_path_list=None, **kw): create_partition=1, software_path_list=None, **kw):
""" """
Launch slapos to build software and partitions Launch slapos to build software and partitions
""" """
slapproxy_log = os.path.join(self.config['log_directory'], slapproxy_log = os.path.join(self.config['log_directory'],
'slapproxy.log') 'slapproxy.log')
self.log('Configured slapproxy log to %r' % slapproxy_log) log('Configured slapproxy log to %r' % slapproxy_log)
reset_software = slapos_instance.retry_software_count > 10 reset_software = slapos_instance.retry_software_count > 10
self.log('testnode, retry_software_count : %r' % \ log('testnode, retry_software_count : %r' % \
slapos_instance.retry_software_count) slapos_instance.retry_software_count)
# XXX:TATUYA TO BE FIXED
self.slapos_controler = SlapOSControler.SlapOSControler( self.slapos_controler = SlapOSControler.SlapOSControler(
working_directory, self.config, self.log) working_directory, self.config, log)
self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log, self.slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.process_manager, reset_software=reset_software, process_manager=self.process_manager, reset_software=reset_software,
software_path_list=software_path_list) software_path_list=software_path_list)
...@@ -275,12 +314,15 @@ branch = %(branch)s ...@@ -275,12 +314,15 @@ branch = %(branch)s
like the building of selenium-runner by default like the building of selenium-runner by default
""" """
return self._prepareSlapOS(self.config['slapos_directory'], return self._prepareSlapOS(self.config['slapos_directory'],
test_node_slapos, create_partition=0, test_node_slapos, self.log, create_partition=0,
software_path_list=self.config.get("software_list")) software_path_list=self.config.get("software_list"))
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
log = node_test_suite.getSuiteLog()
if log is None:
log = self.log
return self._prepareSlapOS(node_test_suite.working_directory, return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, node_test_suite, log,
software_path_list=[node_test_suite.custom_profile_path]) software_path_list=[node_test_suite.custom_profile_path])
def _dealShebang(self,run_test_suite_path): def _dealShebang(self,run_test_suite_path):
...@@ -290,7 +332,7 @@ branch = %(branch)s ...@@ -290,7 +332,7 @@ branch = %(branch)s
invocation_list = line[2:].split() invocation_list = line[2:].split()
return invocation_list return invocation_list
def runTestSuite(self, node_test_suite, portal_url): def runTestSuite(self, node_test_suite, portal_url, log=None):
config = self.config config = self.config
parameter_list = [] parameter_list = []
run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" % \ run_test_suite_path_list = glob.glob("%s/*/bin/runTestSuite" % \
...@@ -339,7 +381,9 @@ branch = %(branch)s ...@@ -339,7 +381,9 @@ branch = %(branch)s
self.process_manager.killPreviousRun() self.process_manager.killPreviousRun()
if test_result is not None: if test_result is not None:
try: try:
test_result.removeWatch(self.config['log_file']) for node_test_suite in self.node_test_suite_dict.values():
log_file = node_test_suite.getSuiteLogPath()
test_result.removeWatch(log_file)
except KeyError: except KeyError:
log("KeyError, Watcher already deleted or not added correctly") log("KeyError, Watcher already deleted or not added correctly")
...@@ -373,7 +417,8 @@ branch = %(branch)s ...@@ -373,7 +417,8 @@ branch = %(branch)s
node_test_suite = self.getNodeTestSuite( node_test_suite = self.getNodeTestSuite(
test_suite["test_suite_reference"]) test_suite["test_suite_reference"])
node_test_suite.edit( node_test_suite.edit(
working_directory=self.config['working_directory']) working_directory=self.config['working_directory'],
log_directory=self.config['log_directory'])
node_test_suite.edit(**test_suite) node_test_suite.edit(**test_suite)
run_software = True run_software = True
# Write our own software.cfg to use the local repository # Write our own software.cfg to use the local repository
...@@ -389,7 +434,7 @@ branch = %(branch)s ...@@ -389,7 +434,7 @@ branch = %(branch)s
remote_test_result_needs_cleanup = True remote_test_result_needs_cleanup = True
log("testnode, test_result : %r" % (test_result, )) log("testnode, test_result : %r" % (test_result, ))
if test_result is not None: if test_result is not None:
log_file_name = self.addWatcher(test_result) log_file_name = self.addWatcher(test_result, node_test_suite)
self.checkRevision(test_result,node_test_suite) self.checkRevision(test_result,node_test_suite)
# Now prepare the installation of SlapOS and create instance # Now prepare the installation of SlapOS and create instance
status_dict = self.prepareSlapOSForTestSuite(node_test_suite) status_dict = self.prepareSlapOSForTestSuite(node_test_suite)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment