Commit ace5021f authored by Jérome Perrin's avatar Jérome Perrin

Testnode: Expose log folder

Test nodes already share a folder with a `suite.log` (and a quick and dirty application to view the log live).

The path of this folder is now passed to test runner so that they can also put some log files there. 

ERP5 test runner uses it by writing the `Z2.log` and `zLOG.log` on the running tests there. 

Slapos SR tests also uses it.

See merge request !948
parents 5b7c94b9 e2f89202
...@@ -75,9 +75,9 @@ class NodeTestSuite(SlapOSInstance): ...@@ -75,9 +75,9 @@ class NodeTestSuite(SlapOSInstance):
while 1: while 1:
log_folder_name = '%s-%s' % (self.reference, log_folder_name = '%s-%s' % (self.reference,
''.join(random.choice(alphabets) for i in range(10))) ''.join(random.choice(alphabets) for i in range(10)))
log_folder_path = os.path.join(self.log_directory, log_folder_name) self.log_folder_path = os.path.join(self.log_directory, log_folder_name)
try: try:
os.makedirs(log_folder_path) os.makedirs(self.log_folder_path)
except OSError as e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
...@@ -85,8 +85,8 @@ class NodeTestSuite(SlapOSInstance): ...@@ -85,8 +85,8 @@ class NodeTestSuite(SlapOSInstance):
break break
# XXX copy the whole content of the log viewer app # XXX copy the whole content of the log viewer app
for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')): for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')):
shutil.copy(fname, log_folder_path) shutil.copy(fname, self.log_folder_path)
self.suite_log_path = os.path.join(log_folder_path, 'suite.log') self.suite_log_path = os.path.join(self.log_folder_path, 'suite.log')
return self.suite_log_path, log_folder_name return self.suite_log_path, log_folder_name
@property @property
......
...@@ -165,6 +165,7 @@ class UnitTestRunner(object): ...@@ -165,6 +165,7 @@ class UnitTestRunner(object):
node_test_suite.working_directory, node_test_suite.working_directory,
True True
).shared_part_list)), ).shared_part_list)),
('--log_directory', lambda: node_test_suite.log_folder_path),
): ):
if option in supported_parameter_set: if option in supported_parameter_set:
invocation_list += option, value() invocation_list += option, value()
......
...@@ -74,7 +74,7 @@ def main(*args): ...@@ -74,7 +74,7 @@ def main(*args):
'proxy_port', 'git_binary','zip_binary','node_quantity', 'proxy_port', 'git_binary','zip_binary','node_quantity',
'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
'computer_id', 'server_url', 'shared_part_list'): 'computer_id', 'server_url', 'shared_part_list', 'keep_log_days'):
CONFIG[key] = config.get('testnode',key) CONFIG[key] = config.get('testnode',key)
for key in ('slapos_directory', 'working_directory', 'test_suite_directory', for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
<a href="logtail.html?noreverse">chronological</a> view. <a href="logtail.html?noreverse">chronological</a> view.
<a id="pause" href='#'>Pause</a>. <a id="pause" href='#'>Pause</a>.
<a href="suite.log">Full log file</a> or <a href="suite.log">Full log file</a> or
<a href="./">snapshots directory</a>
</div> </div>
<pre id="data" style="font-size: xx-small;">Loading...</pre> <pre id="data" style="font-size: xx-small;">Loading...</pre>
</body> </body>
......
...@@ -120,10 +120,10 @@ function get_log() { ...@@ -120,10 +120,10 @@ function get_log() {
log_data = ""; log_data = "";
show_log(); show_log();
setTimeout(get_log, poll);
} else { } else {
throw "Unknown AJAX Error (status " + xhr.status + ")"; console.error("Unknown AJAX Error (status " + xhr.status + ")");
} }
setTimeout(get_log, poll);
} }
}); });
} }
......
...@@ -41,8 +41,6 @@ from .Utils import deunicodeData ...@@ -41,8 +41,6 @@ from .Utils import deunicodeData
from .Utils import rmtree from .Utils import rmtree
from .. import taskdistribution from .. import taskdistribution
MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through
# httd
MAX_TEMP_TIME = 0.01 # time in days we should keep temp files MAX_TEMP_TIME = 0.01 # time in days we should keep temp files
PROFILE_PATH_KEY = 'profile_path' PROFILE_PATH_KEY = 'profile_path'
...@@ -54,14 +52,13 @@ test_type_registry = { ...@@ -54,14 +52,13 @@ test_type_registry = {
class TestNode(object): class TestNode(object):
def __init__(self, config, max_log_time=MAX_LOG_TIME, def __init__(self, config, max_temp_time=MAX_TEMP_TIME):
max_temp_time=MAX_TEMP_TIME):
self.config = config or {} self.config = config or {}
self.process_manager = ProcessManager() self.process_manager = ProcessManager()
self.working_directory = config['working_directory'] self.working_directory = config['working_directory']
self.node_test_suite_dict = {} self.node_test_suite_dict = {}
self.file_handler = None self.file_handler = None
self.max_log_time = max_log_time self.max_log_time = float(config.get('keep_log_days', 15))
self.max_temp_time = max_temp_time self.max_temp_time = max_temp_time
self.url_access = "https://[0::0]:0123" # Ipv6 + port of the node self.url_access = "https://[0::0]:0123" # Ipv6 + port of the node
......
...@@ -248,7 +248,8 @@ class EggTestSuite(TestSuite): ...@@ -248,7 +248,8 @@ class EggTestSuite(TestSuite):
status_dict = self.spawn( status_dict = self.spawn(
self.python_interpreter, 'setup.py', 'test', self.python_interpreter, 'setup.py', 'test',
cwd=self.egg_test_path_dict[test], cwd=self.egg_test_path_dict[test],
SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list) SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list,
SLAPOS_TEST_LOG_DIRECTORY=self.log_directory)
except SubprocessError as e: except SubprocessError as e:
status_dict = e.status_dict status_dict = e.status_dict
test_log = status_dict['stderr'] test_log = status_dict['stderr']
...@@ -298,6 +299,9 @@ def runTestSuite(): ...@@ -298,6 +299,9 @@ def runTestSuite():
parser.add_argument('--frontend_url', parser.add_argument('--frontend_url',
help='The url of the frontend of this test node', help='The url of the frontend of this test node',
default=None) default=None)
parser.add_argument('--log_directory',
help='Directory to store logs',
default=None)
parser.add_argument('--python_interpreter', parser.add_argument('--python_interpreter',
help='Path to python interpreter used to run the test suite', help='Path to python interpreter used to run the test suite',
default='python') default='python')
...@@ -329,7 +333,8 @@ def runTestSuite(): ...@@ -329,7 +333,8 @@ def runTestSuite():
revision=revision, revision=revision,
python_interpreter=args.python_interpreter, python_interpreter=args.python_interpreter,
egg_test_path_dict=egg_test_path_dict, egg_test_path_dict=egg_test_path_dict,
shared_part_list=args.shared_part_list shared_part_list=args.shared_part_list,
log_directory=args.log_directory,
) )
test_result = master.createTestResult(revision, suite.getTestList(), test_result = master.createTestResult(revision, suite.getTestList(),
......
...@@ -22,7 +22,13 @@ class ERP5TypeTestSuite(TestSuite): ...@@ -22,7 +22,13 @@ class ERP5TypeTestSuite(TestSuite):
instance_home = self.instance and 'unit_test.%u' % self.instance \ instance_home = self.instance and 'unit_test.%u' % self.instance \
or 'unit_test' or 'unit_test'
if self.instance: if self.instance:
args = ('--instance_home=' + instance_home, ) + args instance_home = 'unit_test.%u' % self.instance
args = ('--instance_home', instance_home,) + args
if self.log_directory:
log_directory = os.path.join(self.log_directory, args[-1].replace(':', '_'))
os.mkdir(log_directory)
args = ('--log_directory', log_directory, ) + args
if self.__dict__.has_key("bt5_path"): if self.__dict__.has_key("bt5_path"):
args = ("--bt5_path=%s" % self.bt5_path,) + args args = ("--bt5_path=%s" % self.bt5_path,) + args
instance_number = self.instance or 1 instance_number = self.instance or 1
......
...@@ -149,8 +149,8 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase): ...@@ -149,8 +149,8 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase):
"""Start HTTP ZServer in background""" """Start HTTP ZServer in background"""
utils = ZopeTestCase.utils utils = ZopeTestCase.utils
if utils._Z2HOST is None: if utils._Z2HOST is None:
from Products.ERP5Type.tests.runUnitTest import tests_home from Products.ERP5Type.tests.runUnitTest import log_directory
log = os.path.join(tests_home, "Z2.log") log = os.path.join(log_directory, "Z2.log")
message = "Running %s server at %s:%s\n" message = "Running %s server at %s:%s\n"
if int(os.environ.get('erp5_wsgi', 0)): if int(os.environ.get('erp5_wsgi', 0)):
from Products.ERP5.bin.zopewsgi import app_wrapper, createServer from Products.ERP5.bin.zopewsgi import app_wrapper, createServer
......
...@@ -103,6 +103,7 @@ def main(): ...@@ -103,6 +103,7 @@ def main():
default=None) default=None)
testnode_group.add_argument("--xvfb_bin", default=None) testnode_group.add_argument("--xvfb_bin", default=None)
testnode_group.add_argument("--firefox_bin", default=None) testnode_group.add_argument("--firefox_bin", default=None)
testnode_group.add_argument("--log_directory", default=None)
args = parser.parse_args() args = parser.parse_args()
if args.bt5_path is not None: if args.bt5_path is not None:
...@@ -132,7 +133,8 @@ def main(): ...@@ -132,7 +133,8 @@ def main():
zserver_frontend_url_list=args.zserver_frontend_url_list, zserver_frontend_url_list=args.zserver_frontend_url_list,
bt5_path=args.bt5_path, bt5_path=args.bt5_path,
firefox_bin=args.firefox_bin, firefox_bin=args.firefox_bin,
xvfb_bin=args.xvfb_bin) xvfb_bin=args.xvfb_bin,
log_directory=args.log_directory)
test_result = master.createTestResult(revision, suite.getTestList(), test_result = master.createTestResult(revision, suite.getTestList(),
args.test_node_title, suite.allow_restart, test_suite_title, args.test_node_title, suite.allow_restart, test_suite_title,
args.project_title) args.project_title)
......
...@@ -158,6 +158,7 @@ Options: ...@@ -158,6 +158,7 @@ Options:
--sys_path=path,path Comma-separated list of paths which will be used to --sys_path=path,path Comma-separated list of paths which will be used to
extend sys.path extend sys.path
--instance_home=PATH Create/use test instance in given path --instance_home=PATH Create/use test instance in given path
--log_directory=PATH Create log files in given path
When no unit test is specified, only activities are processed. When no unit test is specified, only activities are processed.
""" """
...@@ -492,7 +493,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None): ...@@ -492,7 +493,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
_print("conflicting options: --zeo_client and --zeo_server") _print("conflicting options: --zeo_client and --zeo_server")
sys.exit(1) sys.exit(1)
instance_home = os.environ['INSTANCE_HOME'] instance_home = os.environ['INSTANCE_HOME']
os.environ.setdefault('EVENT_LOG_FILE', os.path.join(tests_home, 'zLOG.log')) os.environ.setdefault('EVENT_LOG_FILE', os.path.join(log_directory, 'zLOG.log'))
os.environ.setdefault('EVENT_LOG_SEVERITY', '-300') os.environ.setdefault('EVENT_LOG_SEVERITY', '-300')
# For numpy parallel-primitives such as numpy.dot() that is used in # For numpy parallel-primitives such as numpy.dot() that is used in
# testReceiptRecognition for example. # testReceiptRecognition for example.
...@@ -759,6 +760,7 @@ def main(argument_list=None): ...@@ -759,6 +760,7 @@ def main(argument_list=None):
"products_path=", "products_path=",
"sys_path=", "sys_path=",
"instance_home=", "instance_home=",
"log_directory="
]) ])
except getopt.GetoptError, msg: except getopt.GetoptError, msg:
usage(sys.stderr, msg) usage(sys.stderr, msg)
...@@ -769,6 +771,7 @@ def main(argument_list=None): ...@@ -769,6 +771,7 @@ def main(argument_list=None):
debug = 0 debug = 0
run_only = None run_only = None
instance_home = os.path.join(real_instance_home, 'unit_test') instance_home = os.path.join(real_instance_home, 'unit_test')
_log_directory = instance_home
bt5_path_list = [] bt5_path_list = []
...@@ -877,6 +880,8 @@ def main(argument_list=None): ...@@ -877,6 +880,8 @@ def main(argument_list=None):
sys.path.extend(arg.split(',')) sys.path.extend(arg.split(','))
elif opt == "--instance_home": elif opt == "--instance_home":
instance_home = os.path.abspath(arg) instance_home = os.path.abspath(arg)
elif opt == "--log_directory":
_log_directory = os.path.abspath(arg)
bt5_path_list += filter(None, bt5_path_list += filter(None,
os.environ.get("erp5_tests_bt5_path", "").split(',')) os.environ.get("erp5_tests_bt5_path", "").split(','))
...@@ -895,6 +900,8 @@ def main(argument_list=None): ...@@ -895,6 +900,8 @@ def main(argument_list=None):
initializeInstanceHome(tests_framework_home, initializeInstanceHome(tests_framework_home,
real_instance_home, real_instance_home,
instance_home) instance_home)
global log_directory
log_directory = _log_directory or test_home
result = runUnitTestList(test_list=args, result = runUnitTestList(test_list=args,
verbosity=verbosity, verbosity=verbosity,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment