From 346012531f9ba6daeeab9f1bd9bae7623ff4ec22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 4 Oct 2019 12:29:09 +0200 Subject: [PATCH 1/7] testnode: pass --log_directory to runTestSuite This way, test suite can place some log files in this directory, so that we can inspect when tests are running. --- erp5/util/testnode/NodeTestSuite.py | 8 ++++---- erp5/util/testnode/UnitTestRunner.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/erp5/util/testnode/NodeTestSuite.py b/erp5/util/testnode/NodeTestSuite.py index 6916d36bf50..6aa078edfbc 100644 --- a/erp5/util/testnode/NodeTestSuite.py +++ b/erp5/util/testnode/NodeTestSuite.py @@ -75,9 +75,9 @@ class NodeTestSuite(SlapOSInstance): while 1: log_folder_name = '%s-%s' % (self.reference, ''.join(random.choice(alphabets) for i in range(10))) - log_folder_path = os.path.join(self.log_directory, log_folder_name) + self.log_folder_path = os.path.join(self.log_directory, log_folder_name) try: - os.makedirs(log_folder_path) + os.makedirs(self.log_folder_path) except OSError as e: if e.errno != errno.EEXIST: raise @@ -85,8 +85,8 @@ class NodeTestSuite(SlapOSInstance): break # XXX copy the whole content of the log viewer app for fname in glob.glob(os.path.join(os.path.dirname(__file__), 'js-logtail', '*')): - shutil.copy(fname, log_folder_path) - self.suite_log_path = os.path.join(log_folder_path, 'suite.log') + shutil.copy(fname, self.log_folder_path) + self.suite_log_path = os.path.join(self.log_folder_path, 'suite.log') return self.suite_log_path, log_folder_name @property diff --git a/erp5/util/testnode/UnitTestRunner.py b/erp5/util/testnode/UnitTestRunner.py index 5d6b5ee93db..a19e15c2ed8 100644 --- a/erp5/util/testnode/UnitTestRunner.py +++ b/erp5/util/testnode/UnitTestRunner.py @@ -165,6 +165,7 @@ class UnitTestRunner(object): node_test_suite.working_directory, True ).shared_part_list)), + ('--log_directory', lambda: node_test_suite.log_folder_path), ): if option in supported_parameter_set: invocation_list += option, value() -- 2.30.9 From 07e28bf721d10fee0368b19fc7de4cfd05c59425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 4 Oct 2019 12:45:48 +0200 Subject: [PATCH 2/7] runUnitTest: new --log_directory option Log files will be created in this directory. --- product/ERP5Type/tests/ProcessingNodeTestCase.py | 4 ++-- product/ERP5Type/tests/runUnitTest.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/product/ERP5Type/tests/ProcessingNodeTestCase.py b/product/ERP5Type/tests/ProcessingNodeTestCase.py index dec31b3bcfb..ada7a4a6173 100644 --- a/product/ERP5Type/tests/ProcessingNodeTestCase.py +++ b/product/ERP5Type/tests/ProcessingNodeTestCase.py @@ -149,8 +149,8 @@ class ProcessingNodeTestCase(ZopeTestCase.TestCase): """Start HTTP ZServer in background""" utils = ZopeTestCase.utils if utils._Z2HOST is None: - from Products.ERP5Type.tests.runUnitTest import tests_home - log = os.path.join(tests_home, "Z2.log") + from Products.ERP5Type.tests.runUnitTest import log_directory + log = os.path.join(log_directory, "Z2.log") message = "Running %s server at %s:%s\n" if int(os.environ.get('erp5_wsgi', 0)): from Products.ERP5.bin.zopewsgi import app_wrapper, createServer diff --git a/product/ERP5Type/tests/runUnitTest.py b/product/ERP5Type/tests/runUnitTest.py index b48e03e382e..5c62595b15a 100755 --- a/product/ERP5Type/tests/runUnitTest.py +++ b/product/ERP5Type/tests/runUnitTest.py @@ -158,6 +158,7 @@ Options: --sys_path=path,path Comma-separated list of paths which will be used to extend sys.path --instance_home=PATH Create/use test instance in given path + --log_directory=PATH Create log files in given path When no unit test is specified, only activities are processed. """ @@ -492,7 +493,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None): _print("conflicting options: --zeo_client and --zeo_server") sys.exit(1) instance_home = os.environ['INSTANCE_HOME'] - os.environ.setdefault('EVENT_LOG_FILE', os.path.join(tests_home, 'zLOG.log')) + os.environ.setdefault('EVENT_LOG_FILE', os.path.join(log_directory, 'zLOG.log')) os.environ.setdefault('EVENT_LOG_SEVERITY', '-300') # For numpy parallel-primitives such as numpy.dot() that is used in # testReceiptRecognition for example. @@ -759,6 +760,7 @@ def main(argument_list=None): "products_path=", "sys_path=", "instance_home=", + "log_directory=" ]) except getopt.GetoptError, msg: usage(sys.stderr, msg) @@ -769,6 +771,7 @@ def main(argument_list=None): debug = 0 run_only = None instance_home = os.path.join(real_instance_home, 'unit_test') + _log_directory = instance_home bt5_path_list = [] @@ -877,6 +880,8 @@ def main(argument_list=None): sys.path.extend(arg.split(',')) elif opt == "--instance_home": instance_home = os.path.abspath(arg) + elif opt == "--log_directory": + _log_directory = os.path.abspath(arg) bt5_path_list += filter(None, os.environ.get("erp5_tests_bt5_path", "").split(',')) @@ -895,6 +900,8 @@ def main(argument_list=None): initializeInstanceHome(tests_framework_home, real_instance_home, instance_home) + global log_directory + log_directory = _log_directory or test_home result = runUnitTestList(test_list=args, verbosity=verbosity, -- 2.30.9 From 4170bc625a5e039ef93d0b4b13ac20193ce8a16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 4 Oct 2019 12:53:21 +0200 Subject: [PATCH 3/7] runTestSuite: propagate --log_directory to runUnitTest Each runUnitTest will use a subdirectory in the log directory used by runTestSuite. --- product/ERP5Type/tests/ERP5TypeTestSuite.py | 8 +++++++- product/ERP5Type/tests/runTestSuite.py | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/product/ERP5Type/tests/ERP5TypeTestSuite.py b/product/ERP5Type/tests/ERP5TypeTestSuite.py index 8f6848f3950..19d7ae99dd8 100644 --- a/product/ERP5Type/tests/ERP5TypeTestSuite.py +++ b/product/ERP5Type/tests/ERP5TypeTestSuite.py @@ -22,7 +22,13 @@ class ERP5TypeTestSuite(TestSuite): instance_home = self.instance and 'unit_test.%u' % self.instance \ or 'unit_test' if self.instance: - args = ('--instance_home=' + instance_home, ) + args + instance_home = 'unit_test.%u' % self.instance + args = ('--instance_home', instance_home,) + args + if self.log_directory: + log_directory = os.path.join(self.log_directory, args[-1].replace(':', '_')) + os.mkdir(log_directory) + args = ('--log_directory', log_directory, ) + args + if self.__dict__.has_key("bt5_path"): args = ("--bt5_path=%s" % self.bt5_path,) + args instance_number = self.instance or 1 diff --git a/product/ERP5Type/tests/runTestSuite.py b/product/ERP5Type/tests/runTestSuite.py index c89cbd3faf5..e1f7077faa0 100644 --- a/product/ERP5Type/tests/runTestSuite.py +++ b/product/ERP5Type/tests/runTestSuite.py @@ -103,6 +103,7 @@ def main(): default=None) testnode_group.add_argument("--xvfb_bin", default=None) testnode_group.add_argument("--firefox_bin", default=None) + testnode_group.add_argument("--log_directory", default=None) args = parser.parse_args() if args.bt5_path is not None: @@ -132,7 +133,8 @@ def main(): zserver_frontend_url_list=args.zserver_frontend_url_list, bt5_path=args.bt5_path, firefox_bin=args.firefox_bin, - xvfb_bin=args.xvfb_bin) + xvfb_bin=args.xvfb_bin, + log_directory=args.log_directory) test_result = master.createTestResult(revision, suite.getTestList(), args.test_node_title, suite.allow_restart, test_suite_title, args.project_title) -- 2.30.9 From 727c3bcbd4220307a6d6c22e92a0d88b75e18c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Fri, 4 Oct 2019 12:57:48 +0200 Subject: [PATCH 4/7] EggTestSuite: support --log_directory Just propagate it as SLAPOS_TEST_LOG_DIRECTORY, then tests (running with python setup.py test) will be able to use it if they wish. --- erp5/util/testsuite/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/erp5/util/testsuite/__init__.py b/erp5/util/testsuite/__init__.py index 8d1dac9799e..753bad8ef87 100644 --- a/erp5/util/testsuite/__init__.py +++ b/erp5/util/testsuite/__init__.py @@ -227,7 +227,7 @@ class TestSuite(object): raise SubprocessError(result) return result -# (XXX) The code bellow is an generic extension to run a test for any egg. +# (XXX) The code bellow is an generic extension to run a test for any egg. # The code above was moved from ERP5 code base, because it is generic # Enough to be used by others. @@ -248,7 +248,8 @@ class EggTestSuite(TestSuite): status_dict = self.spawn( self.python_interpreter, 'setup.py', 'test', cwd=self.egg_test_path_dict[test], - SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list) + SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list, + SLAPOS_TEST_LOG_DIRECTORY=self.log_directory) except SubprocessError as e: status_dict = e.status_dict test_log = status_dict['stderr'] @@ -298,6 +299,9 @@ def runTestSuite(): parser.add_argument('--frontend_url', help='The url of the frontend of this test node', default=None) + parser.add_argument('--log_directory', + help='Directory to store logs', + default=None) parser.add_argument('--python_interpreter', help='Path to python interpreter used to run the test suite', default='python') @@ -329,7 +333,8 @@ def runTestSuite(): revision=revision, python_interpreter=args.python_interpreter, egg_test_path_dict=egg_test_path_dict, - shared_part_list=args.shared_part_list + shared_part_list=args.shared_part_list, + log_directory=args.log_directory, ) test_result = master.createTestResult(revision, suite.getTestList(), -- 2.30.9 From b4e9a9b4b7d6f9b080bff089fdf4045286103d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Mon, 13 Apr 2020 03:43:23 +0200 Subject: [PATCH 5/7] testnode: include a link to snapshot dir in log viewer From there we can now access zLOG.log and Z2.log for erp5 and snapshots from slapos software release tests --- erp5/util/testnode/js-logtail/logtail.html | 1 + 1 file changed, 1 insertion(+) diff --git a/erp5/util/testnode/js-logtail/logtail.html b/erp5/util/testnode/js-logtail/logtail.html index 9df52ed672c..5045a5e3ba2 100644 --- a/erp5/util/testnode/js-logtail/logtail.html +++ b/erp5/util/testnode/js-logtail/logtail.html @@ -14,6 +14,7 @@ chronological view. Pause. Full log file or + snapshots directory 
Loading...
-- 2.30.9 From e6034f398912f2a91c1cd4eadb2e434ef180b1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Mon, 13 Apr 2020 03:44:43 +0200 Subject: [PATCH 6/7] testnode: don't crash log viewer app on network error --- erp5/util/testnode/js-logtail/logtail.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erp5/util/testnode/js-logtail/logtail.js b/erp5/util/testnode/js-logtail/logtail.js index 5e1a4f39b6e..d166840cf2b 100644 --- a/erp5/util/testnode/js-logtail/logtail.js +++ b/erp5/util/testnode/js-logtail/logtail.js @@ -120,10 +120,10 @@ function get_log() { log_data = ""; show_log(); - setTimeout(get_log, poll); } else { - throw "Unknown AJAX Error (status " + xhr.status + ")"; + console.error("Unknown AJAX Error (status " + xhr.status + ")"); } + setTimeout(get_log, poll); } }); } -- 2.30.9 From e2f89202402beaa57d79f1dc66182906f7bb06e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Mon, 13 Apr 2020 06:29:53 +0200 Subject: [PATCH 7/7] testnode: make the number of days to keep log configurable --- erp5/util/testnode/__init__.py | 2 +- erp5/util/testnode/testnode.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/erp5/util/testnode/__init__.py b/erp5/util/testnode/__init__.py index 02c9504a112..9160f4b108a 100644 --- a/erp5/util/testnode/__init__.py +++ b/erp5/util/testnode/__init__.py @@ -74,7 +74,7 @@ def main(*args): 'proxy_port', 'git_binary','zip_binary','node_quantity', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', - 'computer_id', 'server_url', 'shared_part_list'): + 'computer_id', 'server_url', 'shared_part_list', 'keep_log_days'): CONFIG[key] = config.get('testnode',key) for key in ('slapos_directory', 'working_directory', 'test_suite_directory', diff --git a/erp5/util/testnode/testnode.py b/erp5/util/testnode/testnode.py index 19f9ad41b03..d224631b1ab 100644 --- a/erp5/util/testnode/testnode.py +++ b/erp5/util/testnode/testnode.py @@ -41,8 +41,6 @@ from .Utils import deunicodeData from .Utils import rmtree from .. import taskdistribution -MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through - # httd MAX_TEMP_TIME = 0.01 # time in days we should keep temp files PROFILE_PATH_KEY = 'profile_path' @@ -54,14 +52,13 @@ test_type_registry = { class TestNode(object): - def __init__(self, config, max_log_time=MAX_LOG_TIME, - max_temp_time=MAX_TEMP_TIME): + def __init__(self, config, max_temp_time=MAX_TEMP_TIME): self.config = config or {} self.process_manager = ProcessManager() self.working_directory = config['working_directory'] self.node_test_suite_dict = {} self.file_handler = None - self.max_log_time = max_log_time + self.max_log_time = float(config.get('keep_log_days', 15)) self.max_temp_time = max_temp_time self.url_access = "https://[0::0]:0123" # Ipv6 + port of the node -- 2.30.9