From 316f23c0741283febfda580a08763dd7f4fbd184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Thu, 26 Sep 2019 01:06:30 +0200 Subject: [PATCH 1/3] testnode: SlapOS shared parts support Shared parts speed up compilation time and is becoming the standard in SlapOS software installations, so it makes sense to use it in our test nodes, as it also gives one more opportunity to test this feature. erp5testnode configuration file supports a new shared_part_list option, that can be set to a \n separated list of paths to use for shared parts, following the same rules as slapos.core and slapos.recipe.cmmi (ie. the first ones are read-only and the last one is read-write). This shared_part_list option will be set in slapos.cfg used to compile both the "software for testnode" (ie. selenium-runner) and later the softwares under tests. The software under tests will also use a local directory for each test suite to install shared suite. The directory structure is now: srv/ shared/ (shared parts to install selenium runner) slapos/ soft/ (selenium-runner software) testnode/ foo/ # test suite with reference foo inst/ (partitions of tested software) shared/ (shared parts to install tested software) soft/ (tested software) and in the configuration srv/shared will be set as initial shared_part_list. When installing selenium-runner, srv/shared/ is used to write shared parts. These shared parts are never removed. When installing software under test, srv/shared/ and srv/testnode/foo/shared/ are used. If parts are found in srv/shared they are used, if they are not found, they are installed in srv/testnode/foo/shared/. In practice, this should mean that the shared parts installed by selenium-runner will be reused for all tested softwares and this should speed up initial installation of these softwares. Currently, nothing is implemented regarding removal of unused shared parts, but in our case: - srv/testnode/foo/shared/ will be removed when "foo" is removed. - srv/shared/ should be used only when installing selenium-runner. If this starts to use too much disk space, one quick and dirty workaround could be to destroy the test node instance and re-create it. --- erp5/tests/testERP5TestNode.py | 105 ++++++++++++++-------- erp5/util/testnode/SlapOSControler.py | 14 ++- erp5/util/testnode/UnitTestRunner.py | 22 +++-- erp5/util/testnode/__init__.py | 2 +- erp5/util/testnode/template/slapos.cfg.in | 1 + 5 files changed, 93 insertions(+), 51 deletions(-) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index b73100ab3a3..bcffa28700c 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -25,6 +25,7 @@ import tempfile import json import time import re +from six.moves.configparser import ConfigParser try: from unittest import mock except ImportError: @@ -100,6 +101,7 @@ class ERP5TestNode(TestCase): config["ipv6_address"] = "::1" config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos" config["srv_directory"] = "srv_directory" + config["shared_part_list"] = "/not/exists\n /not/exists_either" testnode = TestNode(config) # By default, keep suite logs to stdout for easier debugging @@ -621,47 +623,72 @@ shared = true test_node_slapos = SlapOSInstance(self.slapos_directory) runner = test_type_registry[my_test_type](test_node) node_test_suite = test_node.getNodeTestSuite('foo') - status_dict = {"status_code" : 0} - global call_list - call_list = [] - class Patch: - def __init__(self, method_name, status_code=0): - self.method_name = method_name - self.status_code = status_code - def __call__(self, *args, **kw): - global call_list - call_list.append({"method_name": self.method_name, - "args": [x for x in args], - "kw": kw}) - return {"status_code": self.status_code} - - original_SlapOSControler_initializeSlapOSControler = SlapOSControler.initializeSlapOSControler - original_SlapOSControler_runSoftwareRelease = SlapOSControler.runSoftwareRelease - original_SlapOSControler_runComputerPartition = SlapOSControler.runComputerPartition - try: - SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler") - SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease") - SlapOSControler.runComputerPartition = Patch("runComputerPartition") - method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler", - "runSoftwareRelease"] - method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler", - "runSoftwareRelease", "runComputerPartition"] + + with mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease', + return_value={"status_code": 0} + ) as runSoftwareRelease,\ + mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition', + return_value={"status_code": 0} + ) as runComputerPartition,\ + mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\ + mock.patch('subprocess.Popen'): + runner.prepareSlapOSForTestNode(test_node_slapos) - self.assertEqual(method_list_for_prepareSlapOSForTestNode, - [x["method_name"] for x in call_list]) - call_list = [] + self.assertEqual(1, runSoftwareRelease.call_count) + self.assertEqual(0, runComputerPartition.call_count) + + with mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease', + return_value={"status_code": 0} + ) as runSoftwareRelease,\ + mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition', + return_value={"status_code": 0} + ) as runComputerPartition,\ + mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\ + mock.patch('subprocess.Popen'): + runner.prepareSlapOSForTestSuite(node_test_suite) - self.assertEqual(method_list_for_prepareSlapOSForTestSuite, - [x["method_name"] for x in call_list]) - call_list = [] - SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1) - # TODO : write a test for scalability case - self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite, - node_test_suite) - finally: - SlapOSControler.initializeSlapOSControler = original_SlapOSControler_initializeSlapOSControler - SlapOSControler.runSoftwareRelease = original_SlapOSControler_runSoftwareRelease - SlapOSControler.runComputerPartition = original_SlapOSControler_runComputerPartition + self.assertEqual(1, runSoftwareRelease.call_count) + self.assertEqual(1, runComputerPartition.call_count) + + # test node slapos slapos uses the shared parts defined in config + cfg_parser = ConfigParser() + with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f: + cfg_parser.readfp(f) + self.assertEqual( + '/not/exists\n/not/exists_either', + cfg_parser.get('slapos', 'shared_part_list')) + + # test suite slapos uses the shared parts from the config, plus + # a "local" folder for used as shared when installing tested + # softwares. + cfg_parser = ConfigParser() + with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f: + cfg_parser.readfp(f) + self.assertEqual( + '/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory, + cfg_parser.get('slapos', 'shared_part_list')) + + # If running software has status_code 1 we have an error + with mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease', + return_value={"status_code": 1} + ) as runSoftwareRelease,\ + mock.patch( + 'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition', + return_value={"status_code": 0} + ) as runComputerPartition,\ + mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\ + mock.patch('subprocess.Popen'): + + self.assertRaises( + SubprocessError, + runner.prepareSlapOSForTestSuite, + node_test_suite) + def test_11_run(self, my_test_type='UnitTest', grade='master'): def doNothing(self, *args, **kw): diff --git a/erp5/util/testnode/SlapOSControler.py b/erp5/util/testnode/SlapOSControler.py index b902a26f0d4..98f1983af0f 100644 --- a/erp5/util/testnode/SlapOSControler.py +++ b/erp5/util/testnode/SlapOSControler.py @@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3 class SlapOSControler(object): - def __init__(self, working_directory, config): + def __init__(self, working_directory, config, use_local_shared_part=False): self.config = config self.software_root = os.path.join(working_directory, 'soft') + self.shared_part_list = [ + path.strip() for path in config['shared_part_list'].splitlines() + ] + if use_local_shared_part: + shared = os.path.join(working_directory, 'shared') + createFolder(shared) + self.shared_part_list = self.shared_part_list + [shared] + self.instance_root = os.path.join(working_directory, 'inst') self.slapos_config = os.path.join(working_directory, 'slapos.cfg') self.proxy_database = os.path.join(working_directory, 'proxy.db') @@ -217,7 +225,9 @@ class SlapOSControler(object): slapos_config_dict = config.copy() slapos_config_dict.update(software_root=self.software_root, instance_root=self.instance_root, - proxy_database=self.proxy_database) + proxy_database=self.proxy_database, + shared_part_list='\n '.join(self.shared_part_list)) + with open(self.slapos_config, 'w') as f: f.write(pkg_resources.resource_string( 'erp5.util.testnode', 'template/slapos.cfg.in').decode() % diff --git a/erp5/util/testnode/UnitTestRunner.py b/erp5/util/testnode/UnitTestRunner.py index f86cc3af878..40e676749d8 100644 --- a/erp5/util/testnode/UnitTestRunner.py +++ b/erp5/util/testnode/UnitTestRunner.py @@ -44,16 +44,17 @@ class UnitTestRunner(object): def __init__(self, testnode): self.testnode = testnode - def _getSlapOSControler(self, working_directory): + def _getSlapOSControler(self, working_directory, use_local_shared_part): """ Create a SlapOSControler """ return SlapOSControler( working_directory, - self.testnode.config) - + self.testnode.config, + use_local_shared_part=use_local_shared_part) + def _prepareSlapOS(self, working_directory, slapos_instance, - create_partition=1, software_path_list=None,**kw): + create_partition=1, software_path_list=None, use_local_shared_part=False, **kw): """ Launch slapos to build software and partitions """ @@ -67,8 +68,10 @@ class UnitTestRunner(object): slapos_instance.retry_software_count) # XXX Create a new controler because working_directory can be - # Diferent depending of the preparation - slapos_controler = self._getSlapOSControler(working_directory) + # Different depending of the preparation + slapos_controler = self._getSlapOSControler( + working_directory, + use_local_shared_part) slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log, process_manager=self.testnode.process_manager, reset_software=reset_software, @@ -113,16 +116,17 @@ class UnitTestRunner(object): def prepareSlapOSForTestSuite(self, node_test_suite): """ - Build softwares needed by testsuites + Build softwares needed by testsuites. """ return self._prepareSlapOS(node_test_suite.working_directory, node_test_suite, software_path_list=[node_test_suite.custom_profile_path], - cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)}) + cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)}, + use_local_shared_part=True) def getInstanceRoot(self, node_test_suite): return self._getSlapOSControler( - node_test_suite.working_directory).instance_root + node_test_suite.working_directory, True).instance_root def runTestSuite(self, node_test_suite, portal_url): config = self.testnode.config diff --git a/erp5/util/testnode/__init__.py b/erp5/util/testnode/__init__.py index e84a7a8760b..02c9504a112 100644 --- a/erp5/util/testnode/__init__.py +++ b/erp5/util/testnode/__init__.py @@ -74,7 +74,7 @@ def main(*args): 'proxy_port', 'git_binary','zip_binary','node_quantity', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', - 'computer_id', 'server_url'): + 'computer_id', 'server_url', 'shared_part_list'): CONFIG[key] = config.get('testnode',key) for key in ('slapos_directory', 'working_directory', 'test_suite_directory', diff --git a/erp5/util/testnode/template/slapos.cfg.in b/erp5/util/testnode/template/slapos.cfg.in index d7508c15e3c..2fa5933089c 100644 --- a/erp5/util/testnode/template/slapos.cfg.in +++ b/erp5/util/testnode/template/slapos.cfg.in @@ -1,6 +1,7 @@ [slapos] software_root = %(software_root)s instance_root = %(instance_root)s +shared_part_list = %(shared_part_list)s master_url = %(master_url)s computer_id = %(computer_id)s root_check = False -- 2.30.9 From 35768fa2e09354b5f7e14430c82cc0cd69e7cfba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Thu, 26 Sep 2019 09:44:02 +0200 Subject: [PATCH 2/3] testnode: pass shared_part_list to runTestSuite who understand it Some test suites who install software during the test, such as SLAPOS-SR tests, could benefit from reusing already installed shared parts. The convention is that --shared_part_list is a os.pathsep (:) separated list of paths of read-only shared parts in which the test is not allowed to write. --- erp5/tests/testERP5TestNode.py | 1 + erp5/util/testnode/UnitTestRunner.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/erp5/tests/testERP5TestNode.py b/erp5/tests/testERP5TestNode.py index bcffa28700c..c3437a23e33 100644 --- a/erp5/tests/testERP5TestNode.py +++ b/erp5/tests/testERP5TestNode.py @@ -613,6 +613,7 @@ shared = true ('--frontend_url', 'http://frontend/'), ('--node_quantity', '3'), ('--xvfb_bin', part('xserver/bin/Xvfb')), + ('--shared_part_list', "/not/exists:/not/exists_either:%s/shared" % node_test_suite.working_directory), ): parser.add_argument(option[0]) expected_parameter_list += option diff --git a/erp5/util/testnode/UnitTestRunner.py b/erp5/util/testnode/UnitTestRunner.py index 40e676749d8..5d6b5ee93db 100644 --- a/erp5/util/testnode/UnitTestRunner.py +++ b/erp5/util/testnode/UnitTestRunner.py @@ -160,6 +160,11 @@ class UnitTestRunner(object): ('--node_quantity', lambda: config['node_quantity']), ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')), ('--project_title', lambda: node_test_suite.project_title), + ('--shared_part_list', lambda: os.pathsep.join( + self._getSlapOSControler( + node_test_suite.working_directory, + True + ).shared_part_list)), ): if option in supported_parameter_set: invocation_list += option, value() -- 2.30.9 From 325395e877de80b36ec349e3deecfbfe97680906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Thu, 26 Sep 2019 12:08:51 +0200 Subject: [PATCH 3/3] Eggtest: support --shared_part_list Shared parts received from test node will be passed as SLAPOS_TEST_SHARED_PART_LIST environment variable to egg tests. This will be useful for SLAPOS-SR tests. --- erp5/util/testsuite/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/erp5/util/testsuite/__init__.py b/erp5/util/testsuite/__init__.py index b3bacf0a6ae..8d1dac9799e 100644 --- a/erp5/util/testsuite/__init__.py +++ b/erp5/util/testsuite/__init__.py @@ -245,8 +245,10 @@ class EggTestSuite(TestSuite): def run(self, test): print(test) try: - status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test', - cwd=self.egg_test_path_dict[test]) + status_dict = self.spawn( + self.python_interpreter, 'setup.py', 'test', + cwd=self.egg_test_path_dict[test], + SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list) except SubprocessError as e: status_dict = e.status_dict test_log = status_dict['stderr'] @@ -302,6 +304,9 @@ def runTestSuite(): parser.add_argument('--source_code_path_list', help='Coma separated list of Eggs folders to test', default='.') + parser.add_argument('--shared_part_list', + help='Shared parts for recursive slapos', + default='') args = parser.parse_args() master = taskdistribution.TaskDistributor(args.master_url) @@ -324,6 +329,7 @@ def runTestSuite(): revision=revision, python_interpreter=args.python_interpreter, egg_test_path_dict=egg_test_path_dict, + shared_part_list=args.shared_part_list ) test_result = master.createTestResult(revision, suite.getTestList(), -- 2.30.9