Commit c30b29ce authored by Jérome Perrin's avatar Jérome Perrin

Testnode: Use shared parts when building softwares

/reviewed-on nexedi/erp5!940
parents 5b3f2426 325395e8
...@@ -25,6 +25,7 @@ import tempfile ...@@ -25,6 +25,7 @@ import tempfile
import json import json
import time import time
import re import re
from six.moves.configparser import ConfigParser
try: try:
from unittest import mock from unittest import mock
except ImportError: except ImportError:
...@@ -100,6 +101,7 @@ class ERP5TestNode(TestCase): ...@@ -100,6 +101,7 @@ class ERP5TestNode(TestCase):
config["ipv6_address"] = "::1" config["ipv6_address"] = "::1"
config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos" config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos"
config["srv_directory"] = "srv_directory" config["srv_directory"] = "srv_directory"
config["shared_part_list"] = "/not/exists\n /not/exists_either"
testnode = TestNode(config) testnode = TestNode(config)
# By default, keep suite logs to stdout for easier debugging # By default, keep suite logs to stdout for easier debugging
...@@ -611,6 +613,7 @@ shared = true ...@@ -611,6 +613,7 @@ shared = true
('--frontend_url', 'http://frontend/'), ('--frontend_url', 'http://frontend/'),
('--node_quantity', '3'), ('--node_quantity', '3'),
('--xvfb_bin', part('xserver/bin/Xvfb')), ('--xvfb_bin', part('xserver/bin/Xvfb')),
('--shared_part_list', "/not/exists:/not/exists_either:%s/shared" % node_test_suite.working_directory),
): ):
parser.add_argument(option[0]) parser.add_argument(option[0])
expected_parameter_list += option expected_parameter_list += option
...@@ -621,47 +624,72 @@ shared = true ...@@ -621,47 +624,72 @@ shared = true
test_node_slapos = SlapOSInstance(self.slapos_directory) test_node_slapos = SlapOSInstance(self.slapos_directory)
runner = test_type_registry[my_test_type](test_node) runner = test_type_registry[my_test_type](test_node)
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
status_dict = {"status_code" : 0}
global call_list with mock.patch(
call_list = [] 'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
class Patch: return_value={"status_code": 0}
def __init__(self, method_name, status_code=0): ) as runSoftwareRelease,\
self.method_name = method_name mock.patch(
self.status_code = status_code 'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
def __call__(self, *args, **kw): return_value={"status_code": 0}
global call_list ) as runComputerPartition,\
call_list.append({"method_name": self.method_name, mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
"args": [x for x in args], mock.patch('subprocess.Popen'):
"kw": kw})
return {"status_code": self.status_code}
original_SlapOSControler_initializeSlapOSControler = SlapOSControler.initializeSlapOSControler
original_SlapOSControler_runSoftwareRelease = SlapOSControler.runSoftwareRelease
original_SlapOSControler_runComputerPartition = SlapOSControler.runComputerPartition
try:
SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
SlapOSControler.runComputerPartition = Patch("runComputerPartition")
method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler",
"runSoftwareRelease"]
method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler",
"runSoftwareRelease", "runComputerPartition"]
runner.prepareSlapOSForTestNode(test_node_slapos) runner.prepareSlapOSForTestNode(test_node_slapos)
self.assertEqual(method_list_for_prepareSlapOSForTestNode, self.assertEqual(1, runSoftwareRelease.call_count)
[x["method_name"] for x in call_list]) self.assertEqual(0, runComputerPartition.call_count)
call_list = []
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 0}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
runner.prepareSlapOSForTestSuite(node_test_suite) runner.prepareSlapOSForTestSuite(node_test_suite)
self.assertEqual(method_list_for_prepareSlapOSForTestSuite, self.assertEqual(1, runSoftwareRelease.call_count)
[x["method_name"] for x in call_list]) self.assertEqual(1, runComputerPartition.call_count)
call_list = []
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1) # test node slapos slapos uses the shared parts defined in config
# TODO : write a test for scalability case cfg_parser = ConfigParser()
self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite, with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f:
node_test_suite) cfg_parser.readfp(f)
finally: self.assertEqual(
SlapOSControler.initializeSlapOSControler = original_SlapOSControler_initializeSlapOSControler '/not/exists\n/not/exists_either',
SlapOSControler.runSoftwareRelease = original_SlapOSControler_runSoftwareRelease cfg_parser.get('slapos', 'shared_part_list'))
SlapOSControler.runComputerPartition = original_SlapOSControler_runComputerPartition
# test suite slapos uses the shared parts from the config, plus
# a "local" folder for used as shared when installing tested
# softwares.
cfg_parser = ConfigParser()
with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f:
cfg_parser.readfp(f)
self.assertEqual(
'/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory,
cfg_parser.get('slapos', 'shared_part_list'))
# If running software has status_code 1 we have an error
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 1}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
self.assertRaises(
SubprocessError,
runner.prepareSlapOSForTestSuite,
node_test_suite)
def test_11_run(self, my_test_type='UnitTest', grade='master'): def test_11_run(self, my_test_type='UnitTest', grade='master'):
def doNothing(self, *args, **kw): def doNothing(self, *args, **kw):
......
...@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3 ...@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3
class SlapOSControler(object): class SlapOSControler(object):
def __init__(self, working_directory, config): def __init__(self, working_directory, config, use_local_shared_part=False):
self.config = config self.config = config
self.software_root = os.path.join(working_directory, 'soft') self.software_root = os.path.join(working_directory, 'soft')
self.shared_part_list = [
path.strip() for path in config['shared_part_list'].splitlines()
]
if use_local_shared_part:
shared = os.path.join(working_directory, 'shared')
createFolder(shared)
self.shared_part_list = self.shared_part_list + [shared]
self.instance_root = os.path.join(working_directory, 'inst') self.instance_root = os.path.join(working_directory, 'inst')
self.slapos_config = os.path.join(working_directory, 'slapos.cfg') self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
self.proxy_database = os.path.join(working_directory, 'proxy.db') self.proxy_database = os.path.join(working_directory, 'proxy.db')
...@@ -217,7 +225,9 @@ class SlapOSControler(object): ...@@ -217,7 +225,9 @@ class SlapOSControler(object):
slapos_config_dict = config.copy() slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root, slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root, instance_root=self.instance_root,
proxy_database=self.proxy_database) proxy_database=self.proxy_database,
shared_part_list='\n '.join(self.shared_part_list))
with open(self.slapos_config, 'w') as f: with open(self.slapos_config, 'w') as f:
f.write(pkg_resources.resource_string( f.write(pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() % 'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
......
...@@ -44,16 +44,17 @@ class UnitTestRunner(object): ...@@ -44,16 +44,17 @@ class UnitTestRunner(object):
def __init__(self, testnode): def __init__(self, testnode):
self.testnode = testnode self.testnode = testnode
def _getSlapOSControler(self, working_directory): def _getSlapOSControler(self, working_directory, use_local_shared_part):
""" """
Create a SlapOSControler Create a SlapOSControler
""" """
return SlapOSControler( return SlapOSControler(
working_directory, working_directory,
self.testnode.config) self.testnode.config,
use_local_shared_part=use_local_shared_part)
def _prepareSlapOS(self, working_directory, slapos_instance, def _prepareSlapOS(self, working_directory, slapos_instance,
create_partition=1, software_path_list=None,**kw): create_partition=1, software_path_list=None, use_local_shared_part=False, **kw):
""" """
Launch slapos to build software and partitions Launch slapos to build software and partitions
""" """
...@@ -67,8 +68,10 @@ class UnitTestRunner(object): ...@@ -67,8 +68,10 @@ class UnitTestRunner(object):
slapos_instance.retry_software_count) slapos_instance.retry_software_count)
# XXX Create a new controler because working_directory can be # XXX Create a new controler because working_directory can be
# Diferent depending of the preparation # Different depending of the preparation
slapos_controler = self._getSlapOSControler(working_directory) slapos_controler = self._getSlapOSControler(
working_directory,
use_local_shared_part)
slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log, slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.testnode.process_manager, reset_software=reset_software, process_manager=self.testnode.process_manager, reset_software=reset_software,
...@@ -113,16 +116,17 @@ class UnitTestRunner(object): ...@@ -113,16 +116,17 @@ class UnitTestRunner(object):
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
""" """
Build softwares needed by testsuites Build softwares needed by testsuites.
""" """
return self._prepareSlapOS(node_test_suite.working_directory, return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, node_test_suite,
software_path_list=[node_test_suite.custom_profile_path], software_path_list=[node_test_suite.custom_profile_path],
cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)}) cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)},
use_local_shared_part=True)
def getInstanceRoot(self, node_test_suite): def getInstanceRoot(self, node_test_suite):
return self._getSlapOSControler( return self._getSlapOSControler(
node_test_suite.working_directory).instance_root node_test_suite.working_directory, True).instance_root
def runTestSuite(self, node_test_suite, portal_url): def runTestSuite(self, node_test_suite, portal_url):
config = self.testnode.config config = self.testnode.config
...@@ -156,6 +160,11 @@ class UnitTestRunner(object): ...@@ -156,6 +160,11 @@ class UnitTestRunner(object):
('--node_quantity', lambda: config['node_quantity']), ('--node_quantity', lambda: config['node_quantity']),
('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')), ('--xvfb_bin', lambda: path('xvfb', 'xserver/bin/Xvfb')),
('--project_title', lambda: node_test_suite.project_title), ('--project_title', lambda: node_test_suite.project_title),
('--shared_part_list', lambda: os.pathsep.join(
self._getSlapOSControler(
node_test_suite.working_directory,
True
).shared_part_list)),
): ):
if option in supported_parameter_set: if option in supported_parameter_set:
invocation_list += option, value() invocation_list += option, value()
......
...@@ -74,7 +74,7 @@ def main(*args): ...@@ -74,7 +74,7 @@ def main(*args):
'proxy_port', 'git_binary','zip_binary','node_quantity', 'proxy_port', 'git_binary','zip_binary','node_quantity',
'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
'computer_id', 'server_url'): 'computer_id', 'server_url', 'shared_part_list'):
CONFIG[key] = config.get('testnode',key) CONFIG[key] = config.get('testnode',key)
for key in ('slapos_directory', 'working_directory', 'test_suite_directory', for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
......
[slapos] [slapos]
software_root = %(software_root)s software_root = %(software_root)s
instance_root = %(instance_root)s instance_root = %(instance_root)s
shared_part_list = %(shared_part_list)s
master_url = %(master_url)s master_url = %(master_url)s
computer_id = %(computer_id)s computer_id = %(computer_id)s
root_check = False root_check = False
......
...@@ -245,8 +245,10 @@ class EggTestSuite(TestSuite): ...@@ -245,8 +245,10 @@ class EggTestSuite(TestSuite):
def run(self, test): def run(self, test):
print(test) print(test)
try: try:
status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test', status_dict = self.spawn(
cwd=self.egg_test_path_dict[test]) self.python_interpreter, 'setup.py', 'test',
cwd=self.egg_test_path_dict[test],
SLAPOS_TEST_SHARED_PART_LIST=self.shared_part_list)
except SubprocessError as e: except SubprocessError as e:
status_dict = e.status_dict status_dict = e.status_dict
test_log = status_dict['stderr'] test_log = status_dict['stderr']
...@@ -302,6 +304,9 @@ def runTestSuite(): ...@@ -302,6 +304,9 @@ def runTestSuite():
parser.add_argument('--source_code_path_list', parser.add_argument('--source_code_path_list',
help='Coma separated list of Eggs folders to test', help='Coma separated list of Eggs folders to test',
default='.') default='.')
parser.add_argument('--shared_part_list',
help='Shared parts for recursive slapos',
default='')
args = parser.parse_args() args = parser.parse_args()
master = taskdistribution.TaskDistributor(args.master_url) master = taskdistribution.TaskDistributor(args.master_url)
...@@ -324,6 +329,7 @@ def runTestSuite(): ...@@ -324,6 +329,7 @@ def runTestSuite():
revision=revision, revision=revision,
python_interpreter=args.python_interpreter, python_interpreter=args.python_interpreter,
egg_test_path_dict=egg_test_path_dict, egg_test_path_dict=egg_test_path_dict,
shared_part_list=args.shared_part_list
) )
test_result = master.createTestResult(revision, suite.getTestList(), test_result = master.createTestResult(revision, suite.getTestList(),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment