Commit 5a8490de authored by Jérome Perrin's avatar Jérome Perrin

testnode: SlapOS shared parts support

Shared parts speed up compilation time and is becoming the standard in
SlapOS software installations, so it makes sense to use it in our test
nodes, as it also gives one more opportunity to test this feature.

erp5testnode configuration file supports a new shared_part_list option,
that can be set to a \n separated list of paths to use for shared parts,
following the same rules as slapos.core and slapos.recipe.cmmi (ie. the
first ones are read-only and the last one is read-write).

This shared_part_list option will be set in slapos.cfg used to compile
both the "software for testnode" (ie. selenium-runner) and later the
softwares under tests.
The software under tests will also use a local directory for each test
suite to install shared suite.

The directory structure is now:

  srv/
    shared/
      (shared parts to install selenium runner)
    slapos/
      soft/
        (selenium-runner software)
    testnode/
      foo/ # test suite with reference foo
        inst/
          (partitions of tested software)
        shared/
          (shared parts to install tested software)
        soft/
          (tested software)

and in the configuration srv/shared will be set as initial
shared_part_list.

When installing selenium-runner, srv/shared/ is used to write shared
parts. These shared parts are never removed.

When installing software under test, srv/shared/ and
srv/testnode/foo/shared/ are used. If parts are found in srv/shared they
are used, if they are not found, they are installed in
srv/testnode/foo/shared/.

In practice, this should mean that the shared parts installed by
selenium-runner will be reused for all tested softwares and this should
speed up initial installation of these softwares.

Currently, nothing is implemented regarding removal of unused shared
parts, but in our case:
 - srv/testnode/foo/shared/ will be removed when "foo" is removed.
 - srv/shared/ should be used only when installing selenium-runner.

If this starts to use too much disk space, one quick and dirty
workaround could be to destroy the test node instance and re-create it.
parent af4c8505
......@@ -24,6 +24,13 @@ import tempfile
import json
import time
import re
from six.moves.configparser import ConfigParser
try:
from unittest import mock
except ImportError:
# BBB python2
import mock
@contextmanager
def dummySuiteLog(_):
......@@ -69,6 +76,9 @@ class ERP5TestNode(TestCase):
# XXX how to get property the git path ?
config = {}
config["git_binary"] = "git"
config["master_url"] = 'http://example.org/'
config["proxy_host"] = ''
config["proxy_port"] = ''
config["slapos_directory"] = self.slapos_directory
config["working_directory"] = self.working_directory
config["software_directory"] = self.software_directory
......@@ -87,8 +97,12 @@ class ERP5TestNode(TestCase):
config["httpd_software_access_port"] = "9080"
config["frontend_url"] = "http://frontend/"
config["software_list"] = ["foo", "bar"]
config["partition_reference"] = "part"
config["ipv4_address"] = "1.2.3.4"
config["ipv6_address"] = "::1"
config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos"
config["srv_directory"] = "srv_directory"
config["shared_part_list"] = "/not/exists\n /not/exists_either"
testnode = TestNode(config)
# By default, keep suite logs to stdout for easier debugging
......@@ -610,39 +624,72 @@ shared = true
test_node_slapos = SlapOSInstance(self.slapos_directory)
runner = test_type_registry[my_test_type](test_node)
node_test_suite = test_node.getNodeTestSuite('foo')
status_dict = {"status_code" : 0}
global call_list
call_list = []
class Patch:
def __init__(self, method_name, status_code=0):
self.method_name = method_name
self.status_code = status_code
def __call__(self, *args, **kw):
global call_list
call_list.append({"method_name": self.method_name,
"args": [x for x in args],
"kw": kw})
return {"status_code": self.status_code}
SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
SlapOSControler.runComputerPartition = Patch("runComputerPartition")
method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler",
"runSoftwareRelease"]
method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler",
"runSoftwareRelease", "runComputerPartition"]
runner.prepareSlapOSForTestNode(test_node_slapos)
self.assertEqual(method_list_for_prepareSlapOSForTestNode,
[x["method_name"] for x in call_list])
call_list = []
runner.prepareSlapOSForTestSuite(node_test_suite)
self.assertEqual(method_list_for_prepareSlapOSForTestSuite,
[x["method_name"] for x in call_list])
call_list = []
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1)
# TODO : write a test for scalability case
self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite,
node_test_suite)
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 0}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
runner.prepareSlapOSForTestNode(test_node_slapos)
self.assertEqual(1, runSoftwareRelease.call_count)
self.assertEqual(0, runComputerPartition.call_count)
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 0}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
runner.prepareSlapOSForTestSuite(node_test_suite)
self.assertEqual(1, runSoftwareRelease.call_count)
self.assertEqual(1, runComputerPartition.call_count)
# test node slapos slapos uses the shared parts defined in config
cfg_parser = ConfigParser()
with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f:
cfg_parser.readfp(f)
self.assertEqual(
'/not/exists\n/not/exists_either',
cfg_parser.get('slapos', 'shared_part_list'))
# test suite slapos uses the shared parts from the config, plus
# a "local" folder for used as shared when installing tested
# softwares.
cfg_parser = ConfigParser()
with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f:
cfg_parser.readfp(f)
self.assertEqual(
'/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory,
cfg_parser.get('slapos', 'shared_part_list'))
# If running software has status_code 1 we have an error
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 1}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
self.assertRaises(
SubprocessError,
runner.prepareSlapOSForTestSuite,
node_test_suite)
def test_11_run(self, my_test_type='UnitTest', grade='master'):
def doNothing(self, *args, **kw):
......
......@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3
class SlapOSControler(object):
def __init__(self, working_directory, config):
def __init__(self, working_directory, config, use_local_shared_part=False):
self.config = config
self.software_root = os.path.join(working_directory, 'soft')
self.shared_part_list = [
path.strip() for path in config['shared_part_list'].splitlines()
]
if use_local_shared_part:
shared = os.path.join(working_directory, 'shared')
createFolder(shared)
self.shared_part_list = self.shared_part_list + [shared]
self.instance_root = os.path.join(working_directory, 'inst')
self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
self.proxy_database = os.path.join(working_directory, 'proxy.db')
......@@ -217,7 +225,9 @@ class SlapOSControler(object):
slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root,
proxy_database=self.proxy_database)
proxy_database=self.proxy_database,
shared_part_list='\n '.join(self.shared_part_list))
with open(self.slapos_config, 'w') as f:
f.write(pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
......
......@@ -44,16 +44,17 @@ class UnitTestRunner(object):
def __init__(self, testnode):
self.testnode = testnode
def _getSlapOSControler(self, working_directory):
def _getSlapOSControler(self, working_directory, use_local_shared_part):
"""
Create a SlapOSControler
"""
return SlapOSControler(
working_directory,
self.testnode.config)
self.testnode.config,
use_local_shared_part=use_local_shared_part)
def _prepareSlapOS(self, working_directory, slapos_instance,
create_partition=1, software_path_list=None,**kw):
create_partition=1, software_path_list=None, use_local_shared_part=False, **kw):
"""
Launch slapos to build software and partitions
"""
......@@ -67,8 +68,10 @@ class UnitTestRunner(object):
slapos_instance.retry_software_count)
# XXX Create a new controler because working_directory can be
# Diferent depending of the preparation
slapos_controler = self._getSlapOSControler(working_directory)
# Different depending of the preparation
slapos_controler = self._getSlapOSControler(
working_directory,
use_local_shared_part)
slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.testnode.process_manager, reset_software=reset_software,
......@@ -113,16 +116,17 @@ class UnitTestRunner(object):
def prepareSlapOSForTestSuite(self, node_test_suite):
"""
Build softwares needed by testsuites
Build softwares needed by testsuites.
"""
return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite,
software_path_list=[node_test_suite.custom_profile_path],
cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)})
cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)},
use_local_shared_part=True)
def getInstanceRoot(self, node_test_suite):
return self._getSlapOSControler(
node_test_suite.working_directory).instance_root
node_test_suite.working_directory, True).instance_root
def runTestSuite(self, node_test_suite, portal_url):
config = self.testnode.config
......
......@@ -74,7 +74,7 @@ def main(*args):
'proxy_port', 'git_binary','zip_binary','node_quantity',
'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
'computer_id', 'server_url'):
'computer_id', 'server_url', 'shared_part_list'):
CONFIG[key] = config.get('testnode',key)
for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
......
[slapos]
software_root = %(software_root)s
instance_root = %(instance_root)s
shared_part_list = %(shared_part_list)s
master_url = %(master_url)s
computer_id = %(computer_id)s
root_check = False
......
......@@ -83,6 +83,7 @@ setup(name=name,
'slapos.core',
'xml_marshaller',
'psutil >= 0.5.0',
'mock; python_version < "3"',
],
)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment