testnode: SlapOS shared parts support

Shared parts speed up compilation time and is becoming the standard in
SlapOS software installations, so it makes sense to use it in our test
nodes, as it also gives one more opportunity to test this feature.

erp5testnode configuration file supports a new shared_part_list option,
that can be set to a \n separated list of paths to use for shared parts,
following the same rules as slapos.core and slapos.recipe.cmmi (ie. the
first ones are read-only and the last one is read-write).

This shared_part_list option will be set in slapos.cfg used to compile
both the "software for testnode" (ie. selenium-runner) and later the
softwares under tests.
The software under tests will also use a local directory for each test
suite to install shared suite.

The directory structure is now:

  srv/
    shared/
      (shared parts to install selenium runner)
    slapos/
      soft/
        (selenium-runner software)
    testnode/
      foo/ # test suite with reference foo
        inst/
          (partitions of tested software)
        shared/
          (shared parts to install tested software)
        soft/
          (tested software)

and in the configuration srv/shared will be set as initial
shared_part_list.

When installing selenium-runner, srv/shared/ is used to write shared
parts. These shared parts are never removed.

When installing software under test, srv/shared/ and
srv/testnode/foo/shared/ are used. If parts are found in srv/shared they
are used, if they are not found, they are installed in
srv/testnode/foo/shared/.

In practice, this should mean that the shared parts installed by
selenium-runner will be reused for all tested softwares and this should
speed up initial installation of these softwares.

Currently, nothing is implemented regarding removal of unused shared
parts, but in our case:
 - srv/testnode/foo/shared/ will be removed when "foo" is removed.
 - srv/shared/ should be used only when installing selenium-runner.

If this starts to use too much disk space, one quick and dirty
workaround could be to destroy the test node instance and re-create it.

erp5/tests/testERP5TestNode.py

@@ -24,6 +24,13 @@ import tempfile
 import json
 import time
 import re
+from six.moves.configparser import ConfigParser
+try:
+  from unittest import mock
+except ImportError:
+  # BBB python2
+  import mock
+
 
 @contextmanager
 def dummySuiteLog(_):
@@ -69,6 +76,9 @@ class ERP5TestNode(TestCase):
     # XXX how to get property the git path ?
     config = {}
     config["git_binary"] = "git"
+    config["master_url"] = 'http://example.org/'
+    config["proxy_host"] = ''
+    config["proxy_port"] = ''
     config["slapos_directory"] = self.slapos_directory
     config["working_directory"] = self.working_directory
     config["software_directory"] = self.software_directory
@@ -87,8 +97,12 @@ class ERP5TestNode(TestCase):
     config["httpd_software_access_port"] = "9080"
     config["frontend_url"] = "http://frontend/"
     config["software_list"] = ["foo", "bar"]
+    config["partition_reference"] = "part"
+    config["ipv4_address"] = "1.2.3.4"
+    config["ipv6_address"] = "::1"
     config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos"
     config["srv_directory"] = "srv_directory"
+    config["shared_part_list"] = "/not/exists\n /not/exists_either"
 
     testnode = TestNode(config)
     # By default, keep suite logs to stdout for easier debugging
@@ -610,39 +624,72 @@ shared = true
     test_node_slapos = SlapOSInstance(self.slapos_directory)
     runner = test_type_registry[my_test_type](test_node)
     node_test_suite = test_node.getNodeTestSuite('foo')
-    status_dict = {"status_code" : 0}
-    global call_list
-    call_list = []
-    class Patch:
-      def __init__(self, method_name, status_code=0):
-        self.method_name = method_name
-        self.status_code = status_code
-      def __call__(self, *args, **kw):
-        global call_list
-        call_list.append({"method_name": self.method_name,
-                         "args": [x for x in args],
-                          "kw": kw})
-        return {"status_code": self.status_code}
-
-    SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
-    SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
-    SlapOSControler.runComputerPartition = Patch("runComputerPartition")
-    method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler",
-                                                   "runSoftwareRelease"]
-    method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler",
-                                 "runSoftwareRelease", "runComputerPartition"]
-    runner.prepareSlapOSForTestNode(test_node_slapos)
-    self.assertEqual(method_list_for_prepareSlapOSForTestNode,
-                      [x["method_name"] for x in call_list])
-    call_list = []
-    runner.prepareSlapOSForTestSuite(node_test_suite)
-    self.assertEqual(method_list_for_prepareSlapOSForTestSuite,
-                      [x["method_name"] for x in call_list])
-    call_list = []
-    SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1)
-    # TODO : write a test for scalability case
-    self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite,
-                     node_test_suite)
+
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 0}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
+      runner.prepareSlapOSForTestNode(test_node_slapos)
+      self.assertEqual(1, runSoftwareRelease.call_count)
+      self.assertEqual(0, runComputerPartition.call_count)
+
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 0}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
+      runner.prepareSlapOSForTestSuite(node_test_suite)
+      self.assertEqual(1, runSoftwareRelease.call_count)
+      self.assertEqual(1, runComputerPartition.call_count)
+
+    # test node slapos slapos uses the shared parts defined in config
+    cfg_parser = ConfigParser()
+    with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    self.assertEqual(
+        '/not/exists\n/not/exists_either',
+        cfg_parser.get('slapos', 'shared_part_list'))
+
+    # test suite slapos uses the shared parts from the config, plus
+    # a "local" folder for used as shared when installing tested
+    # softwares.
+    cfg_parser = ConfigParser()
+    with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    self.assertEqual(
+        '/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory,
+        cfg_parser.get('slapos', 'shared_part_list'))
+
+    # If running software has status_code 1 we have an error
+    with mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
+          return_value={"status_code": 1}
+        ) as runSoftwareRelease,\
+        mock.patch(
+          'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
+          return_value={"status_code": 0}
+        )  as runComputerPartition,\
+        mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
+        mock.patch('subprocess.Popen'):
+
+      self.assertRaises(
+          SubprocessError,
+          runner.prepareSlapOSForTestSuite,
+          node_test_suite)
+
 
   def test_11_run(self, my_test_type='UnitTest', grade='master'):
     def doNothing(self, *args, **kw):

erp5/util/testnode/SlapOSControler.py

@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3
 
 class SlapOSControler(object):
 
-  def __init__(self, working_directory, config):
+  def __init__(self, working_directory, config, use_local_shared_part=False):
     self.config = config
     self.software_root = os.path.join(working_directory, 'soft')
+    self.shared_part_list = [
+        path.strip() for path in config['shared_part_list'].splitlines()
+    ]
+    if use_local_shared_part:
+      shared = os.path.join(working_directory, 'shared')
+      createFolder(shared)
+      self.shared_part_list = self.shared_part_list + [shared]
+
     self.instance_root = os.path.join(working_directory, 'inst')
     self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
     self.proxy_database = os.path.join(working_directory, 'proxy.db')
@@ -217,7 +225,9 @@ class SlapOSControler(object):
     slapos_config_dict = config.copy()
     slapos_config_dict.update(software_root=self.software_root,
                               instance_root=self.instance_root,
-                              proxy_database=self.proxy_database)
+                              proxy_database=self.proxy_database,
+                              shared_part_list='\n  '.join(self.shared_part_list))
+
     with open(self.slapos_config, 'w') as f:
       f.write(pkg_resources.resource_string(
          'erp5.util.testnode', 'template/slapos.cfg.in').decode() %

erp5/util/testnode/UnitTestRunner.py

@@ -44,16 +44,17 @@ class UnitTestRunner(object):
   def __init__(self, testnode):
     self.testnode = testnode
 
-  def _getSlapOSControler(self, working_directory):
+  def _getSlapOSControler(self, working_directory, use_local_shared_part):
     """
     Create a SlapOSControler
     """
     return SlapOSControler(
                working_directory,
-               self.testnode.config)
- 
+               self.testnode.config,
+               use_local_shared_part=use_local_shared_part)
+
   def _prepareSlapOS(self, working_directory, slapos_instance,
-          create_partition=1, software_path_list=None,**kw):
+          create_partition=1, software_path_list=None, use_local_shared_part=False, **kw):
     """
     Launch slapos to build software and partitions
     """
@@ -67,8 +68,10 @@ class UnitTestRunner(object):
              slapos_instance.retry_software_count)
 
     # XXX Create a new controler because working_directory can be
-    #     Diferent depending of the preparation
-    slapos_controler = self._getSlapOSControler(working_directory)
+    #     Different depending of the preparation
+    slapos_controler = self._getSlapOSControler(
+        working_directory,
+        use_local_shared_part)
 
     slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
        process_manager=self.testnode.process_manager, reset_software=reset_software,
@@ -113,16 +116,17 @@ class UnitTestRunner(object):
 
   def prepareSlapOSForTestSuite(self, node_test_suite):
     """
-    Build softwares needed by testsuites
+    Build softwares needed by testsuites.
     """
     return self._prepareSlapOS(node_test_suite.working_directory,
               node_test_suite,
               software_path_list=[node_test_suite.custom_profile_path],
-              cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)})
+              cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)},
+              use_local_shared_part=True)
 
   def getInstanceRoot(self, node_test_suite):
     return self._getSlapOSControler(
-      node_test_suite.working_directory).instance_root
+      node_test_suite.working_directory, True).instance_root
 
   def runTestSuite(self, node_test_suite, portal_url):
     config = self.testnode.config

erp5/util/testnode/__init__.py

@@ -74,7 +74,7 @@ def main(*args):
               'proxy_port', 'git_binary','zip_binary','node_quantity',
               'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
               'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
-              'computer_id', 'server_url'):
+              'computer_id', 'server_url', 'shared_part_list'):
     CONFIG[key] = config.get('testnode',key)
 
   for key in ('slapos_directory', 'working_directory', 'test_suite_directory',

erp5/util/testnode/template/slapos.cfg.in

 [slapos]
 software_root = %(software_root)s
 instance_root = %(instance_root)s
+shared_part_list = %(shared_part_list)s
 master_url = %(master_url)s
 computer_id = %(computer_id)s
 root_check = False

setup.py

@@ -83,6 +83,7 @@ setup(name=name,
         'slapos.core',
         'xml_marshaller',
         'psutil >= 0.5.0',
+        'mock; python_version < "3"',
       ],
     )