grid: new shared_part_list config file option

This new option for slapos.cfg allow to define a list of paths to use
for shared parts feature of slapos.recipe.cmmmi.
This will be injected in the buildout profile used to install software,
as ${buildout:shared-part-list} which is the option slapos.recipe.cmmi
use since 0.10.

Note that the option changed in slapos.recipe.cmmi because this was
incompatible. In 0.9, the option was named ${buildout:shared-parts} and
it was supposed to be a single path that slapos.recipe.cmmi was trying
to create. If we continued using this ${buildout:shared-parts} option
and passed a newline-separated list of paths, then slapos.recipe.cmmi
would just create one directory with \n in the name and will probably
fail.
Because generally, profiles using slapos.recipe.cmmi 0.9 were not
"safely" shared, we don't even provide compatibility with old option.

slapos.cfg.example

@@ -8,6 +8,12 @@ cert_file =  /etc/opt/slapos/ssl/computer.crt
 certificate_repository_path = /etc/opt/slapos/ssl/partition_pki
 software_root = /opt/slapgrid
 instance_root = /srv/slapgrid
+# A list of directories containing shared parts for slapos.recipe.cmmi.
+# Only the last one is used to install new parts, other ones can have
+# already installed parts.
+shared_part_list =
+  /opt/slapgrid/read_only_shared
+  /opt/slapgrid/shared
 
 [slapformat]
 # Replace by your network interface providing IPv6 if you don't use re6st

slapos/grid/SlapObject.py

@@ -39,6 +39,7 @@ import tarfile
 import tempfile
 import time
 from six.moves import xmlrpc_client as xmlrpclib, range
+from six.moves.configparser import ConfigParser
 
 from supervisor import xmlrpc
 
@@ -112,7 +113,8 @@ class Software(object):
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
                software_min_free_space=None,
-               buildout_debug=False,):
+               buildout_debug=False,
+               shared_part_list=''):
     """Initialisation of class parameters
     """
 
@@ -127,6 +129,7 @@ class Software(object):
     self.software_url_hash = md5digest(self.url)
     self.software_path = os.path.join(self.software_root,
                                       self.software_url_hash)
+    self.shared_part_list = shared_part_list
     self.buildout = buildout
     self.buildout_debug = buildout_debug
     self.logger = logger
@@ -260,7 +263,7 @@ class Software(object):
     try:
       buildout_cfg = os.path.join(self.software_path, 'buildout.cfg')
       if not os.path.exists(buildout_cfg):
-        self._create_buildout_profile(buildout_cfg, self.url)
+        self._create_buildout_profile(buildout_cfg, self.url, self.shared_part_list)
 
       additional_parameters = list(self._additional_buildout_parameters(extends_cache))
       additional_parameters.extend(['-c', buildout_cfg])
@@ -292,9 +295,16 @@ class Software(object):
       if f is not None:
         f.close()
 
-  def _create_buildout_profile(self, buildout_cfg, url):
+  def _create_buildout_profile(self, buildout_cfg, url, shared_part_list):
+    parser = ConfigParser()
+    parser.add_section('buildout')
+    parser.set('buildout', 'extends', url)
+    # Set shared-part-list for slapos.recipe.cmmi >= 0.10.
+    # slapos.recipe.cmmi was using shared-parts and expecting a single path.
+    # This is not supported.
+    parser.set('buildout', 'shared-part-list', shared_part_list)
     with open(buildout_cfg, 'w') as fout:
-      fout.write('[buildout]\nextends = ' + url + '\n')
+      parser.write(fout)
     self._set_ownership(buildout_cfg)
 
   def uploadSoftwareRelease(self, tarpath):

slapos/grid/slapgrid.py

@@ -241,6 +241,7 @@ def create_slapgrid_object(options, logger):
 
   return Slapgrid(software_root=op['software_root'],
                   instance_root=op['instance_root'],
+                  shared_part_list=op.get('shared_part_list', ''),
                   master_url=op['master_url'],
                   computer_id=op['computer_id'],
                   buildout=op.get('buildout'),
@@ -340,6 +341,7 @@ class Slapgrid(object):
                firewall_conf={},
                config=None,
                buildout_debug=False,
+               shared_part_list=''
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -405,6 +407,7 @@ class Slapgrid(object):
     self.firewall_conf = firewall_conf
     self.config = config
     self._manager_list = slapmanager.from_config(config)
+    self.shared_part_list = shared_part_list
 
   def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
@@ -562,7 +565,8 @@ stderr_logfile_backups=1
             shadir_ca_file=self.shadir_ca_file,
             shadir_cert_file=self.shadir_cert_file,
             shadir_key_file=self.shadir_key_file,
-            software_min_free_space=self.software_min_free_space)
+            software_min_free_space=self.software_min_free_space,
+            shared_part_list=self.shared_part_list)
 
         # call manager for every software release
         for manager in self._manager_list:

slapos/tests/test_slapgrid.py

@@ -113,6 +113,7 @@ class BasicMixin(object):
     self._tempdir = tempfile.mkdtemp()
     self.manager_list = []
     self.software_root = os.path.join(self._tempdir, 'software')
+    self.shared_parts_root = os.path.join(self._tempdir, 'shared')
     self.instance_root = os.path.join(self._tempdir, 'instance')
     if 'SLAPGRID_INSTANCE_ROOT' in os.environ:
       del os.environ['SLAPGRID_INSTANCE_ROOT']
@@ -134,7 +135,8 @@ class BasicMixin(object):
                                   self.computer_id,
                                   self.buildout,
                                   develop=develop,
-                                  logger=logging.getLogger())
+                                  logger=logging.getLogger(),
+                                  shared_part_list=self.shared_parts_root)
     self.grid._manager_list = self.manager_list
     # monkey patch buildout bootstrap
 
@@ -1867,6 +1869,16 @@ echo %s; echo %s; exit 42""" % (line1, line2))
       self.assertIn(line2, software.error_log)
       self.assertIn('Failed to run buildout', software.error_log)
 
+  def test_software_install_generate_buildout_cfg_with_shared_part_list(self):
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    with httmock.HTTMock(computer.request_handler):
+      software = computer.software_list[0]
+      # examine the genrated buildout
+      software.setBuildout("""#!/bin/sh
+        cat buildout.cfg; exit 1""")
+      self.launchSlapgridSoftware()
+    self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log)
+
 
 class SlapgridInitialization(unittest.TestCase):
   """