From 5348eedcca35e372e540d9a5d795877ba03dbb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9rome=20Perrin?= Date: Sat, 21 Sep 2019 07:06:30 +0200 Subject: [PATCH] grid: new shared_part_list config file option This new option for slapos.cfg allow to define a list of paths to use for shared parts feature of slapos.recipe.cmmmi. This will be injected in the buildout profile used to install software, as ${buildout:shared-part-list} which is the option slapos.recipe.cmmi use since 0.10. Note that the option changed in slapos.recipe.cmmi because this was incompatible. In 0.9, the option was named ${buildout:shared-parts} and it was supposed to be a single path that slapos.recipe.cmmi was trying to create. If we continued using this ${buildout:shared-parts} option and passed a newline-separated list of paths, then slapos.recipe.cmmi would just create one directory with \n in the name and will probably fail. Because generally, profiles using slapos.recipe.cmmi 0.9 were not "safely" shared, we don't even provide compatibility with old option. --- slapos.cfg.example | 6 ++++++ slapos/grid/SlapObject.py | 18 ++++++++++++++---- slapos/grid/slapgrid.py | 6 +++++- slapos/tests/test_slapgrid.py | 14 +++++++++++++- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/slapos.cfg.example b/slapos.cfg.example index b87826640..d8fcc5df1 100644 --- a/slapos.cfg.example +++ b/slapos.cfg.example @@ -8,6 +8,12 @@ cert_file = /etc/opt/slapos/ssl/computer.crt certificate_repository_path = /etc/opt/slapos/ssl/partition_pki software_root = /opt/slapgrid instance_root = /srv/slapgrid +# A list of directories containing shared parts for slapos.recipe.cmmi. +# Only the last one is used to install new parts, other ones can have +# already installed parts. +shared_part_list = + /opt/slapgrid/read_only_shared + /opt/slapgrid/shared [slapformat] # Replace by your network interface providing IPv6 if you don't use re6st diff --git a/slapos/grid/SlapObject.py b/slapos/grid/SlapObject.py index d9cccd42a..aaf039870 100644 --- a/slapos/grid/SlapObject.py +++ b/slapos/grid/SlapObject.py @@ -39,6 +39,7 @@ import tarfile import tempfile import time from six.moves import xmlrpc_client as xmlrpclib, range +from six.moves.configparser import ConfigParser from supervisor import xmlrpc @@ -112,7 +113,8 @@ class Software(object): download_from_binary_cache_url_blacklist=None, upload_to_binary_cache_url_blacklist=None, software_min_free_space=None, - buildout_debug=False,): + buildout_debug=False, + shared_part_list=''): """Initialisation of class parameters """ @@ -127,6 +129,7 @@ class Software(object): self.software_url_hash = md5digest(self.url) self.software_path = os.path.join(self.software_root, self.software_url_hash) + self.shared_part_list = shared_part_list self.buildout = buildout self.buildout_debug = buildout_debug self.logger = logger @@ -260,7 +263,7 @@ class Software(object): try: buildout_cfg = os.path.join(self.software_path, 'buildout.cfg') if not os.path.exists(buildout_cfg): - self._create_buildout_profile(buildout_cfg, self.url) + self._create_buildout_profile(buildout_cfg, self.url, self.shared_part_list) additional_parameters = list(self._additional_buildout_parameters(extends_cache)) additional_parameters.extend(['-c', buildout_cfg]) @@ -292,9 +295,16 @@ class Software(object): if f is not None: f.close() - def _create_buildout_profile(self, buildout_cfg, url): + def _create_buildout_profile(self, buildout_cfg, url, shared_part_list): + parser = ConfigParser() + parser.add_section('buildout') + parser.set('buildout', 'extends', url) + # Set shared-part-list for slapos.recipe.cmmi >= 0.10. + # slapos.recipe.cmmi was using shared-parts and expecting a single path. + # This is not supported. + parser.set('buildout', 'shared-part-list', shared_part_list) with open(buildout_cfg, 'w') as fout: - fout.write('[buildout]\nextends = ' + url + '\n') + parser.write(fout) self._set_ownership(buildout_cfg) def uploadSoftwareRelease(self, tarpath): diff --git a/slapos/grid/slapgrid.py b/slapos/grid/slapgrid.py index 21ee86780..1416750d4 100644 --- a/slapos/grid/slapgrid.py +++ b/slapos/grid/slapgrid.py @@ -241,6 +241,7 @@ def create_slapgrid_object(options, logger): return Slapgrid(software_root=op['software_root'], instance_root=op['instance_root'], + shared_part_list=op.get('shared_part_list', ''), master_url=op['master_url'], computer_id=op['computer_id'], buildout=op.get('buildout'), @@ -340,6 +341,7 @@ class Slapgrid(object): firewall_conf={}, config=None, buildout_debug=False, + shared_part_list='' ): """Makes easy initialisation of class parameters""" # Parses arguments @@ -405,6 +407,7 @@ class Slapgrid(object): self.firewall_conf = firewall_conf self.config = config self._manager_list = slapmanager.from_config(config) + self.shared_part_list = shared_part_list def _getWatchdogLine(self): invocation_list = [WATCHDOG_PATH] @@ -562,7 +565,8 @@ stderr_logfile_backups=1 shadir_ca_file=self.shadir_ca_file, shadir_cert_file=self.shadir_cert_file, shadir_key_file=self.shadir_key_file, - software_min_free_space=self.software_min_free_space) + software_min_free_space=self.software_min_free_space, + shared_part_list=self.shared_part_list) # call manager for every software release for manager in self._manager_list: diff --git a/slapos/tests/test_slapgrid.py b/slapos/tests/test_slapgrid.py index 481c60cdc..c77e3c26a 100644 --- a/slapos/tests/test_slapgrid.py +++ b/slapos/tests/test_slapgrid.py @@ -113,6 +113,7 @@ class BasicMixin(object): self._tempdir = tempfile.mkdtemp() self.manager_list = [] self.software_root = os.path.join(self._tempdir, 'software') + self.shared_parts_root = os.path.join(self._tempdir, 'shared') self.instance_root = os.path.join(self._tempdir, 'instance') if 'SLAPGRID_INSTANCE_ROOT' in os.environ: del os.environ['SLAPGRID_INSTANCE_ROOT'] @@ -134,7 +135,8 @@ class BasicMixin(object): self.computer_id, self.buildout, develop=develop, - logger=logging.getLogger()) + logger=logging.getLogger(), + shared_part_list=self.shared_parts_root) self.grid._manager_list = self.manager_list # monkey patch buildout bootstrap @@ -1867,6 +1869,16 @@ echo %s; echo %s; exit 42""" % (line1, line2)) self.assertIn(line2, software.error_log) self.assertIn('Failed to run buildout', software.error_log) + def test_software_install_generate_buildout_cfg_with_shared_part_list(self): + computer = ComputerForTest(self.software_root, self.instance_root, 1, 1) + with httmock.HTTMock(computer.request_handler): + software = computer.software_list[0] + # examine the genrated buildout + software.setBuildout("""#!/bin/sh + cat buildout.cfg; exit 1""") + self.launchSlapgridSoftware() + self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log) + class SlapgridInitialization(unittest.TestCase): """ -- 2.30.9