slaprunner: support shared parts

If slaprunner itself was installed on a slapos which had shared parts
enabled, these shared parts will also be used while installing softwares
inside the slaprunner.

Because ${buildout:shared-part-list} is a multi line property, it was
not possible to generate .cfg files (which needs special handling of
multi-line strings) directly from software buildout (where jinja
templates are not yet available), so we use a trick of generating a
plain text file in software and parsing it during instance.

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = c44a7481bb85e3258128afe3fcf23f44
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 0b71f354260ab48615ed62c1c8ffc9d4
+md5sum = 5d4b60523b21c777fdeca438c200320d
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -50,7 +50,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = 035e027e9cb9bbdca0509ac895fc4696
+md5sum = da113b3e3e7bac9cc215fede7c4911a5
 
 [template-parameters]
 filename = parameters.xml.in
@@ -79,3 +79,7 @@ md5sum = 2451072826a9ad9425d62c9e9c7f6284
 [template-slapuser-script]
 filename = template/slapos-slapuser-script.in
 md5sum = 2e5fdd6e8e07429f83a38b148b2f1cb0
+
+[template-buildout-shared-part-list]
+filename = template/buildout-shared-part-list.in
+md5sum = 3203c9ad0b30d3ee39a809a067efff8d
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -146,6 +146,7 @@ project = $${:home}/project
 public = $${:home}/public
 software-root = {{ slapparameter_dict.get('software-root', '$${:home}/software') }}
 instance-root = $${:home}/instance
+shared-root = $${:home}/shared
 project-test = $${:test}/project
 software-test = $${:test}/software
 instance-test = $${:test}/instance
@@ -165,6 +166,8 @@ working-directory = $${runnerdirectory:home}
 project-directory = $${runnerdirectory:project}
 instance_root = $${runnerdirectory:instance-root}
 software_root = $${runnerdirectory:software-root}
+shared_root = $${runnerdirectory:shared-root}
+buildout-shared-part-list-dump = ${template-buildout-shared-part-list:output}
 pidfile-software = $${directory:run}/slapgrid-cp.pid
 pidfile-instance = $${directory:run}/slapgrid-sr.pid
 ssh_client = ${openssh:location}/bin/ssh
@@ -703,6 +706,8 @@ rendered = $${slaprunner:slapos.cfg}
 mode = 700
 context =
   section slaprunner slaprunner
+  import codecs codecs
+  raw buildout_shared_part_list_dump ${template-buildout-shared-part-list:output}
 
 [slapos-test-cfg]
 recipe = slapos.recipe.template:jinja2

software/slaprunner/software.cfg

@@ -27,6 +27,10 @@ extends =
   ../../stack/logrotate/buildout.cfg
   ../../stack/monitor/buildout.cfg
 
+# make sure shared-part-list is available, even for old versions
+# of slapos who do not set it.
+shared-part-list =
+
 # stacks are listed from most generic to most specific,
 # to avoid versioning issues
 
@@ -131,6 +135,10 @@ filename = resilient_software_release_information.py.in
 < = template-download-base
 filename = slapos-slapuser-script.in
 
+[template-buildout-shared-part-list]
+< = template-base
+output = ${buildout:directory}/buildout-shared-part-list
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =

software/slaprunner/template/buildout-shared-part-list.in

+${buildout:shared-part-list}
\ No newline at end of file

software/slaprunner/template/slapos.cfg.in

 [slapos]
 software_root = {{ slaprunner['software_root'] }}
 instance_root = {{ slaprunner['instance_root'] }}
+shared_part_list =
+  {#- buildout_shared_part_list_dump is ${buildout:shared-part-list} rendered as a
+  template during software step.
+  Because it can contain new lines, it's not possible to use it directly when generating
+  buildout config files from buildout, because the newlines don't get indented and cause
+  the instance buildout to be invalid ini file.
+  So we had to dump it in a simple text file and now we parse again that text file. -#}
+  {%- for line in codecs.open(buildout_shared_part_list_dump).readlines() %}
+  {{   line.strip() }}
+  {%- endfor %}
+  {{ slaprunner['shared_root'] }}
 master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
 computer_id = slaprunner
 maximal_delay = 0

software/slaprunner/test/test.py

@@ -30,8 +30,10 @@ import paramiko
 import contextlib
 import base64
 import hashlib
+import subprocess
 from six.moves.urllib.parse import urlparse
 from six.moves.urllib.parse import quote
+from six.moves.configparser import ConfigParser
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -122,6 +124,45 @@ class TestSSH(SlaprunnerTestCase):
           client.exec_command("pwd")[1].read(1000).strip())
 
 
+class TestSlapOS(SlaprunnerTestCase):
+  def test_slapos_command(self):
+    # in ~/bin/slapos there is a wrapper setting configuration to use slapos from
+    # the web runner.
+    proxy_show_output = subprocess.check_output(
+        (
+            os.path.join(self.computer_partition_root_path, 'bin', 'slapos'),
+            'proxy',
+            'show',
+        ),
+        env={})
+    self.assertIn('slaprunner', proxy_show_output)
+
+  def test_shared_part_list(self):
+    # this slapos used shared_part_list
+    cfg_parser = ConfigParser()
+    with open(os.path.join(self.computer_partition_root_path,
+                           'etc',
+                           'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    shared_part_list = cfg_parser.get('slapos', 'shared_part_list').splitlines()
+
+    # web runner own shared parts. Note that there is intentionnaly a double
+    # slash in this path, because slaprunner has double slash in paths since
+    # early releases, including for the path of slapos repository that will be
+    # used to develop and install software. If we fix this duplication, then
+    # the URL of installed software will be different and it will get a different
+    # hash and be reinstalled. To prevent this, we keep that // between srv and runner.
+    self.assertEqual(
+        '{}/srv//runner//shared'.format(self.computer_partition_root_path.rstrip('/')),
+        shared_part_list[-1])
+
+    # shared parts from outer slapos
+    outer_shared_part_list = os.getenv('SLAPOS_TEST_SHARED_PART_LIST',
+                                       '').split(os.pathsep)
+    for outer_shared_part in outer_shared_part_list:
+      self.assertIn(outer_shared_part, shared_part_list)
+
+
 class ServicesTestCase(SlaprunnerTestCase):
   def test_hashes(self):
     hash_files = [