Support shared parts in slaprunner

 * improve test coverage of instance.
 * version up ssh (for no special reason)
 * other minor fixes

/reviewed-on nexedi/slapos!629

component/openssh/buildout.cfg

@@ -15,12 +15,12 @@ parts =
 
 [openssh]
 recipe = slapos.recipe.cmmi
-md5sum = b2db2a83caf66a208bb78d6d287cdaa3
-url = http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.4p1.tar.gz
+md5sum = 68ba883aff6958297432e5877e9a0fe2
+url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
 patch-binary = ${patch:location}/bin/patch
-#patch-options = -p1
+patch-options = -p1
 patches =
-  ${:_profile_base_location_}/no_create_privsep_path.patch#d5b61a2442fffa457cebe4ad1dc68f4e
+  ${:_profile_base_location_}/no_create_privsep_path.patch#f341dc11d73df6f43c7ae1fa47b8c003
 environment = 
   CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib

component/openssh/no_create_privsep_path.patch

---- Makefile.in	2016-07-28 00:54:27.000000000 +0200
-+++ Makefile.in	2016-08-19 13:02:30.227177750 +0200
-@@ -304,7 +304,6 @@
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)5
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)8
- 	$(srcdir)/mkinstalldirs $(DESTDIR)$(libexecdir)
--	(umask 022 ; $(srcdir)/mkinstalldirs $(DESTDIR)$(PRIVSEP_PATH))
+From 46cf5eba19cf52ffae48ea95f07a36b4c107ebd8 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
+Date: Sun, 20 May 2018 20:47:16 +0900
+Subject: [PATCH] Do not create PRIVSEP_PATH
+
+For SlapOS, this would need to be in instance, but instance paths are
+not known at software compilation time.  Because we don't do privileges
+separation, just disable creation of this directory.
+---
+ Makefile.in | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/Makefile.in b/Makefile.in
+index 04e1c8e5..9bd5d01b 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -329,7 +329,6 @@ install-files:
+ 	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)5
+ 	$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)8
+ 	$(MKDIR_P) $(DESTDIR)$(libexecdir)
+-	$(MKDIR_P) -m 0755 $(DESTDIR)$(PRIVSEP_PATH)
  	$(INSTALL) -m 0755 $(STRIP_OPT) ssh$(EXEEXT) $(DESTDIR)$(bindir)/ssh$(EXEEXT)
  	$(INSTALL) -m 0755 $(STRIP_OPT) scp$(EXEEXT) $(DESTDIR)$(bindir)/scp$(EXEEXT)
  	$(INSTALL) -m 0755 $(STRIP_OPT) ssh-add$(EXEEXT) $(DESTDIR)$(bindir)/ssh-add$(EXEEXT)
+-- 
+2.11.0
+

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = c44a7481bb85e3258128afe3fcf23f44
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 58a6f21021279ae52c91813f61cdbdd7
+md5sum = 19475112d4dccee1263798c67fd2351b
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -50,7 +50,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = 035e027e9cb9bbdca0509ac895fc4696
+md5sum = da113b3e3e7bac9cc215fede7c4911a5
 
 [template-parameters]
 filename = parameters.xml.in
@@ -58,7 +58,7 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
 
 [template-bash-profile]
 filename = template/bash_profile.in
-md5sum = 37eea89042a58127c85e6b3886260e59
+md5sum = 7645048216fcf957f7773534cd0408dc
 
 [template-supervisord]
 filename = template/supervisord.conf.in
@@ -78,4 +78,8 @@ md5sum = 2451072826a9ad9425d62c9e9c7f6284
 
 [template-slapuser-script]
 filename = template/slapos-slapuser-script.in
-md5sum = becafae59ab9973724e0edb238f4cb7a
+md5sum = 75aab99c995ca841f93fc77fc9116c37
+
+[template-buildout-shared-part-list]
+filename = template/buildout-shared-part-list.in
+md5sum = 3203c9ad0b30d3ee39a809a067efff8d
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -120,6 +120,10 @@ recipe = slapos.cookbook:mkdirectory
 home = $${buildout:directory}
 etc = $${:home}/etc/
 var = $${:home}/var/
+# This srv path has an extra slash, which will cause runnerdirectory:home
+# to be .../srv//runner/.. but for compatibility reasons we don't fix this,
+# because this is the path that will be used as software URL installed by
+# webrunner and would cause software release hash to become different.
 srv = $${:home}/srv/
 bin = $${:home}/bin/
 tmp = $${:home}/tmp/
@@ -146,6 +150,7 @@ project = $${:home}/project
 public = $${:home}/public
 software-root = {{ slapparameter_dict.get('software-root', '$${:home}/software') }}
 instance-root = $${:home}/instance
+shared-root = $${:home}/shared
 project-test = $${:test}/project
 software-test = $${:test}/software
 instance-test = $${:test}/instance
@@ -165,6 +170,8 @@ working-directory = $${runnerdirectory:home}
 project-directory = $${runnerdirectory:project}
 instance_root = $${runnerdirectory:instance-root}
 software_root = $${runnerdirectory:software-root}
+shared_root = $${runnerdirectory:shared-root}
+buildout-shared-part-list-dump = ${template-buildout-shared-part-list:output}
 pidfile-software = $${directory:run}/slapgrid-cp.pid
 pidfile-instance = $${directory:run}/slapgrid-sr.pid
 ssh_client = ${openssh:location}/bin/ssh
@@ -257,7 +264,7 @@ template = inline:
   PasswordAuthentication no
   PubkeyAuthentication yes
   AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
-  ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then $${shell-environment:shell} -l; else SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
+  ForceCommand cd $${directory:home}; if [ -z "$SSH_ORIGINAL_COMMAND" ]; then HOME=$${directory:home} $${shell-environment:shell} -l; else HOME=$${directory:home} SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
   Subsystem sftp ${openssh:location}/libexec/sftp-server
 
 [runner-sshd-raw-server]
@@ -326,6 +333,27 @@ recipe = slapos.cookbook:dropbear.add_authorized_key
 home = $${buildout:directory}
 key = $${slap-parameter:user-authorized-key}
 
+[runner-sshkeys-publickey-fingerprint-cmd]
+recipe = plone.recipe.command
+command = bash -o pipefail -c "$${runner-sshkeys-authority:keygen-binary} -lf $${runner-sshkeys-sshd:public-key} | cut  -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
+
+[runner-sshkeys-publickey-fingerprint-shelloutput]
+recipe = collective.recipe.shelloutput
+# XXX because collective.recipe.shelloutput ignore errors, we run the same
+# command in a plone.recipe.command so that if fails if something goes wrong.
+commands =
+  fingerprint = $${runner-sshkeys-publickey-fingerprint-cmd:command}
+
+[runner-sshkeys-publickey-fingerprint]
+# fingerprint for ssh url, see
+# https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
+# https://winscp.net/eng/docs/session_url#hostkey
+
+# format is host-key-alg-fingerprint, but we know that
+# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg
+# we just use use rsa.
+fingerprint = ssh-rsa-$${runner-sshkeys-publickey-fingerprint-shelloutput:fingerprint}
+
 #---------------------------
 #--
 #-- Set nginx frontend
@@ -612,6 +640,7 @@ backend-url = $${slaprunner:access-url}
 init-user = $${runner-htpasswd:user}
 init-password = $${runner-htpasswd:password}
 ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
+ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshkeys-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
 git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
 git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
 monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
@@ -681,6 +710,8 @@ rendered = $${slaprunner:slapos.cfg}
 mode = 700
 context =
   section slaprunner slaprunner
+  import codecs codecs
+  raw buildout_shared_part_list_dump ${template-buildout-shared-part-list:output}
 
 [slapos-test-cfg]
 recipe = slapos.recipe.template:jinja2
@@ -773,8 +804,13 @@ rendered = $${buildout:directory}/.bash_profile
 context =
     raw path $${shell-environment:path}
     raw shell $${shell-environment:shell}
+    key terminfo terminfo:location
     key instance_name slap-parameter:instance-name
     key workdir runnerdirectory:home
+    key home directory:home
+
+[terminfo]
+location = ${ncurses:location}/share/terminfo/
 
 #---------------------------
 #--

software/slaprunner/software.cfg

@@ -27,6 +27,10 @@ extends =
   ../../stack/logrotate/buildout.cfg
   ../../stack/monitor/buildout.cfg
 
+# make sure shared-part-list is available, even for old versions
+# of slapos who do not set that.
+shared-part-list =
+
 # stacks are listed from most generic to most specific,
 # to avoid versioning issues
 
@@ -131,6 +135,10 @@ filename = resilient_software_release_information.py.in
 < = template-download-base
 filename = slapos-slapuser-script.in
 
+[template-buildout-shared-part-list]
+< = template-base
+output = ${buildout:directory}/buildout-shared-part-list
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -140,6 +148,7 @@ eggs =
   erp5.util
   lock-file
   plone.recipe.command
+  collective.recipe.shelloutput
   slapos.recipe.build
   slapos.toolbox[flask_auth]
   gunicorn==19.7.1
@@ -162,6 +171,7 @@ gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
+collective.recipe.shelloutput = 0.1
 collective.recipe.environment = 0.2.0
 smmap = 0.9.0
 lockfile = 0.12.2

software/slaprunner/template/bash_profile.in

@@ -3,6 +3,7 @@
 # If you want to load your custom bash configuration, please use a .bashrc file
 
 cd {{ workdir }}
+export HOME={{- home }}
 export PATH={{- path }}
 export SHELL={{- shell }}
 {%- if instance_name %}
@@ -10,6 +11,8 @@ export PROMPT_COMMAND='echo -en "\033]0;{{-instance_name}}\a"'
 {% endif %}
 export PS1="$ "
 
+export TERMINFO={{- terminfo }}
+
 if [ -f "$HOME/.bashrc" ] ; then
   source $HOME/.bashrc
 fi
@@ -32,5 +35,5 @@ alias la='ls -A'
 alias l='ls -CF'
 
 echo "Welcome to SlapOS slaprunner shell"
-echo 
+echo
 

software/slaprunner/template/buildout-shared-part-list.in

+${buildout:shared-part-list}
\ No newline at end of file

software/slaprunner/template/slapos-slapuser-script.in

 #!/bin/sh
 # run slapos command inside slaprunner with appropriate config
 
-SLAPOS_CONFIGURATION={{ config_location }}   \
-SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION   \
+export SLAPOS_CONFIGURATION=${SLAPOS_CONFIGURATION-"{{ config_location }}"}
+export SLAPOS_CLIENT_CONFIGURATION=${SLAPOS_CLIENT_CONFIGURATION-$SLAPOS_CONFIGURATION}
 exec {{ slapos_python_file_location }} "$@"

software/slaprunner/template/slapos.cfg.in

 [slapos]
 software_root = {{ slaprunner['software_root'] }}
 instance_root = {{ slaprunner['instance_root'] }}
+shared_part_list =
+  {#- buildout_shared_part_list_dump is ${buildout:shared-part-list} rendered as a
+  template during software step.
+  Because it can contain new lines, it's not possible to use it directly when generating
+  buildout config files from buildout, because the newlines don't get indented and cause
+  the instance buildout to be invalid ini file.
+  So we had to dump it in a simple text file and now we parse again that text file. -#}
+  {%- for line in codecs.open(buildout_shared_part_list_dump).readlines() %}
+  {{   line.strip() }}
+  {%- endfor %}
+  {{ slaprunner['shared_root'] }}
 master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
 computer_id = slaprunner
 maximal_delay = 0

software/slaprunner/test/setup.py

@@ -47,6 +47,9 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
+          'paramiko',
+          'six',
+          'requests',
       ],
       zip_safe=True,
       test_suite='test',

software/slaprunner/test/test.py

@@ -26,6 +26,18 @@
 ##############################################################################
 
 import os
+import unittest
+import paramiko
+import contextlib
+import base64
+import hashlib
+import subprocess
+
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import quote
+from six.moves.urllib.parse import urljoin
+from six.moves.configparser import ConfigParser
+import requests
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -40,6 +52,193 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
   __partition_reference__ = 's'
 
 
+class TestWeb(SlaprunnerTestCase):
+  def test_slaprunner(self):
+    # slaprunner main interface is password protected
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = parameter_dict['url']
+    resp = requests.get(url, verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    resp = requests.get(
+        url,
+        verify=False,
+        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertIn('SlapOS', resp.text)
+
+  def test_shellinabox(self):
+    # shellinabox exists at /shellinabox and is password protected
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = urljoin(parameter_dict['url'], '/shellinabox')
+    resp = requests.get(url, verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    resp = requests.get(
+        url,
+        verify=False,
+        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertIn('ShellInABox', resp.text)
+    self.assertNotIn('SlapOS', resp.text)
+
+  def test_public_url(self):
+    # ~/srv/runner/public/ is served over http
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    public_url = parameter_dict['public-url']
+
+    hello_file = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'runner',
+        'public',
+        'hello.html')
+    self.addCleanup(os.remove, hello_file)
+    with open(hello_file, 'w') as f:
+      f.write('<b>Hello</b>')
+
+    index = requests.get(public_url, verify=False)
+    self.assertEqual(requests.codes.ok, index.status_code)
+    self.assertIn('hello.html', index.text)
+
+    hello = requests.get(urljoin(public_url, 'hello.html'), verify=False)
+    self.assertEqual(requests.codes.ok, hello.status_code)
+    self.assertIn('<b>Hello</b>', hello.text)
+
+  # git seems broken, these are 404 now...
+  @unittest.expectedFailure
+  def test_git_private(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = parameter_dict['git-private']
+    resp = requests.get(url, verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    resp = requests.get(
+        url,
+        verify=False,
+        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+  @unittest.expectedFailure
+  def test_git_public(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    url = parameter_dict['git-public']
+    resp = requests.get(url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+
+class TestSSH(SlaprunnerTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.ssh_key = paramiko.RSAKey.generate(1024)
+    return {
+        'user-authorized-key': 'ssh-rsa {}'.format(cls.ssh_key.get_base64())
+    }
+
+  def test_connect(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    ssh_url = parameter_dict['ssh-url']
+    parsed = urlparse(ssh_url)
+    self.assertEqual('ssh', parsed.scheme)
+
+    # username contain a fingerprint (only, so we simplify the parsing)
+    #
+    # relevant parts of the grammar defined in
+    # https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html
+    #
+    #   ssh-info      =  [ userinfo ] [";" c-param *("," c-param)]
+    #   c-param       =  paramname "=" paramvalue
+    ssh_info = parsed.username
+    username, fingerprint_from_url = ssh_info.split(';fingerprint=')
+    client = paramiko.SSHClient()
+
+    self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), '')
+    fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
+
+    class KeyPolicy(object):
+      """Accept server key and keep it in self.key for inspection
+      """
+      def missing_host_key(self, client, hostname, key):
+        self.key = key
+
+    key_policy = KeyPolicy()
+    client.set_missing_host_key_policy(key_policy)
+
+    with contextlib.closing(client):
+      client.connect(
+          username=username,
+          hostname=parsed.hostname,
+          port=parsed.port,
+          pkey=self.ssh_key,
+      )
+      # Check fingerprint from server matches the published one.
+      # Paramiko does not allow to get the fingerprint as SHA256 easily yet
+      # https://github.com/paramiko/paramiko/pull/1103
+      self.assertEqual(
+          fingerprint_from_url,
+          quote(
+              # base64 encoded fingerprint adds an extra = at the end
+              base64.b64encode(
+                  hashlib.sha256(key_policy.key.asbytes()).digest())[:-1],
+              # also encode /
+              safe=''))
+
+      # Check shell is usable
+      channel = client.invoke_shell()
+      channel.settimeout(30)
+      received = ''
+      while True:
+        r = channel.recv(1024)
+        self.logger.debug("received >%s<", r)
+        if not r:
+          break
+        received += r
+        if 'slaprunner shell' in received:
+          break
+      self.assertIn("Welcome to SlapOS slaprunner shell", received)
+
+      # simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
+      self.assertEqual(
+          self.computer_partition_root_path,
+          client.exec_command("pwd")[1].read(1000).strip())
+
+
+class TestSlapOS(SlaprunnerTestCase):
+  def test_slapos_command(self):
+    # in ~/bin/slapos there is a wrapper setting configuration to use slapos from
+    # the web runner.
+    proxy_show_output = subprocess.check_output(
+        (
+            os.path.join(self.computer_partition_root_path, 'bin', 'slapos'),
+            'proxy',
+            'show',
+        ),
+        env={})
+    self.assertIn('slaprunner', proxy_show_output)
+
+  def test_shared_part_list(self):
+    # this slapos used shared_part_list
+    cfg_parser = ConfigParser()
+    with open(os.path.join(self.computer_partition_root_path,
+                           'etc',
+                           'slapos.cfg')) as f:
+      cfg_parser.readfp(f)
+    shared_part_list = cfg_parser.get('slapos', 'shared_part_list').splitlines()
+
+    # web runner own shared parts. Note that there is intentionnaly a double
+    # slash in this path, because slaprunner has double slash in paths since
+    # early releases, including for the path of slapos repository that will be
+    # used to develop and install software. If we fix this duplication, then
+    # the URL of installed software will be different and it will get a different
+    # hash and be reinstalled. To prevent this, we keep that // between srv and runner.
+    self.assertEqual(
+        '{}/srv//runner//shared'.format(self.computer_partition_root_path.rstrip('/')),
+        shared_part_list[-1])
+
+    # shared parts from outer slapos
+    outer_shared_part_list = os.getenv('SLAPOS_TEST_SHARED_PART_LIST',
+                                       '').split(os.pathsep)
+    for outer_shared_part in outer_shared_part_list:
+      self.assertIn(outer_shared_part, shared_part_list)
+
+
 class ServicesTestCase(SlaprunnerTestCase):
   def test_hashes(self):
     hash_files = [