Commit 5616562f authored by Jérome Perrin's avatar Jérome Perrin

Support shared parts in slaprunner

 * improve test coverage of instance.
 * version up ssh (for no special reason)
 * other minor fixes

/reviewed-on nexedi/slapos!629
parents 846d5b43 26741e83
Pipeline #6464 failed with stage
in 0 seconds
......@@ -15,12 +15,12 @@ parts =
[openssh]
recipe = slapos.recipe.cmmi
md5sum = b2db2a83caf66a208bb78d6d287cdaa3
url = http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.4p1.tar.gz
md5sum = 68ba883aff6958297432e5877e9a0fe2
url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
patch-binary = ${patch:location}/bin/patch
#patch-options = -p1
patch-options = -p1
patches =
${:_profile_base_location_}/no_create_privsep_path.patch#d5b61a2442fffa457cebe4ad1dc68f4e
${:_profile_base_location_}/no_create_privsep_path.patch#f341dc11d73df6f43c7ae1fa47b8c003
environment =
CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
......
--- Makefile.in 2016-07-28 00:54:27.000000000 +0200
+++ Makefile.in 2016-08-19 13:02:30.227177750 +0200
@@ -304,7 +304,6 @@
$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)5
$(srcdir)/mkinstalldirs $(DESTDIR)$(mandir)/$(mansubdir)8
$(srcdir)/mkinstalldirs $(DESTDIR)$(libexecdir)
- (umask 022 ; $(srcdir)/mkinstalldirs $(DESTDIR)$(PRIVSEP_PATH))
From 46cf5eba19cf52ffae48ea95f07a36b4c107ebd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Sun, 20 May 2018 20:47:16 +0900
Subject: [PATCH] Do not create PRIVSEP_PATH
For SlapOS, this would need to be in instance, but instance paths are
not known at software compilation time. Because we don't do privileges
separation, just disable creation of this directory.
---
Makefile.in | 1 -
1 file changed, 1 deletion(-)
diff --git a/Makefile.in b/Makefile.in
index 04e1c8e5..9bd5d01b 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -329,7 +329,6 @@ install-files:
$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)5
$(MKDIR_P) $(DESTDIR)$(mandir)/$(mansubdir)8
$(MKDIR_P) $(DESTDIR)$(libexecdir)
- $(MKDIR_P) -m 0755 $(DESTDIR)$(PRIVSEP_PATH)
$(INSTALL) -m 0755 $(STRIP_OPT) ssh$(EXEEXT) $(DESTDIR)$(bindir)/ssh$(EXEEXT)
$(INSTALL) -m 0755 $(STRIP_OPT) scp$(EXEEXT) $(DESTDIR)$(bindir)/scp$(EXEEXT)
$(INSTALL) -m 0755 $(STRIP_OPT) ssh-add$(EXEEXT) $(DESTDIR)$(bindir)/ssh-add$(EXEEXT)
--
2.11.0
......@@ -18,7 +18,7 @@ md5sum = c44a7481bb85e3258128afe3fcf23f44
[template-runner]
filename = instance-runner.cfg
md5sum = 58a6f21021279ae52c91813f61cdbdd7
md5sum = 19475112d4dccee1263798c67fd2351b
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......@@ -50,7 +50,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
[template-slapos-cfg]
filename = template/slapos.cfg.in
md5sum = 035e027e9cb9bbdca0509ac895fc4696
md5sum = da113b3e3e7bac9cc215fede7c4911a5
[template-parameters]
filename = parameters.xml.in
......@@ -58,7 +58,7 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
[template-bash-profile]
filename = template/bash_profile.in
md5sum = 37eea89042a58127c85e6b3886260e59
md5sum = 7645048216fcf957f7773534cd0408dc
[template-supervisord]
filename = template/supervisord.conf.in
......@@ -78,4 +78,8 @@ md5sum = 2451072826a9ad9425d62c9e9c7f6284
[template-slapuser-script]
filename = template/slapos-slapuser-script.in
md5sum = becafae59ab9973724e0edb238f4cb7a
md5sum = 75aab99c995ca841f93fc77fc9116c37
[template-buildout-shared-part-list]
filename = template/buildout-shared-part-list.in
md5sum = 3203c9ad0b30d3ee39a809a067efff8d
\ No newline at end of file
......@@ -120,6 +120,10 @@ recipe = slapos.cookbook:mkdirectory
home = $${buildout:directory}
etc = $${:home}/etc/
var = $${:home}/var/
# This srv path has an extra slash, which will cause runnerdirectory:home
# to be .../srv//runner/.. but for compatibility reasons we don't fix this,
# because this is the path that will be used as software URL installed by
# webrunner and would cause software release hash to become different.
srv = $${:home}/srv/
bin = $${:home}/bin/
tmp = $${:home}/tmp/
......@@ -146,6 +150,7 @@ project = $${:home}/project
public = $${:home}/public
software-root = {{ slapparameter_dict.get('software-root', '$${:home}/software') }}
instance-root = $${:home}/instance
shared-root = $${:home}/shared
project-test = $${:test}/project
software-test = $${:test}/software
instance-test = $${:test}/instance
......@@ -165,6 +170,8 @@ working-directory = $${runnerdirectory:home}
project-directory = $${runnerdirectory:project}
instance_root = $${runnerdirectory:instance-root}
software_root = $${runnerdirectory:software-root}
shared_root = $${runnerdirectory:shared-root}
buildout-shared-part-list-dump = ${template-buildout-shared-part-list:output}
pidfile-software = $${directory:run}/slapgrid-cp.pid
pidfile-instance = $${directory:run}/slapgrid-sr.pid
ssh_client = ${openssh:location}/bin/ssh
......@@ -257,7 +264,7 @@ template = inline:
PasswordAuthentication no
PubkeyAuthentication yes
AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
ForceCommand if [ -z "$SSH_ORIGINAL_COMMAND" ]; then $${shell-environment:shell} -l; else SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
ForceCommand cd $${directory:home}; if [ -z "$SSH_ORIGINAL_COMMAND" ]; then HOME=$${directory:home} $${shell-environment:shell} -l; else HOME=$${directory:home} SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
Subsystem sftp ${openssh:location}/libexec/sftp-server
[runner-sshd-raw-server]
......@@ -326,6 +333,27 @@ recipe = slapos.cookbook:dropbear.add_authorized_key
home = $${buildout:directory}
key = $${slap-parameter:user-authorized-key}
[runner-sshkeys-publickey-fingerprint-cmd]
recipe = plone.recipe.command
command = bash -o pipefail -c "$${runner-sshkeys-authority:keygen-binary} -lf $${runner-sshkeys-sshd:public-key} | cut -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
[runner-sshkeys-publickey-fingerprint-shelloutput]
recipe = collective.recipe.shelloutput
# XXX because collective.recipe.shelloutput ignore errors, we run the same
# command in a plone.recipe.command so that if fails if something goes wrong.
commands =
fingerprint = $${runner-sshkeys-publickey-fingerprint-cmd:command}
[runner-sshkeys-publickey-fingerprint]
# fingerprint for ssh url, see
# https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
# https://winscp.net/eng/docs/session_url#hostkey
# format is host-key-alg-fingerprint, but we know that
# $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg
# we just use use rsa.
fingerprint = ssh-rsa-$${runner-sshkeys-publickey-fingerprint-shelloutput:fingerprint}
#---------------------------
#--
#-- Set nginx frontend
......@@ -612,6 +640,7 @@ backend-url = $${slaprunner:access-url}
init-user = $${runner-htpasswd:user}
init-password = $${runner-htpasswd:password}
ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshkeys-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
......@@ -681,6 +710,8 @@ rendered = $${slaprunner:slapos.cfg}
mode = 700
context =
section slaprunner slaprunner
import codecs codecs
raw buildout_shared_part_list_dump ${template-buildout-shared-part-list:output}
[slapos-test-cfg]
recipe = slapos.recipe.template:jinja2
......@@ -773,8 +804,13 @@ rendered = $${buildout:directory}/.bash_profile
context =
raw path $${shell-environment:path}
raw shell $${shell-environment:shell}
key terminfo terminfo:location
key instance_name slap-parameter:instance-name
key workdir runnerdirectory:home
key home directory:home
[terminfo]
location = ${ncurses:location}/share/terminfo/
#---------------------------
#--
......
......@@ -27,6 +27,10 @@ extends =
../../stack/logrotate/buildout.cfg
../../stack/monitor/buildout.cfg
# make sure shared-part-list is available, even for old versions
# of slapos who do not set that.
shared-part-list =
# stacks are listed from most generic to most specific,
# to avoid versioning issues
......@@ -131,6 +135,10 @@ filename = resilient_software_release_information.py.in
< = template-download-base
filename = slapos-slapuser-script.in
[template-buildout-shared-part-list]
< = template-base
output = ${buildout:directory}/buildout-shared-part-list
[eggs]
recipe = zc.recipe.egg
eggs =
......@@ -140,6 +148,7 @@ eggs =
erp5.util
lock-file
plone.recipe.command
collective.recipe.shelloutput
slapos.recipe.build
slapos.toolbox[flask_auth]
gunicorn==19.7.1
......@@ -162,6 +171,7 @@ gunicorn = 19.7.1
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.3
collective.recipe.shelloutput = 0.1
collective.recipe.environment = 0.2.0
smmap = 0.9.0
lockfile = 0.12.2
......
......@@ -3,6 +3,7 @@
# If you want to load your custom bash configuration, please use a .bashrc file
cd {{ workdir }}
export HOME={{- home }}
export PATH={{- path }}
export SHELL={{- shell }}
{%- if instance_name %}
......@@ -10,6 +11,8 @@ export PROMPT_COMMAND='echo -en "\033]0;{{-instance_name}}\a"'
{% endif %}
export PS1="$ "
export TERMINFO={{- terminfo }}
if [ -f "$HOME/.bashrc" ] ; then
source $HOME/.bashrc
fi
......@@ -32,5 +35,5 @@ alias la='ls -A'
alias l='ls -CF'
echo "Welcome to SlapOS slaprunner shell"
echo
echo
${buildout:shared-part-list}
\ No newline at end of file
#!/bin/sh
# run slapos command inside slaprunner with appropriate config
SLAPOS_CONFIGURATION={{ config_location }} \
SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION \
export SLAPOS_CONFIGURATION=${SLAPOS_CONFIGURATION-"{{ config_location }}"}
export SLAPOS_CLIENT_CONFIGURATION=${SLAPOS_CLIENT_CONFIGURATION-$SLAPOS_CONFIGURATION}
exec {{ slapos_python_file_location }} "$@"
[slapos]
software_root = {{ slaprunner['software_root'] }}
instance_root = {{ slaprunner['instance_root'] }}
shared_part_list =
{#- buildout_shared_part_list_dump is ${buildout:shared-part-list} rendered as a
template during software step.
Because it can contain new lines, it's not possible to use it directly when generating
buildout config files from buildout, because the newlines don't get indented and cause
the instance buildout to be invalid ini file.
So we had to dump it in a simple text file and now we parse again that text file. -#}
{%- for line in codecs.open(buildout_shared_part_list_dump).readlines() %}
{{ line.strip() }}
{%- endfor %}
{{ slaprunner['shared_root'] }}
master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
computer_id = slaprunner
maximal_delay = 0
......
......@@ -47,6 +47,9 @@ setup(name=name,
'erp5.util',
'supervisor',
'psutil',
'paramiko',
'six',
'requests',
],
zip_safe=True,
test_suite='test',
......
......@@ -26,6 +26,18 @@
##############################################################################
import os
import unittest
import paramiko
import contextlib
import base64
import hashlib
import subprocess
from six.moves.urllib.parse import urlparse
from six.moves.urllib.parse import quote
from six.moves.urllib.parse import urljoin
from six.moves.configparser import ConfigParser
import requests
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -40,6 +52,193 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
__partition_reference__ = 's'
class TestWeb(SlaprunnerTestCase):
def test_slaprunner(self):
# slaprunner main interface is password protected
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['url']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
resp = requests.get(
url,
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertIn('SlapOS', resp.text)
def test_shellinabox(self):
# shellinabox exists at /shellinabox and is password protected
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = urljoin(parameter_dict['url'], '/shellinabox')
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
resp = requests.get(
url,
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
self.assertIn('ShellInABox', resp.text)
self.assertNotIn('SlapOS', resp.text)
def test_public_url(self):
# ~/srv/runner/public/ is served over http
parameter_dict = self.computer_partition.getConnectionParameterDict()
public_url = parameter_dict['public-url']
hello_file = os.path.join(
self.computer_partition_root_path,
'srv',
'runner',
'public',
'hello.html')
self.addCleanup(os.remove, hello_file)
with open(hello_file, 'w') as f:
f.write('<b>Hello</b>')
index = requests.get(public_url, verify=False)
self.assertEqual(requests.codes.ok, index.status_code)
self.assertIn('hello.html', index.text)
hello = requests.get(urljoin(public_url, 'hello.html'), verify=False)
self.assertEqual(requests.codes.ok, hello.status_code)
self.assertIn('<b>Hello</b>', hello.text)
# git seems broken, these are 404 now...
@unittest.expectedFailure
def test_git_private(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-private']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
resp = requests.get(
url,
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
@unittest.expectedFailure
def test_git_public(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-public']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
class TestSSH(SlaprunnerTestCase):
@classmethod
def getInstanceParameterDict(cls):
cls.ssh_key = paramiko.RSAKey.generate(1024)
return {
'user-authorized-key': 'ssh-rsa {}'.format(cls.ssh_key.get_base64())
}
def test_connect(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
ssh_url = parameter_dict['ssh-url']
parsed = urlparse(ssh_url)
self.assertEqual('ssh', parsed.scheme)
# username contain a fingerprint (only, so we simplify the parsing)
#
# relevant parts of the grammar defined in
# https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html
#
# ssh-info = [ userinfo ] [";" c-param *("," c-param)]
# c-param = paramname "=" paramvalue
ssh_info = parsed.username
username, fingerprint_from_url = ssh_info.split(';fingerprint=')
client = paramiko.SSHClient()
self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), '')
fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
class KeyPolicy(object):
"""Accept server key and keep it in self.key for inspection
"""
def missing_host_key(self, client, hostname, key):
self.key = key
key_policy = KeyPolicy()
client.set_missing_host_key_policy(key_policy)
with contextlib.closing(client):
client.connect(
username=username,
hostname=parsed.hostname,
port=parsed.port,
pkey=self.ssh_key,
)
# Check fingerprint from server matches the published one.
# Paramiko does not allow to get the fingerprint as SHA256 easily yet
# https://github.com/paramiko/paramiko/pull/1103
self.assertEqual(
fingerprint_from_url,
quote(
# base64 encoded fingerprint adds an extra = at the end
base64.b64encode(
hashlib.sha256(key_policy.key.asbytes()).digest())[:-1],
# also encode /
safe=''))
# Check shell is usable
channel = client.invoke_shell()
channel.settimeout(30)
received = ''
while True:
r = channel.recv(1024)
self.logger.debug("received >%s<", r)
if not r:
break
received += r
if 'slaprunner shell' in received:
break
self.assertIn("Welcome to SlapOS slaprunner shell", received)
# simple commands can also be executed ( this would be like `ssh bash -c 'pwd'` )
self.assertEqual(
self.computer_partition_root_path,
client.exec_command("pwd")[1].read(1000).strip())
class TestSlapOS(SlaprunnerTestCase):
def test_slapos_command(self):
# in ~/bin/slapos there is a wrapper setting configuration to use slapos from
# the web runner.
proxy_show_output = subprocess.check_output(
(
os.path.join(self.computer_partition_root_path, 'bin', 'slapos'),
'proxy',
'show',
),
env={})
self.assertIn('slaprunner', proxy_show_output)
def test_shared_part_list(self):
# this slapos used shared_part_list
cfg_parser = ConfigParser()
with open(os.path.join(self.computer_partition_root_path,
'etc',
'slapos.cfg')) as f:
cfg_parser.readfp(f)
shared_part_list = cfg_parser.get('slapos', 'shared_part_list').splitlines()
# web runner own shared parts. Note that there is intentionnaly a double
# slash in this path, because slaprunner has double slash in paths since
# early releases, including for the path of slapos repository that will be
# used to develop and install software. If we fix this duplication, then
# the URL of installed software will be different and it will get a different
# hash and be reinstalled. To prevent this, we keep that // between srv and runner.
self.assertEqual(
'{}/srv//runner//shared'.format(self.computer_partition_root_path.rstrip('/')),
shared_part_list[-1])
# shared parts from outer slapos
outer_shared_part_list = os.getenv('SLAPOS_TEST_SHARED_PART_LIST',
'').split(os.pathsep)
for outer_shared_part in outer_shared_part_list:
self.assertIn(outer_shared_part, shared_part_list)
class ServicesTestCase(SlaprunnerTestCase):
def test_hashes(self):
hash_files = [
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment