Commit 430cbc77 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 93f7f494 85a521b6
[buildout]
extends = ../xz-utils/buildout.cfg
parts = bash-completion
[bash-completion]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/scop/bash-completion/releases/download/2.11/bash-completion-2.11.tar.xz
md5sum = 2514c6772d0de6254758b98c53f91861
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
......@@ -52,13 +52,23 @@ environment-extra =
[golang1.14]
<= golang-common
url = https://golang.org/dl/go1.14.10.src.tar.gz
md5sum = c0b485e8ac8b1978b5164e3e200150fa
url = https://golang.org/dl/go1.14.12.src.tar.gz
md5sum = ae124560e0185a82883b57cc9662aa73
# go1.14 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
[golang1.15]
<= golang-common
url = https://golang.org/dl/go1.15.5.src.tar.gz
md5sum = 12a60d319ff58c03c323e8dcc125ae81
# go1.15 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
# ---- infrastructure to build Go workspaces / projects ----
# gowork is a top-level section representing workspace
......
......@@ -25,7 +25,7 @@ parts =
install =
lab.nexedi.com/nexedi/helloweb/go/...
golang = ${golang1.14:location}
golang = ${golang1.15:location}
# -*- go -*-
[helloweb-go]
......
[buildout]
extends =
../pygolang/buildout.cfg
../git/buildout.cfg
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${pygolang:egg}
${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
exe = ${buildout:bin-directory}/nxdtest
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = 40e2c4ab
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
[buildout]
extends =
../defaults.cfg
test.cfg
[python]
part = python3
# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
[buildout]
extends =
../../stack/nxdtest.cfg
../pytest/buildout.cfg
parts =
python-interpreter
slapos-cookbook
instance.cfg
# override to install test extra
[nxdtest-egg]
egg = nxdtest[test]
[python-interpreter]
eggs +=
${pytest:eggs}
${nxdtest:eggs}
# env.sh for interpreter to be on $PATH.
[nxdtest-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
export PS1="(nxdtest-env) $PS1"
export PATH=${buildout:bin-directory}:$PATH
# instance to run nxdtest.
[instance.cfg]
<= jinja2-template
template = inline:
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
[runTestSuite]
env.sh = ${nxdtest-env.sh:rendered}
workdir = ${nxdtest-repository:location}
......@@ -37,6 +37,14 @@ interpreter = python
# interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
# -> use pymain from gpython to workaround that.
initialization =
# minimal support for -O argument.
if '-O' in sys.argv:
import os
with open(sys.argv[0]) as f:
executable = f.readlines()[0][2:-1]
sys.argv.remove('-O')
os.execve(executable, sys.argv, dict(os.environ, PYTHONOPTIMIZE='1'))
# set sys.executable to self, so that subprocess and friends go through us
# and this way spawn children with correct sys.path where all eggs that
# parent have are present. TODO consider migrating this into pymain
......
# SlapOS software release to test pygolang/py3 on Nexedi testing infrastructure.
[buildout]
extends = test.cfg
[python]
part = python3
......@@ -16,7 +16,7 @@ parts =
trafficserver
[gcc]
# For ATS v8.0.0 and later, a compiler with support for C++17 is required.
# For ATS v8.0.0 and later, a compiler with support for C++17 is required.
# https://cwiki.apache.org/confluence/display/TS/Building
min_version = 8
......@@ -39,7 +39,7 @@ configure-options =
environment =
PATH=${libtool:location}/bin:${make:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
make-target =
check
install
......@@ -3,6 +3,7 @@
[buildout]
extends =
../pygolang/buildout.cfg
../numpy/buildout.cfg
../git/buildout.cfg
parts =
......@@ -27,6 +28,11 @@ setup-eggs =
${pygolang:egg}[pyx.build]
environment = wendelin.core-env
# dependent eggs that must come through in-tree recipes
depends =
${numpy:egg}
[wendelin.core-env]
# wendelin.core needs git(dev) to build
PATH = ${git:location}/bin:%(PATH)s
......
......@@ -36,7 +36,7 @@ class Recipe(GenericBaseRecipe):
self.path_list = []
options = self.options.copy()
del options['recipe']
CONFIG = {k.replace('-', '_'): v for k, v in options.iteritems()}
CONFIG = {k.replace('-', '_'): v for k, v in options.items()}
CONFIG['PATH'] = os.environ['PATH']
if self.options['instance-dict']:
......@@ -44,7 +44,7 @@ class Recipe(GenericBaseRecipe):
config_instance_dict.add_section('instance_dict')
instance_dict = json.loads(self.options['instance-dict'])
for k ,v in instance_dict.iteritems():
for k ,v in instance_dict.items():
config_instance_dict.set('instance_dict', k, v)
value = io.StringIO()
config_instance_dict.write(value)
......
import functools
import os
import shutil
import tempfile
import unittest
from six.moves import configparser
import zc.buildout.testing
class UserInfoTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmp_dir)
self.get_temp_path = functools.partial(os.path.join, self.tmp_dir)
self.buildout = buildout = zc.buildout.testing.Buildout()
buildout['slap-connection'] = {
'computer-id': 'computer-id',
'server-url': 'https://slapos.example.com',
}
# dummy defaults
buildout['erp5testnode'] = {
'apache-binary': '/bin/httpd',
'apache-htpasswd': '/bin/htpasswd',
'apache-mime-file': '/etc/mime.types',
'apache-modules-dir': '/srv/modules',
'frontend-url': 'https://example.com/',
'git-binary': '/bin/git',
'httpd-cert-file': 'etc/httpd-public.crt',
'httpd-key-file': 'etc/httpd-public.key',
'httpd-conf-file': 'etc/httpd.conf',
'httpd-ip': '::1',
'httpd-lock-file': 'var/run/httpd.lock',
'httpd-log-directory': 'var/log',
'httpd-pid-file': 'var/run/httpd.pid',
'httpd-port': '9080',
'httpd-software-access-port': '9081',
'httpd-software-directory': 'srv/software',
'httpd-wrapper': 'bin/httpd',
'instance-dict': '',
'ipv4-address': '127.0.0.1',
'ipv6-address': '::1',
'keep-log-days': '3',
'log-directory': 'var/log/testnode',
'log-file': 'var/log/erp5testnode.log',
'log-frontend-url': 'https://example.com',
'node-quantity': '2',
'proxy-host': '127.0.0.1',
'proxy-port': '5000',
'recipe': 'slapos.cookbook:erp5testnode',
'run-directory': 'var/run/testnode',
'shared-part-list': 'srv/shared',
'slapos-binary': '/bin/slapos',
'slapos-directory': 'srv/slapos',
'software-directory': 'srv/software',
'software-path-list': '[""]',
'srv-directory': 'srv',
'test-node-title': 'TEST NODE TITLE',
'test-suite-directory': 'srv/test_suite',
'test-suite-master-url': 'https://testnode.example.com',
'testnode': '/bin/testnode',
'working-directory': 'srv/testnode',
'wrapper': 'bin/erp5testnode-service',
'zip-binary': '/bin/zip',
}
# values for test
buildout['erp5testnode']['configuration-file'] = self.get_temp_path(
'configuration-file')
buildout['erp5testnode']['wrapper'] = self.get_temp_path('wrapper')
buildout['erp5testnode']['httpd-conf-file'] = self.get_temp_path(
'httpd-conf-file')
buildout['erp5testnode']['httpd-wrapper'] = self.get_temp_path(
'httpd-wrapper')
buildout['erp5testnode']['log-directory'] = self.get_temp_path(
'log-directory')
os.mkdir(self.get_temp_path('log-directory'))
# software URLs are given as a json encoded string
buildout['erp5testnode'][
'software-path-list'] = '["https://example.com/slapos/software.cfg", "https://example.com/slapos/another-software.cfg"]'
from slapos.recipe import erp5testnode
self.recipe = erp5testnode.Recipe(
buildout,
'erp5testnode',
buildout['erp5testnode'],
)
def test_installed_paths(self):
self.assertEqual(
sorted(self.recipe.install()),
sorted([
self.get_temp_path('configuration-file'),
self.get_temp_path('wrapper'),
self.get_temp_path('httpd-conf-file'),
self.get_temp_path('httpd-wrapper'),
]))
def test_configuration_file(self):
self.recipe.install()
parser = configparser.ConfigParser()
parser.read(self.get_temp_path('configuration-file'))
# this is generally a valid configparser file.
self.assertEqual(parser.get('testnode', 'slapos_directory'), 'srv/slapos')
# software URLs are specified comma separated (XXX not sure it was
# good idea, but it's like this)
self.assertEqual(parser.get('software_list', 'path_list'),
"https://example.com/slapos/software.cfg,https://example.com/slapos/another-software.cfg")
def test_log_directory_apache(self):
self.recipe.install()
self.assertTrue(
os.path.exists(self.get_temp_path('log-directory', 'index.html')))
# apache to expose log directory
with open(self.get_temp_path('httpd-conf-file')) as f:
self.assertIn(
'DocumentRoot "{}"'.format(self.get_temp_path('log-directory')),
f.read())
# wrapper references the config file
with open(self.get_temp_path('httpd-wrapper')) as f:
self.assertIn(self.get_temp_path('httpd-conf-file'), f.read())
......@@ -14,4 +14,4 @@
# not need these here).
[template-instance]
filename = instance.cfg
md5sum = aa91245be4b946d0537f6e007f0d72ea
md5sum = 0974248c0b0ad5da45670386a5301e47
# Jinja2 template of a buildout file to create runnable scripts
#
# Received variables
# - software_dir: location where this repository was cloned to
# - instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
# - project_dir: location where this repository was cloned to
# - software_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
[buildout]
parts =
......@@ -12,6 +12,7 @@ parts =
template-full-build-script
template-sudo-wrapper
template-cros-sdk-wrapper
resiliency-exclude-file
extends = {{ monitor_template }}
# standard declaration of eggs directories
......@@ -30,7 +31,7 @@ git-executable = {{ git_path }}/bin/git
[customize-path]
# add depot tools directory (for cros_sdk binary among others) and git directory to the path
command =
export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
export PATH="${directory:wrapper_dir}":"{{ software_dir }}/bin":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
[nayuos-ebuilds]
recipe = slapos.recipe.build:gitclone
......@@ -56,8 +57,10 @@ cert = ${slap_connection:cert_file}
recipe = slapos.cookbook:mkdirectory
log = ${buildout:directory}/var/log
run = ${buildout:directory}/etc/run
backup = ${buildout:directory}/srv/backup
wrapper_dir = ${buildout:directory}/wrapper_bin
cros_location = ${buildout:directory}/parts/chromiumos
images = ${:cros_location}/images
[bin]
# dummy section to hold references to frequently used binaries
......@@ -79,7 +82,7 @@ config-command = ${promise-sudo-on-host-bin:wrapper-path}
[template-sudo-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/wrapper_sudo.in
template = {{ project_dir }}/scripts/wrapper_sudo.in
rendered = ${directory:wrapper_dir}/sudo
md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
mode = 0700
......@@ -90,7 +93,7 @@ context =
[template-cros-sdk-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
template = {{ project_dir }}/scripts/wrapper_cros_sdk.in
rendered = ${bin:wrapper_cros_sdk}
md5sum = 7159fe3d5b85a283733cf686c4ee0a74
mode = 0700
......@@ -104,25 +107,32 @@ context =
# create the rendered script in the buildout-directory/etc/run
# (use jinja for templating)
recipe = slapos.recipe.template:jinja2
template = {{ software_dir }}/scripts/cros_full_build.in
template = {{ project_dir }}/scripts/cros_full_build.in
rendered = ${directory:run}/cros_full_build
md5sum = f3b3b5408b69f752f2221d3965caf590
#md5sum = f3b3b5408b69f752f2221d3965caf590
mode = 0700
context =
key bash_path bin:bash
key instance_log_dir directory:log
key instance_dir buildout:directory
key cros_location directory:cros_location
key export_path_cmd customize-path:command
key branch parameters:configuration.branch
key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache
key ebuilds_dir nayuos-ebuilds:location
raw scripts_dir {{ software_dir }}/scripts
raw logo_dir {{ software_dir }}/logo
raw wallpaper_dir {{ software_dir }}/wallpaper
raw patch_dir {{ software_dir }}/patch
raw scripts_dir {{ project_dir }}/scripts
raw logo_dir {{ project_dir }}/logo
raw wallpaper_dir {{ project_dir }}/wallpaper
raw patch_dir {{ project_dir }}/patch
# packages to be copied from our overlay into ChromiuOS' overlay
# those packages will be marked as dependency of root filesystem
raw nayu_dev_rootfs_packages app-misc/nayuos-chromium-policy
# those packages will be marked as dependency of the chromium build itself
raw nayu_dev_packages dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
[resiliency-exclude-file]
# Generate rdiff exclude file for resiliency
recipe = collective.recipe.template
input = inline: parts/chromiumos/release*
output = ${directory:srv}/exporter.exclude
......@@ -8,19 +8,29 @@ extends =
../../component/libexpat/buildout.cfg
../../component/libaio/buildout.cfg
../../component/boost-lib/buildout.cfg
../../component/rsync/buildout.cfg
parts +=
eggs
slapos-cookbook
template-instance
git
curl
thin-provisioning-tools
versions = versions
[gcc]
min_version = 0
[versions]
slapos.recipe.template = 4.4
[python]
# we need python3 for repo/cros_sdk command
part = python3
[eggs]
# name is python3 so that cros_sdk uses it automatically through "/usr/bin/env python3"
recipe = zc.recipe.egg
interpreter = python3
eggs =
setuptools
[template-instance]
# jinja2 render instance.cfg file in the buildout directory
......@@ -29,10 +39,11 @@ template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/${:filename}
mode = 0644
context =
key software_dir :_profile_base_location_
key instance_dir buildout:directory
key project_dir :_profile_base_location_
key software_dir buildout:directory
key curl_path curl:location
key git_path git:location
key rsync_path rsync:location
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key thin_provisioning_tools thin-provisioning-tools:location
......@@ -56,3 +67,6 @@ dependencies =
libexpat
libaio
boost-lib
[versions]
slapos.recipe.template = 4.4
......@@ -14,7 +14,7 @@
# not need these here).
[neotest-env.sh]
filename = neotest-env.sh.in
md5sum = bd46e95f1cea62c3b0082fe8c0c9c90b
md5sum = dd41bd736561b7d59b5ec7e0ab7cde98
[neotest]
filename = neotest.in
......
......@@ -9,24 +9,7 @@ PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH"
PATH="${mariadb:location}/bin:$PATH"
PATH="${mariadb:location}/scripts:$PATH"
PATH="${python:location}/bin:$PATH"
# add all eggs and develop-eggs to py path
#
# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
# TODO(kirr) try doing so
PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH # for .egg-link to be found by setuptools
for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
done
for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
export PYTHONPATH=$egg:$PYTHONPATH
done
PATH="${buildout:bin-directory}/:$PATH" # for python with eggs
export PATH
export PYTHONPATH
export PS1="(neotest-`basename ${buildout:directory}`) $PS1"
......@@ -18,7 +18,6 @@ extends =
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
../../component/pygolang/buildout.cfg
../../component/numpy/buildout.cfg
../../stack/nxdtest.cfg
......@@ -36,8 +35,12 @@ parts =
neoppod-develop
neoppod
wendelin.core-dev
eggs
scripts
# for ZEO scripts (runzeo)
ZEO/scripts
neotest-python
neotest-env.sh
neotest
......@@ -100,23 +103,21 @@ template = inline:
# eggs:
[eggs]
recipe = zc.recipe.egg
eggs =
# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
ZODB3
# also for wc
${numpy:egg}
# to install not only wendelin.core modules but also scripts
wendelin.core
# for ZEO scripts (runzeo)
${ZEO:egg}
# for e.g. tcpu.py
${pygolang:egg}
# bin/python is preinstalled with sys.path to wendelin.core & friends.
[neotest-python]
<= python-interpreter
eggs +=
${wendelin.core-dev:egg}
${neoppod:eggs}
# for instance
plone.recipe.command
[scripts]
recipe = zc.recipe.egg:scripts
eggs =
# to install not only wendelin.core modules but also scripts
${wendelin.core-dev:egg}
# wendelin.core: latest not yet released
[wendelin.core-repository]
......
......@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns]
filename = instance-powerdns.cfg
md5sum = 9cd4e436fa432f37b9f8f4de8350581b
md5sum = 0920200cb05a68b1b4a161a927d9488f
[template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2
md5sum = e45d72de87b4adb89c195ba463be4077
md5sum = 20c37ea06a8fa405bc02470d5115fd11
[template-dns-replicate]
_update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
md5sum = a23e241a236f90ae1afbb5bd5ba0b32d
md5sum = c2bd424f588ad57d37f4cf1329734fb6
[iso-list]
_update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
......
......@@ -43,6 +43,7 @@ context =
{% set dns_name = 'ns%s' % i -%}
{% set dns_domain = dns_name_template_string % i %}
{% set request_section_title = 'request-%s' % dns_name -%}
{% set promise_section_title = 'promise-listen-port-%s' % dns_name -%}
{% set sla_key = "-sla-%s-" % i -%}
{% set sla_key_length = sla_key | length %}
{% set sla_parameters = [] %}
......@@ -55,6 +56,7 @@ context =
{% do dns_domain_list.append(dns_domain) -%}
{% do dns_section_list.append(request_section_title) -%}
{% do part_list.append(request_section_title) -%}
{% do part_list.append(promise_section_title) -%}
[{{request_section_title}}]
<= replicate
......@@ -69,6 +71,15 @@ config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
{% endfor -%}
[{{promise_section_title}}]
<= monitor-promise-base
module = check_port_listening
name = pdns-port-listening.py
{% set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
config-hostname = {{ipv6}}
{% set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
config-port = {{port}}
{% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
{% endfor -%}
......@@ -101,7 +112,7 @@ software-url = {{ slapparameter_dict.pop(dns_software_url_key) }}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{dns_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
return = slave-instance-information-list monitor-base-url powerdns-ipv6 powerdns-port powerdns-ipv4
config-server-admin = {{ server_admin }}
config-ns-record = {{ ns_record }}
{% for parameter, value in slapparameter_dict.items() -%}
......@@ -114,12 +125,17 @@ config-monitor-password = ${monitor-htpasswd:passwd}
[publish-information]
recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
ns-record = {{ ns_record }}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
{% for dns_name, dns_section in zip(dns_list, dns_section_list) -%}
{% set dns_ipv6 = '${' ~ dns_section ~ ':connection-powerdns-ipv6}' -%}
{% set dns_port = '${' ~ dns_section ~ ':connection-powerdns-port}' -%}
{{ dns_name }}-ipv6 = {{ dns_ipv6 }}
{{ dns_name }}-port = {{ dns_port }}
{% endfor -%}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% if monitor_interface_url -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}
......
......@@ -55,7 +55,7 @@ context =
#
[pdns]
configuration = $${pdns-directory:configuration}/pdns.conf
local-ipv4 = $${instance-parameter:ipv4-random}
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 5353
socket-directory = $${pdns-directory:socket-directory}
......@@ -139,7 +139,7 @@ extra-context =
<= monitor-promise-base
module = check_port_listening
name = pdns-port-listening.py
config-hostname = $${pdns:local-ipv4}
config-hostname = $${pdns:ipv4}
config-port = $${pdns:port}
[publish-connection-informations]
......@@ -147,6 +147,9 @@ recipe = slapos.cookbook:publish
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
powerdns-ipv4 = $${pdns:ipv4}
powerdns-ipv6 = $${pdns:ipv6}
powerdns-port = $${pdns:port}
[buildout]
parts =
......
......@@ -55,7 +55,6 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.4
dnspython = 1.15.0
passlib = 1.7.1
GitPython = 2.1.11
lockfile = 0.12.2
......
# -------------------------------------------------------------------------
# Configure ip/port binding
local-address={{ pdns.get('local-ipv4') }}
local-address={{ pdns.get('ipv4') }}
local-ipv6={{ pdns.get('ipv6') }}
local-port={{ pdns.get('port') }}
......@@ -10,7 +10,7 @@ socket-dir={{ pdns.get('socket-directory') }}
# Monitoring
webserver=yes
webserver-address={{ pdns.get('local-ipv4') }}
webserver-address={{ pdns.get('ipv4') }}
webserver-port={{ pdns.get('webserver-port') }}
# These totally disable query+packet caching for all zones. This is necessary
......
......@@ -47,6 +47,7 @@ setup(name=name,
'erp5.util',
'supervisor',
'psutil',
'dnspython',
],
zip_safe=True,
test_suite='test',
......
......@@ -25,7 +25,12 @@
#
##############################################################################
import dns.edns
import dns.message
import dns.query
import http.client
import os
import requests
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -35,10 +40,64 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
DNS_PORT = 5353
AFRICAN_SUBNET = '41.0.0.0'
CHINA_TELECOM_SUBNET = '1.0.32.0'
CHINA_UNICOM_SUBNET = '116.181.0.0'
CHINA_MOBILE_SUBNET = '112.21.42.5'
EAST_ASIAN_SUBNET = '1.11.0.0'
EUROPEAN_SUBNET = '5.42.160.0'
HONG_KONG_SUBNET = '1.32.192.0'
JAPANESE_SUBNET = '1.0.16.0'
NORTH_AMERICAN_SUBNET = '3.0.0.0'
OCEANIAN_SUBNET = '1.120.0.0'
SOUTH_AMERICAN_SUBNET = '45.70.188.0'
WEST_ASIAN_SUBNET = '46.70.0.0'
class PowerDNSTestCase(SlapOSInstanceTestCase):
# power dns uses sockets and need shorter paths on test nodes.
__partition_reference__ = 'pdns'
default_zone = 'domain.com'
# focus to test connexion parameters only depending on PowerDNS
def getPowerDNSParameterDict(self, parameter_dict):
new_parameter_dict = {}
for key, value in parameter_dict.items():
if key in [
'ns-record',
'ns1-port',
'ns1-ipv6',
'slave-amount',
]:
new_parameter_dict[key] = value
return new_parameter_dict
def getPowerDNSConnexionParameterDict(self):
return self.getPowerDNSParameterDict(
self.requestDefaultInstance().getConnectionParameterDict()
)
def _test_parameter_dict(self, zone=None, dns_quantity=1, slave_amount=0):
if zone is None:
zone = self.default_zone
parameter_dict = self.getPowerDNSConnexionParameterDict()
expected_dict = {
'ns-record': '',
}
ns_record = ''
for replicate_nb in range(1, dns_quantity + 1):
prefix = 'ns%s' % replicate_nb
ns_record += prefix + '.%s' % zone
expected_dict[prefix + '-port'] = str(DNS_PORT)
expected_dict[prefix + '-ipv6'] = self._ipv6_address
expected_dict['ns-record'] = ns_record
expected_dict['slave-amount'] = str(slave_amount)
self.assertEqual(expected_dict, parameter_dict)
class ServicesTestCase(PowerDNSTestCase):
......@@ -63,3 +122,194 @@ class ServicesTestCase(PowerDNSTestCase):
expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_names)
class TestMonitorAccess(PowerDNSTestCase):
def test(self):
connection_parameter_dict = self.requestDefaultInstance()\
.getConnectionParameterDict()
monitor_base_url = connection_parameter_dict.get('monitor-url')
result = requests.get(
monitor_base_url, verify=False, auth=(
connection_parameter_dict.get('monitor-user'),
connection_parameter_dict.get('monitor-password')
)
)
self.assertEqual(
http.client.OK,
result.status_code
)
class TestMasterRequest(PowerDNSTestCase):
def test(self):
self._test_parameter_dict()
class TestMasterRequestDomain(PowerDNSTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'zone': 'toto.example.com',
}
def test(self):
self._test_parameter_dict(zone=self.getInstanceParameterDict()['zone'])
class PowerDNSSlaveTestCase(PowerDNSTestCase):
@classmethod
def requestDefaultInstance(cls):
default_instance = super(PowerDNSSlaveTestCase, cls)\
.requestDefaultInstance()
cls.requestSlaves()
return default_instance
@classmethod
def requestSlaves(cls):
software_url = cls.getSoftwareURL()
software_type = cls.getInstanceSoftwareType()
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
cls.logger.debug(
'requesting slave "%s" software:%s parameters:%s',
slave_reference, software_url, partition_parameter_kw)
cls.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
)
@classmethod
def getSlaveConnectionParameterDictList(cls):
parameter_dict_list = []
for slave_reference, partition_parameter_kw in cls\
.getSlaveParameterDictDict().items():
parameter_dict_list.append(cls.slap.request(
software_release=cls.getSoftwareURL(),
software_type=cls.getInstanceSoftwareType(),
partition_reference=slave_reference,
partition_parameter_kw=partition_parameter_kw,
shared=True
).getConnectionParameterDict())
return parameter_dict_list
@classmethod
def getSlaveParameterDictDict(cls):
return {
'slave-pdns1': {
'record': 'test1',
'origin': 'nexedi.com',
'default': 'test1.com.',
'africa': 'test1africa.com.',
'china-telecom': 'test1china-telecom.com.',
'china-unicom': 'test1china-unicom.com.',
'china-mobile': 'test1china-mobile.com.',
'east-asia': 'test1east-asia.com.',
'europe': 'test1europe.com.',
'hong-kong': 'test1hong-kong.com.',
'japan': 'test1japan.com.',
'north-america': 'test1north-america.com.',
'oceania': 'test1oceania.com.',
'south-america': 'test1south-america.com.',
'west-asia': 'test1west-asia.com.',
},
'slave-pdns2': {
'record': 'test2',
'origin': 'nexedi.com',
'default': 'test2.com.',
'china-telecom': 'test2china-telecom.com.',
'europe': 'test2europe.com.',
'japan': 'test2japan.com.',
}
}
def dns_query(self, domain_name, subnet):
message = dns.message.make_query(domain_name, 'A')
client_subnet_option = dns.edns.ECSOption(subnet)
message.use_edns(options=[client_subnet_option])
answer = dns.query.udp(message, self._ipv6_address, port=DNS_PORT)
return answer.find_rrset(
dns.message.ANSWER,
dns.name.from_text(domain_name),
dns.rdataclass.IN,
dns.rdatatype.CNAME
).to_text().split()[-1]
def _test_dns_resolver(self, zone):
slave_parameter_dict_dict = self.getSlaveParameterDictDict()
subnet_dict = {
'africa': AFRICAN_SUBNET,
'china-telecom': CHINA_TELECOM_SUBNET,
'china-unicom': CHINA_UNICOM_SUBNET,
'china-mobile': CHINA_MOBILE_SUBNET,
'east-asia': EAST_ASIAN_SUBNET,
'europe': EUROPEAN_SUBNET,
'hong-kong': HONG_KONG_SUBNET,
'japan': JAPANESE_SUBNET,
'north-america': NORTH_AMERICAN_SUBNET,
'oceania': OCEANIAN_SUBNET,
'south-america': SOUTH_AMERICAN_SUBNET,
'west-asia': WEST_ASIAN_SUBNET,
}
default_rr_dict = {
'europe': 'eu',
'africa': 'af',
'south-america': 'sa',
'north-america': 'na',
'china-telecom': 'cn-t',
'china-unicom': 'cn-u',
'china-mobile': 'cn-m',
'japan': 'jp',
'hong-kong': 'hk',
'east-asia': 'as',
'west-asia': 'eu',
'oceania': 'oc',
}
for slave_name in slave_parameter_dict_dict:
slave_parameter_dict = slave_parameter_dict_dict[slave_name]
domain_name = '%s.%s' % (slave_parameter_dict['record'], zone)
for region in subnet_dict:
self.assertEqual(
slave_parameter_dict.pop(
region,
'%s.%s.' % (default_rr_dict[region], slave_parameter_dict['origin'])
),
self.dns_query(domain_name, subnet_dict[region])
)
def _test(self, zone=None):
if zone is None:
zone = self.default_zone
self._test_parameter_dict(
zone=zone,
slave_amount=len(self.getSlaveParameterDictDict())
)
self._test_dns_resolver(zone)
class TestSlaveRequest(PowerDNSSlaveTestCase):
def test(self):
self._test()
class TestSlaveRequestDomain(PowerDNSSlaveTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'zone': 'toto.example.com',
}
def test(self):
self._test(zone=self.getInstanceParameterDict()['zone'])
......@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# and load this script to set environment variables
source ( environment-script from step above )
# Clone a working copy somewhere
cd ~/srv/runner/project/
git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappartXXX/parts/slapos/
# change directory to the directory containing test for this software
cd ~/srv/runner/project/slapos_work/software/helloworld/test/
cd ./software/helloworld/test/
# make changes to test code or profile
# run test (with debugging features activated)
SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test
# run test for helloworld software release (with debugging features activated)
SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
```
## Environment variables
......
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 4246cde0a27138e057ba1635cc621edf
md5sum = 9639d0c0e161c094454808fb95fc9781
[buildout]
extends = {{ nxdtest_instance }}
parts =
slapos-test-runner
runTestSuite
publish
eggs-directory = {{ buildout['eggs-directory'] }}
......@@ -26,46 +27,53 @@ repository = {{ slapos_location }}
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
working-dir = ${buildout:directory}/tmp
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
test-working-dir = ${buildout:directory}/tmp
nxdtest-working-dir = ${:var}/nxdtest
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/runTestSuite
command-line =
{{ buildout['bin-directory'] }}/runTestSuite
--python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
--source_code_path_list={{ ','.join(tests.splitlines()) }}
[slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}
environment =
PATH=${slapos-test-runner-environment:PATH}
SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-dot-nxdtest]
recipe = slapos.recipe.template:jinja2
rendered = ${:workdir}/.nxdtest
workdir = ${directory:nxdtest-working-dir}
template = inline:
{% for test in tests.splitlines() %}
TestCase(
{{ repr(test.split()[0]) }},
[ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
cwd={{ repr(test.split()[1]) }},
summaryf=UnitTest.summary,
)
{% endfor %}
[slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
[runTestSuite]
env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
workdir = ${slapos-test-runner-dot-nxdtest:workdir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
export PATH=${slapos-test-runner-environment:PATH}:$PATH
export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
source ${slapos-test-runner-nxdtest-environment.sh:rendered}
{% set shared_part_list = [] %}
{% for shared_part in buildout['shared-part-list'].splitlines() -%}
{% do shared_part_list.append(shared_part) %}
{%- endfor %}
export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
echo "Environment loaded."
echo "The embedded SlapOS is in $SLAPOS_TEST_WORKING_DIR"
echo "To work on a test, execute:"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test"
echo "from test folder"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} -m unittest discover -v"
echo "from a folder containing software release test."
echo
[publish]
......
......@@ -12,9 +12,9 @@ eggs +=
[template]
extra =
${slapos.test.helloworld-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
${slapos.test.repman-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
plantuml ${slapos.test.plantuml-setup:setup}
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
......@@ -13,11 +13,12 @@ extends =
../../component/python-backports-lzma/buildout.cfg
../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg
parts =
eggs
eggs/scripts
slapos-cookbook
template
......@@ -167,8 +168,8 @@ egg = slapos.core
setup = ${slapos.core-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
<= python-interpreter
eggs +=
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
......@@ -202,21 +203,24 @@ eggs =
${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg}
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
scripts =
runTestSuite
slapos
supervisorctl
supervisord
interpreter=
python_for_test
# We don't name this interpreter `python`, so that when we run slapos node
# software, installation scripts running `python` use a python without any
# custom eggs pre-installed, not our special python interpreter.
interpreter = python_for_test
# patches for eggs
patch-binary = ${patch:location}/bin/patch
PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
PyPDF2-patch-options = -p1
[eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts =
slapos
supervisord
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
......@@ -236,6 +240,7 @@ mode = 640
context =
section buildout buildout
import os os
key nxdtest_instance nxdtest-instance.cfg:rendered
key git_location git:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
......@@ -243,29 +248,31 @@ context =
key openssl_location openssl-output:bin
key faketime_location faketime:location
key tests :tests
tests =
${slapos.test.kvm-setup:setup}
${slapos.test.slaprunner-setup:setup}
${slapos.test.metabase-setup:setup}
json-schemas ${slapos.cookbook-setup:setup}
kvm ${slapos.test.kvm-setup:setup}
slaprunner ${slapos.test.slaprunner-setup:setup}
metabase ${slapos.test.metabase-setup:setup}
${:extra}
extra =
${slapos.cookbook-setup:setup}
${slapos.test.backupserver-setup:setup}
${slapos.test.caddy-frontend-setup:setup}
${slapos.test.erp5-setup:setup}
${slapos.test.htmlvalidatorserver-setup:setup}
${slapos.test.slapos-master-setup:setup}
${slapos.test.re6stnet-setup:setup}
${slapos.test.seleniumserver-setup:setup}
${slapos.test.jstestnode-setup:setup}
${slapos.test.jupyter-setup:setup}
${slapos.test.nextcloud-setup:setup}
${slapos.test.turnserver-setup:setup}
${slapos.test.theia-setup:setup}
${slapos.test.grafana-setup:setup}
${slapos.test.gitlab-setup:setup}
${slapos.test.cloudooo-setup:setup}
${slapos.test.dream-setup:setup}
backupserver ${slapos.test.backupserver-setup:setup}
caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
erp5 ${slapos.test.erp5-setup:setup}
htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
slapos-master ${slapos.test.slapos-master-setup:setup}
re6stnet ${slapos.test.re6stnet-setup:setup}
seleniumserver ${slapos.test.seleniumserver-setup:setup}
jstestnode ${slapos.test.jstestnode-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
nextcloud ${slapos.test.nextcloud-setup:setup}
turnserver ${slapos.test.turnserver-setup:setup}
theia ${slapos.test.theia-setup:setup}
grafana ${slapos.test.grafana-setup:setup}
gitlab ${slapos.test.gitlab-setup:setup}
cloudooo ${slapos.test.cloudooo-setup:setup}
dream ${slapos.test.dream-setup:setup}
repman ${slapos.test.repman-setup:setup}
[versions]
# slapos.core is used from the clone always
......
......@@ -2,13 +2,11 @@
This software release is used to run unit test of slapos eggs.
The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
The approach is to use nxdtest test runner, which will run tests for each
projects, as described in `.nxdtest` file.
The `python` used in this command will be a `zc.recipe.egg` interpreter with
all eggs pre-installed by this software release.
The results of this test suite running on Nexedi ERP5 are published as `SlapOS.Eggs.UnitTest-Master.Python3`
and `SlapOS.Eggs.UnitTest-Master.Python2`.
The results of this test suite running on Nexedi ERP5 are published as
`SlapOS.Eggs.UnitTest-Master.Python3` and `SlapOS.Eggs.UnitTest-Master.Python2`.
Here's an example session of how a developer could use this software release in
......@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP
slapos supply $SR $COMP
slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
slapos node instance
# The path of a an environment script was published by slapos parameters, as
# "environment-script"
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# sourcing the script in the shell configure all environment variables and
# print a message explaining how to run tests
source ( environment script from step above )
# The source code is a git clone working copy on the instance
cd ~/srv/runner/instance/slappart0/parts/slapos.core/
cd ~/srv/runner/instance/slappartXXX/parts/slapos.core/
# make some changes to the code
vim slapos/tests/client.py
# run tests, using bundled python intepreter with pre-installed eggs dependencies
SLAPOS_TEST_IPV6=::1 \
SLAPOS_TEST_IPV4=127.0.0.1 \
SLAPOS_TEST_VERBOSE=1 \
SLAPOS_TEST_DEBUG=1 \
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
# run slapos.core tests
runTestSuite --run slapos.core
# ... or run all eggs tests
runTestSuite
# when satified, commit changes
git add -p && git commit
......
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5
md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751
[buildout]
extends = ${nxdtest-instance.cfg:rendered}
parts =
phantomjs-wrapper
slapos-test-runner
runTestSuite
publish
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc
services = $${:etc}/run
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
nxdtest-working-dir = $${:var}/nxdtest
[download-source]
recipe = slapos.recipe.build:gitclone
......@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location}
repository = ${rubygemsrecipe-repository:location}
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
wrapper-path = $${create-directory:bin}/runTestSuite
command-line =
${buildout:bin-directory}/runTestSuite
--python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
--source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location}
# Notes about environment:
# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
# needs ping, which is a setuid binary that cannot be installed via slapos
# way of building software without root access, so we keep "standard"
# /usr/bin and /bin in $PATH
# * SLAPOS_TEST_environment is safe to be used by tests, but there is no
# guarantee about free ports on IPV4 and IPV6
# * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4
environment =
PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/
LOCAL_IPV4=$${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
[phantomjs-wrapper]
recipe = slapos.cookbook:wrapper
command-line = ${phantomjs:location}/phantomjs-slapos
wrapper-path = $${create-directory:bin}/phantomjs
[slapos-test-runner-nxdtest-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline:
export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
[slapos-test-runner-dot-nxdtest]
recipe = slapos.recipe.template:jinja2
rendered = $${:workdir}/.nxdtest
workdir = $${create-directory:nxdtest-working-dir}
template = inline:
TestCase(
"kedifa",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${kedifa:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"caucase",
# XXX caucase uses 2to3 dynamically in setup.py, so it only supports
# runnning tests with python setup.py test
['python', 'setup.py', 'test'],
cwd="""$${caucase:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"erp5.util",
['python', '-m', 'unittest', 'discover', '-v', '--start', 'erp5/tests/'],
cwd="""$${erp5.util:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.cookbook",
# slapos/test contain both tests for recipes and tests for
# json schemas, we only care about recipe tests here, json
# schemas are tested in software/slapos-sr-testing
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test/recipe'],
cwd="""$${slapos.cookbook:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.core",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX some test fail when running with unittest (slapos*.cfg.example
# files cannot be found with pkg_resources.resource_string), so we keep
# using setup.py test for now.
['python', 'setup.py', 'test'],
cwd="""$${slapos.core:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.build",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.build:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.cmmi",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${slapos.recipe.cmmi:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.recipe.template",
# ['python', '-m', 'unittest', 'slapos.recipe.template.tests.test_suite'],
# XXX some test fail when running with unittest, so we keep using setup.py test
['python', 'setup.py', 'test'],
cwd="""$${slapos.recipe.template:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.toolbox",
# ['python', '-m', 'unittest', 'discover', '-v'],
# XXX We can not just run unittest discover here, since slapos/lamp
# imports MySQLDb and we currently don't have installed
# slapos.toolbox[lampconfigure] and on python3 discovery make a
# failing test for this import error.
# Currently slapos/lamp is not tested, but if it it is still used,
# the TODO seem to also install slapos.toolbox[lampconfigure] here.
['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test', '--top-level-directory', '.'],
cwd="""$${slapos.toolbox:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.libnetworkcache",
['python', '-m', 'unittest', '-v', 'slapos.libnetworkcachetests'],
cwd="""$${slapos.libnetworkcache:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"slapos.rebootstrap",
['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],
cwd="""$${slapos.rebootstrap:location}""",
summaryf=UnitTest.summary,
)
TestCase(
"rubygemsrecipe",
['python', '-m', 'unittest', 'discover', '-v'],
cwd="""$${rubygemsrecipe:location}""",
summaryf=UnitTest.summary,
)
[runTestSuite]
env.sh = $${slapos-test-runner-nxdtest-environment.sh:rendered}
workdir = $${slapos-test-runner-dot-nxdtest:workdir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = $${create-directory:etc}/$${:_buildout_section_name_}
template = inline:
source $${slapos-test-runner-nxdtest-environment.sh:rendered}
echo "Environment loaded."
echo "To work on a test, execute:"
echo " $${runTestSuite:wrapper-path} -k test_name"
echo "replacing test_name by the name of the test."
echo
[publish]
recipe = slapos.cookbook:publish
environment-script = $${slapos-local-development-environment.sh:rendered}
......@@ -14,11 +14,12 @@ extends =
../../component/lmsensors/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/slapos.cfg
../../stack/nxdtest.cfg
./buildout.hash.cfg
parts =
bootstrap-slapos.recipe.cmmi
eggs
eggs/scripts
phantomjs
template
......@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test]
setup = ${rubygemsrecipe-repository:location}
[eggs]
recipe = zc.recipe.egg
eggs =
<= python-interpreter
eggs +=
${lxml-python:egg}
${python-cryptography:egg}
${backports.lzma:egg}
......@@ -126,14 +127,13 @@ eggs =
${rubygemsrecipe-setup:egg}
zope.testing
supervisor
entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite
[eggs/scripts]
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
scripts =
runTestSuite
slapos
supervisord
interpreter=
python_for_test
[git-clone-repository]
recipe = slapos.recipe.build:gitclone
......
......@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 6e279c46b07bf56b7b037a8ee2c6587e
md5sum = 2582723c31166244ff25cb3d8c839ffa
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
......@@ -46,6 +46,7 @@ common-runner-parts =
custom-frontend-promise
{% endif %}
{% endif %}
project-link
## Monitoring part
monitor-base
monitor-check-webrunner-internal-instance
......@@ -159,6 +160,11 @@ software-test = $${:test}/software
instance-test = $${:test}/instance
sessions = $${buildout:directory}/.sessions
[project-link]
recipe = slapos.cookbook:symbolic.link
target-directory = $${directory:srv}
link-binary = $${runnerdirectory:project}
[slaprunner]
slaprunner = ${buildout:bin-directory}/slaprunner
slapos = ${buildout:bin-directory}/slapos
......
......@@ -15,11 +15,11 @@
[instance]
filename = instance.cfg.in
md5sum = 2ceb9389281c00261abd864fc8ed566f
md5sum = 1c60191f8724854f979a17d2624e65d8
[yarn.lock]
filename = yarn.lock
md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
md5sum = 5a89742266a9f9d4115efa6d641fd5bb
[python-language-server-requirements.txt]
filename = python-language-server-requirements.txt
......
......@@ -173,7 +173,7 @@ template =
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
......@@ -197,7 +197,7 @@ template =
],
"options": {
"env": {
"SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
"SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
"GIT_EXEC_PATH": ""
}
},
......@@ -227,6 +227,7 @@ template =
export TMP=$${directory:tmp}
export TEMP=$TMP
export LC_ALL=C.UTF-8
export TERMINFO=${ncurses:location}/lib/terminfo/
export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
exec ${theia-wrapper:rendered} $@
ip = $${instance-parameter:ipv4-random}
......@@ -241,7 +242,7 @@ hash-existing-files =
${yarn.lock:output}
${theia-wrapper:rendered}
ip = $${instance-parameter:ipv4-random}
hostname = $${:ip}
hostname = $${:ip}
port = $${theia-service:port}
base-url = $${theia-service:base-url}
......@@ -256,36 +257,76 @@ template = inline:
import time
args = sys.argv[1:]
# when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ]
if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash --rcfile $${theia-bashrc:rendered}", ]
# otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
else: time.sleep(1)
os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash'] + args)
[theia-bashrc]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:etc}/$${:_buildout_section_name_}
template =
inline:
# enable bash completion
. ${bash-completion:location}/etc/profile.d/bash_completion.sh
# source user's .bashrc
[ -f ~/.bashrc ] && . ~/.bashrc
depends =
$${shell-setup-completion:recipe}
[shell-setup-completion]
recipe = plone.recipe.command
stop-on-error = true
command =
${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
[slapos-standalone-config]
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 4000
slapos-configuration = $${directory:runner}/etc/slapos.cfg
computer-id = slaprunner
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
template =
inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH
${slapos-standalone:script-path} \
$${directory:slapos} \
$${:ipv4} \
$${:ipv6} \
$${:port} \
export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
[slapos-standalone]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
template =
inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH
exec ${slapos-standalone:script-path} \
$${directory:runner} \
$${slapos-standalone-config:ipv4} \
$${slapos-standalone-config:ipv6} \
$${slapos-standalone-config:port} \
$${slapos-standalone-config:computer-id} \
$${slap-connection:server-url} \
$${slap-connection:computer-id} \
$${slap-connection:partition-id} \
--key='$${slap-connection:key-file}' \
--cert='$${slap-connection:cert-file}'
export SLAPOS_CONFIGURATION=$${:slapos-configuration}
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 4000
slapos-configuration = $${directory:slapos}/etc/slapos.cfg
[slapos-standalone-instance]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = $${slapos-standalone:rendered}
hash-files =
$${slapos-standalone:rendered}
hostname = $${slapos-standalone-config:ipv4}
port = $${slapos-standalone-config:port}
[promises]
recipe =
......@@ -293,6 +334,7 @@ instance-promises =
$${theia-listen-promise:name}
$${frontend-listen-promise:name}
$${apache-frontend-url-available-promise:name}
$${slapos-standalone-listen-promise:name}
[theia-listen-promise]
<= monitor-promise-base
......@@ -315,6 +357,14 @@ name = $${:_buildout_section_name_}.py
config-url = $${apache-frontend:connection-secure_access}
config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[apache-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
......@@ -353,8 +403,11 @@ dot-theia = $${buildout:directory}/.theia/
pidfiles = $${:var}/run
services = $${:etc}/service
runner = $${:srv}/runner
project = $${:srv}/project
slapos = $${:srv}/slapos
frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
frontend-static-css = $${:frontend-static}/css
bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
fish-completions = $${buildout:directory}/.config/fish/completions/
......@@ -4,6 +4,7 @@ extends =
../../component/caddy/buildout.cfg
../../component/git/buildout.cfg
../../component/bash/buildout.cfg
../../component/bash-completion/buildout.cfg
../../component/fish-shell/buildout.cfg
../../component/tmux/buildout.cfg
../../component/tig/buildout.cfg
......@@ -55,9 +56,11 @@ entry-points =
${:scripts}=not_used:main
initialization =
import argparse
import glob
import os.path
import sys
import glob
import signal
import time
import slapos.slap.standalone
......@@ -66,6 +69,7 @@ initialization =
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
parser.add_argument('computer_id')
forwarded_arguments = parser.add_argument_group('forwarded')
forwarded_arguments.add_argument('master_url')
forwarded_arguments.add_argument('computer')
......@@ -91,7 +95,10 @@ initialization =
args.base_directory,
args.ipv4,
args.server_port,
computer_id=args.computer_id,
shared_part_list=shared_part_list,
software_root="%s/software" % args.base_directory,
instance_root="%s/instance" % args.base_directory,
partition_forward_configuration=partition_forward_configuration,
)
standalone.start()
......@@ -104,7 +111,26 @@ initialization =
args.ipv4,
args.ipv6
)
print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
# Run instance at least once, to start the supervisor managing instances.
try:
standalone.waitForInstance(max_retry=0)
except slapos.slap.standalone.SlapOSNodeCommandError as e:
print("Error instanciating: {}".format(e))
quit_requested = []
def signal_handler(signum, frame):
print("Signal {signum} received".format(signum=signum))
quit_requested.append(True)
signal.signal(signal.SIGTERM, signal_handler)
print("Standalone SlapOS ready")
while not quit_requested:
time.sleep(.1)
print("Stopping standalone subsystem")
standalone.stop()
print("Exiting")
sys.exit(0)
needs-these-eggs-scripts-in-path =
......@@ -343,7 +369,7 @@ command =
[cli-utilities]
PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/
PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
[theia-wrapper]
recipe = slapos.recipe.template:jinja2
......
......@@ -36,9 +36,13 @@ import re
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import psutil
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.grid.svcbackend import getSupervisorRPC
from slapos.grid.svcbackend import _getSupervisordSocketPath
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
......@@ -110,12 +114,11 @@ class TestTheia(SlapOSInstanceTestCase):
pass
process.logfile = DebugLogFile()
process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
process.expect_exact('Standalone SlapOS for computer `local` activated')
process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
# try to supply and install a software to check that this slapos is usable
process.sendline(
'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local'
'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg slaprunner'
)
process.expect(
'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
......@@ -135,12 +138,6 @@ class TestTheia(SlapOSInstanceTestCase):
# interrupt this, we don't want to actually wait for software installation
process.sendcontrol('c')
# shutdown this slapos
process.sendline(
'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
self.computer_partition_root_path))
process.expect('Shut down')
process.terminate()
process.wait()
......@@ -153,3 +150,37 @@ class TestTheia(SlapOSInstanceTestCase):
'touch "{}"'.format(test_file)
])
self.assertTrue(os.path.exists(test_file))
class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for sockets in included slapos
def test_stopping_instance_stops_embedded_slapos(self):
embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
os.path.join(
self.computer_partition_root_path,
'srv',
'slapos',
'inst',
), self.logger)
# Wait a bit for this supervisor to be started.
for _ in range(20):
if os.path.exists(embedded_slapos_supervisord_socket):
break
time.sleep(1)
# get the pid of the supervisor used to manage instances
with getSupervisorRPC(embedded_slapos_supervisord_socket) as embedded_slapos_supervisor:
embedded_slapos_process = psutil.Process(embedded_slapos_supervisor.getPID())
# Stop theia's services
with self.slap.instance_supervisor_rpc as instance_supervisor:
process_info, = [
p for p in instance_supervisor.getAllProcessInfo()
if p['name'].startswith('slapos-standalone-instance-')
]
instance_supervisor.stopProcessGroup(process_info['group'])
# the supervisor controlling instances is also stopped
self.assertFalse(embedded_slapos_process.is_running())
This diff is collapsed.
......@@ -32,7 +32,7 @@
[buildout]
extends =
slapos.cfg
../component/git/buildout.cfg
../component/nxdtest/buildout.cfg
nxdtest/buildout.hash.cfg
[jinja2-template]
......@@ -52,25 +52,5 @@ context =
section nxdtest nxdtest
[nxdtest]
recipe = zc.recipe.egg:scripts
eggs = ${nxdtest-egg:egg}
scripts = nxdtest
# convenience for nxdtest users
exe = ${buildout:bin-directory}/nxdtest
[nxdtest-egg]
recipe = zc.recipe.egg:develop
setup = ${nxdtest-repository:location}
egg = nxdtest
[nxdtest-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nxdtest.git
revision = bd91f6f1579a
location = ${buildout:parts-directory}/nxdtest
git-executable = ${git:location}/bin/git
[versions]
slapos.recipe.template = 4.4
......@@ -164,6 +164,7 @@ dateparser = 0.7.6
decorator = 4.3.0
funcsigs = 1.0.2
gevent = 20.9.0
geventmp = 0.0.1
greenlet = 0.4.17
idna = 2.9
importlib-metadata = 1.7.0
......@@ -191,7 +192,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
six = 1.12.0
slapos.cookbook = 1.0.167
slapos.core = 1.6.2
slapos.core = 1.6.3
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
......@@ -247,7 +248,7 @@ croniter = 0.3.25
# Required by:
# slapos.toolbox==0.94
dnspython = 1.15.0
dnspython = 1.16.0
# Required by:
# cryptography==1.8.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment