Update Release Candidate

component/bash-completion/buildout.cfg

+[buildout]
+extends = ../xz-utils/buildout.cfg
+parts = bash-completion
+
+[bash-completion]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/scop/bash-completion/releases/download/2.11/bash-completion-2.11.tar.xz
+md5sum = 2514c6772d0de6254758b98c53f91861
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/golang/buildout.cfg

@@ -52,13 +52,23 @@ environment-extra =
 
 [golang1.14]
 <= golang-common
-url = https://golang.org/dl/go1.14.10.src.tar.gz
-md5sum = c0b485e8ac8b1978b5164e3e200150fa
+url = https://golang.org/dl/go1.14.12.src.tar.gz
+md5sum = ae124560e0185a82883b57cc9662aa73
 
 # go1.14 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
+[golang1.15]
+<= golang-common
+url = https://golang.org/dl/go1.15.5.src.tar.gz
+md5sum = 12a60d319ff58c03c323e8dcc125ae81
+
+# go1.15 needs go1.4 to bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}
+
+
 # ---- infrastructure to build Go workspaces / projects ----
 
 # gowork is a top-level section representing workspace

component/helloweb/buildout.cfg

@@ -25,7 +25,7 @@ parts =
 install =
     lab.nexedi.com/nexedi/helloweb/go/...
 
-golang  = ${golang1.14:location}
+golang  = ${golang1.15:location}
 
 # -*- go -*-
 [helloweb-go]

component/nxdtest/buildout.cfg

+[buildout]
+extends =
+  ../pygolang/buildout.cfg
+  ../git/buildout.cfg
+
+
+[nxdtest]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${pygolang:egg}
+          ${nxdtest-egg:egg}
+scripts = nxdtest
+# convenience for nxdtest users
+exe     = ${buildout:bin-directory}/nxdtest
+
+[nxdtest-egg]
+recipe  = zc.recipe.egg:develop
+setup   = ${nxdtest-repository:location}
+egg     = nxdtest
+
+[nxdtest-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/nxdtest.git
+revision = 40e2c4ab
+location = ${buildout:parts-directory}/nxdtest
+git-executable = ${git:location}/bin/git

component/nxdtest/test-py3.cfg

+# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
+[buildout]
+extends =
+  ../defaults.cfg
+  test.cfg
+
+[python]
+part = python3

component/nxdtest/test.cfg

+# SlapOS software release to test nxdtest on Nexedi testing infrastructure.
+
+[buildout]
+extends =
+    ../../stack/nxdtest.cfg
+    ../pytest/buildout.cfg
+
+parts =
+    python-interpreter
+    slapos-cookbook
+    instance.cfg
+
+# override to install test extra
+[nxdtest-egg]
+egg = nxdtest[test]
+
+[python-interpreter]
+eggs +=
+  ${pytest:eggs}
+  ${nxdtest:eggs}
+
+# env.sh for interpreter to be on $PATH.
+[nxdtest-env.sh]
+recipe   = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:_buildout_section_name_}
+template = inline:
+  export PS1="(nxdtest-env) $PS1"
+  export PATH=${buildout:bin-directory}:$PATH
+
+# instance to run nxdtest.
+[instance.cfg]
+<= jinja2-template
+template = inline:
+  [buildout]
+  extends = ${nxdtest-instance.cfg:rendered}
+
+  [runTestSuite]
+  env.sh  = ${nxdtest-env.sh:rendered}
+  workdir = ${nxdtest-repository:location}
+

component/pygolang/buildout.cfg

@@ -37,6 +37,14 @@ interpreter = python
 # interpreter code that buildout generates cannot process `-m pytest --<pytest-option>`
 # -> use pymain from gpython to workaround that.
 initialization =
+  # minimal support for -O argument.
+  if '-O' in sys.argv:
+    import os
+    with open(sys.argv[0]) as f:
+        executable = f.readlines()[0][2:-1]
+    sys.argv.remove('-O')
+    os.execve(executable, sys.argv, dict(os.environ, PYTHONOPTIMIZE='1'))
+
   # set sys.executable to self, so that subprocess and friends go through us
   # and this way spawn children with correct sys.path where all eggs that
   # parent have are present. TODO consider migrating this into pymain

component/pygolang/test-py3.cfg

+# SlapOS software release to test pygolang/py3 on Nexedi testing infrastructure.
+
+[buildout]
+extends = test.cfg
+
+[python]
+part = python3

component/trafficserver/buildout.cfg

@@ -16,7 +16,7 @@ parts =
   trafficserver
 
 [gcc]
-# For ATS v8.0.0 and later, a compiler with support for C++17 is required. 
+# For ATS v8.0.0 and later, a compiler with support for C++17 is required.
 # https://cwiki.apache.org/confluence/display/TS/Building
 min_version = 8
 
@@ -39,7 +39,7 @@ configure-options =
 environment =
   PATH=${libtool:location}/bin:${make:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   LDFLAGS =-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${tcl:location}/lib -Wl,-rpath=${tcl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  
+
 make-target =
   check
   install

component/wendelin.core/buildout.cfg

@@ -3,6 +3,7 @@
 [buildout]
 extends =
   ../pygolang/buildout.cfg
+  ../numpy/buildout.cfg
   ../git/buildout.cfg
 
 parts =
@@ -27,6 +28,11 @@ setup-eggs =
     ${pygolang:egg}[pyx.build]
 environment = wendelin.core-env
 
+# dependent eggs that must come through in-tree recipes
+depends =
+    ${numpy:egg}
+
+
 [wendelin.core-env]
 # wendelin.core needs git(dev) to build
 PATH = ${git:location}/bin:%(PATH)s

slapos/recipe/erp5testnode/__init__.py

@@ -36,7 +36,7 @@ class Recipe(GenericBaseRecipe):
     self.path_list = []
     options = self.options.copy()
     del options['recipe']
-    CONFIG = {k.replace('-', '_'): v for k, v in options.iteritems()}
+    CONFIG = {k.replace('-', '_'): v for k, v in options.items()}
     CONFIG['PATH'] = os.environ['PATH']
 
     if self.options['instance-dict']:
@@ -44,7 +44,7 @@ class Recipe(GenericBaseRecipe):
       config_instance_dict.add_section('instance_dict')
       instance_dict = json.loads(self.options['instance-dict'])
 
-      for k ,v in instance_dict.iteritems():
+      for k ,v in instance_dict.items():
         config_instance_dict.set('instance_dict', k, v)
       value = io.StringIO()
       config_instance_dict.write(value)

slapos/test/recipe/test_erp5testnode.py

+import functools
+import os
+import shutil
+import tempfile
+import unittest
+
+from six.moves import configparser
+import zc.buildout.testing
+
+
+class UserInfoTest(unittest.TestCase):
+  def setUp(self):
+    self.tmp_dir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, self.tmp_dir)
+
+    self.get_temp_path = functools.partial(os.path.join, self.tmp_dir)
+
+    self.buildout = buildout = zc.buildout.testing.Buildout()
+    buildout['slap-connection'] = {
+        'computer-id': 'computer-id',
+        'server-url': 'https://slapos.example.com',
+    }
+    # dummy defaults
+    buildout['erp5testnode'] = {
+        'apache-binary': '/bin/httpd',
+        'apache-htpasswd': '/bin/htpasswd',
+        'apache-mime-file': '/etc/mime.types',
+        'apache-modules-dir': '/srv/modules',
+        'frontend-url': 'https://example.com/',
+        'git-binary': '/bin/git',
+        'httpd-cert-file': 'etc/httpd-public.crt',
+        'httpd-key-file': 'etc/httpd-public.key',
+        'httpd-conf-file': 'etc/httpd.conf',
+        'httpd-ip': '::1',
+        'httpd-lock-file': 'var/run/httpd.lock',
+        'httpd-log-directory': 'var/log',
+        'httpd-pid-file': 'var/run/httpd.pid',
+        'httpd-port': '9080',
+        'httpd-software-access-port': '9081',
+        'httpd-software-directory': 'srv/software',
+        'httpd-wrapper': 'bin/httpd',
+        'instance-dict': '',
+        'ipv4-address': '127.0.0.1',
+        'ipv6-address': '::1',
+        'keep-log-days': '3',
+        'log-directory': 'var/log/testnode',
+        'log-file': 'var/log/erp5testnode.log',
+        'log-frontend-url': 'https://example.com',
+        'node-quantity': '2',
+        'proxy-host': '127.0.0.1',
+        'proxy-port': '5000',
+        'recipe': 'slapos.cookbook:erp5testnode',
+        'run-directory': 'var/run/testnode',
+        'shared-part-list': 'srv/shared',
+        'slapos-binary': '/bin/slapos',
+        'slapos-directory': 'srv/slapos',
+        'software-directory': 'srv/software',
+        'software-path-list': '[""]',
+        'srv-directory': 'srv',
+        'test-node-title': 'TEST NODE TITLE',
+        'test-suite-directory': 'srv/test_suite',
+        'test-suite-master-url': 'https://testnode.example.com',
+        'testnode': '/bin/testnode',
+        'working-directory': 'srv/testnode',
+        'wrapper': 'bin/erp5testnode-service',
+        'zip-binary': '/bin/zip',
+    }
+
+    # values for test
+    buildout['erp5testnode']['configuration-file'] = self.get_temp_path(
+        'configuration-file')
+    buildout['erp5testnode']['wrapper'] = self.get_temp_path('wrapper')
+    buildout['erp5testnode']['httpd-conf-file'] = self.get_temp_path(
+        'httpd-conf-file')
+    buildout['erp5testnode']['httpd-wrapper'] = self.get_temp_path(
+        'httpd-wrapper')
+
+    buildout['erp5testnode']['log-directory'] = self.get_temp_path(
+        'log-directory')
+    os.mkdir(self.get_temp_path('log-directory'))
+    # software URLs are given as a json encoded string
+    buildout['erp5testnode'][
+        'software-path-list'] = '["https://example.com/slapos/software.cfg", "https://example.com/slapos/another-software.cfg"]'
+
+    from slapos.recipe import erp5testnode
+    self.recipe = erp5testnode.Recipe(
+        buildout,
+        'erp5testnode',
+        buildout['erp5testnode'],
+    )
+
+  def test_installed_paths(self):
+    self.assertEqual(
+        sorted(self.recipe.install()),
+        sorted([
+            self.get_temp_path('configuration-file'),
+            self.get_temp_path('wrapper'),
+            self.get_temp_path('httpd-conf-file'),
+            self.get_temp_path('httpd-wrapper'),
+        ]))
+
+  def test_configuration_file(self):
+    self.recipe.install()
+
+    parser = configparser.ConfigParser()
+    parser.read(self.get_temp_path('configuration-file'))
+
+    # this is generally a valid configparser file.
+    self.assertEqual(parser.get('testnode', 'slapos_directory'), 'srv/slapos')
+
+    # software URLs are specified comma separated (XXX not sure it was
+    # good idea, but it's like this)
+    self.assertEqual(parser.get('software_list', 'path_list'),
+      "https://example.com/slapos/software.cfg,https://example.com/slapos/another-software.cfg")
+
+
+  def test_log_directory_apache(self):
+    self.recipe.install()
+
+    self.assertTrue(
+        os.path.exists(self.get_temp_path('log-directory', 'index.html')))
+
+    # apache to expose log directory
+    with open(self.get_temp_path('httpd-conf-file')) as f:
+      self.assertIn(
+          'DocumentRoot "{}"'.format(self.get_temp_path('log-directory')),
+          f.read())
+
+    # wrapper references the config file
+    with open(self.get_temp_path('httpd-wrapper')) as f:
+      self.assertIn(self.get_temp_path('httpd-conf-file'), f.read())

software/nayuos/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [template-instance]
 filename = instance.cfg
-md5sum = aa91245be4b946d0537f6e007f0d72ea
+md5sum = 0974248c0b0ad5da45670386a5301e47

software/nayuos/instance.cfg

 # Jinja2 template of a buildout file to create runnable scripts
 #
 # Received variables
-# -  software_dir: location where this repository was cloned to
-# -  instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
+# -  project_dir: location where this repository was cloned to
+# -  software_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
 
 [buildout]
 parts =
@@ -12,6 +12,7 @@ parts =
   template-full-build-script
   template-sudo-wrapper
   template-cros-sdk-wrapper
+  resiliency-exclude-file
 
 extends = {{ monitor_template }}
 # standard declaration of eggs directories
@@ -30,7 +31,7 @@ git-executable = {{ git_path }}/bin/git
 [customize-path]
 # add depot tools directory (for cros_sdk binary among others) and git directory to the path
 command =
-  export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
+  export PATH="${directory:wrapper_dir}":"{{ software_dir }}/bin":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
 
 [nayuos-ebuilds]
 recipe = slapos.recipe.build:gitclone
@@ -56,8 +57,10 @@ cert = ${slap_connection:cert_file}
 recipe = slapos.cookbook:mkdirectory
 log = ${buildout:directory}/var/log
 run = ${buildout:directory}/etc/run
+backup = ${buildout:directory}/srv/backup
 wrapper_dir = ${buildout:directory}/wrapper_bin
 cros_location = ${buildout:directory}/parts/chromiumos
+images = ${:cros_location}/images
 
 [bin]
 # dummy section to hold references to frequently used binaries
@@ -79,7 +82,7 @@ config-command = ${promise-sudo-on-host-bin:wrapper-path}
 
 [template-sudo-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ software_dir }}/scripts/wrapper_sudo.in
+template = {{ project_dir }}/scripts/wrapper_sudo.in
 rendered = ${directory:wrapper_dir}/sudo
 md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
 mode = 0700
@@ -90,7 +93,7 @@ context =
 
 [template-cros-sdk-wrapper]
 recipe = slapos.recipe.template:jinja2
-template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
+template = {{ project_dir }}/scripts/wrapper_cros_sdk.in
 rendered = ${bin:wrapper_cros_sdk}
 md5sum = 7159fe3d5b85a283733cf686c4ee0a74
 mode = 0700
@@ -104,25 +107,32 @@ context =
 # create the rendered script in the buildout-directory/etc/run
 # (use jinja for templating)
 recipe = slapos.recipe.template:jinja2
-template = {{ software_dir }}/scripts/cros_full_build.in
+template = {{ project_dir }}/scripts/cros_full_build.in
 rendered = ${directory:run}/cros_full_build
-md5sum = f3b3b5408b69f752f2221d3965caf590
+#md5sum = f3b3b5408b69f752f2221d3965caf590
 mode = 0700
 context =
   key bash_path                         bin:bash
   key instance_log_dir                  directory:log
+  key instance_dir                      buildout:directory
   key cros_location                     directory:cros_location
   key export_path_cmd                   customize-path:command
   key branch                            parameters:configuration.branch
   key boards_list                       parameters:configuration.boards
   key keep_cache                        parameters:configuration.keep_cache
   key ebuilds_dir                       nayuos-ebuilds:location
-  raw scripts_dir                       {{ software_dir }}/scripts
-  raw logo_dir                          {{ software_dir }}/logo
-  raw wallpaper_dir                     {{ software_dir }}/wallpaper
-  raw patch_dir                         {{ software_dir }}/patch
+  raw scripts_dir                       {{ project_dir }}/scripts
+  raw logo_dir                          {{ project_dir }}/logo
+  raw wallpaper_dir                     {{ project_dir }}/wallpaper
+  raw patch_dir                         {{ project_dir }}/patch
 # packages to be copied from our overlay into ChromiuOS' overlay
 # those packages will be marked as dependency of root filesystem
   raw nayu_dev_rootfs_packages          app-misc/nayuos-chromium-policy
 # those packages will be marked as dependency of the chromium build itself
   raw nayu_dev_packages                 dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
+
+[resiliency-exclude-file]
+# Generate rdiff exclude file for resiliency
+recipe = collective.recipe.template
+input = inline: parts/chromiumos/release*
+output = ${directory:srv}/exporter.exclude

software/nayuos/software.cfg

@@ -8,19 +8,29 @@ extends =
   ../../component/libexpat/buildout.cfg
   ../../component/libaio/buildout.cfg
   ../../component/boost-lib/buildout.cfg
+  ../../component/rsync/buildout.cfg
 
 parts +=
+  eggs
   slapos-cookbook
   template-instance
   git
   curl
   thin-provisioning-tools
 
-versions = versions
+[gcc]
+min_version = 0
 
-[versions]
-slapos.recipe.template = 4.4
+[python]
+# we need python3 for repo/cros_sdk command
+part = python3
 
+[eggs]
+# name is python3 so that cros_sdk uses it automatically through "/usr/bin/env python3"
+recipe = zc.recipe.egg
+interpreter = python3
+eggs =
+  setuptools
 
 [template-instance]
 # jinja2 render instance.cfg file in the buildout directory
@@ -29,10 +39,11 @@ template = ${:_profile_base_location_}/${:filename}
 rendered = ${buildout:directory}/${:filename}
 mode = 0644
 context =
-  key software_dir            :_profile_base_location_
-  key instance_dir            buildout:directory
+  key project_dir             :_profile_base_location_
+  key software_dir            buildout:directory
   key curl_path               curl:location
   key git_path                git:location
+  key rsync_path              rsync:location
   key eggs_directory          buildout:eggs-directory
   key develop_eggs_directory  buildout:develop-eggs-directory
   key thin_provisioning_tools thin-provisioning-tools:location
@@ -56,3 +67,6 @@ dependencies =
   libexpat
   libaio
   boost-lib
+
+[versions]
+slapos.recipe.template = 4.4

software/neotest/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [neotest-env.sh]
 filename = neotest-env.sh.in
-md5sum = bd46e95f1cea62c3b0082fe8c0c9c90b
+md5sum = dd41bd736561b7d59b5ec7e0ab7cde98
 
 [neotest]
 filename = neotest.in

software/neotest/neotest-env.sh.in

@@ -9,24 +9,7 @@ PATH="${ioping:location}/bin:$PATH"
 PATH="${lmbench:location}/bin:$PATH"
 PATH="${mariadb:location}/bin:$PATH"
 PATH="${mariadb:location}/scripts:$PATH"
-PATH="${python:location}/bin:$PATH"
-
-# add all eggs and develop-eggs to py path
-#
-# XXX better leverage something like zc.recipe.egg to generate eggs path buildout way
-# XXX see e.g. https://lab.nexedi.com/nexedi/slapos/merge_requests/242#note_49644 for why
-# TODO(kirr) try doing so
-PYTHONPATH=${buildout:develop-eggs-directory}:$PYTHONPATH       # for .egg-link to be found by setuptools
-for egglink in ${buildout:develop-eggs-directory}/*.egg-link; do
-    PYTHONPATH=`cat $egglink |head -1`:$PYTHONPATH
-done
-
-for egg in ${buildout:develop-eggs-directory}/*.egg ${buildout:eggs-directory}/*.egg; do
-        export PYTHONPATH=$egg:$PYTHONPATH
-done
-
+PATH="${buildout:bin-directory}/:$PATH"  # for python with eggs
 
 export PATH
-export PYTHONPATH
-
 export PS1="(neotest-`basename ${buildout:directory}`) $PS1"

software/neotest/software.cfg

@@ -18,7 +18,6 @@ extends =
     ../../component/coreutils/buildout.cfg
     ../../component/util-linux/buildout.cfg
     ../../component/pygolang/buildout.cfg
-    ../../component/numpy/buildout.cfg
     ../../stack/nxdtest.cfg
 
 
@@ -36,8 +35,12 @@ parts =
     neoppod-develop
     neoppod
     wendelin.core-dev
-    eggs
+    scripts
 
+# for ZEO scripts (runzeo)
+    ZEO/scripts
+
+    neotest-python
     neotest-env.sh
     neotest
 
@@ -100,23 +103,21 @@ template = inline:
 
 
 
-# eggs:
-[eggs]
-recipe  = zc.recipe.egg
-eggs    =
-# wendelin.core still requires ZODB3 but having ZODB4 or 5 installed satisfies latest ZODB3
-    ZODB3
-# also for wc
-    ${numpy:egg}
-# to install not only wendelin.core modules but also scripts
-    wendelin.core
-# for ZEO scripts (runzeo)
-    ${ZEO:egg}
-# for e.g. tcpu.py
-    ${pygolang:egg}
+# bin/python is preinstalled with sys.path to wendelin.core & friends.
+[neotest-python]
+<= python-interpreter
+eggs +=
+    ${wendelin.core-dev:egg}
+    ${neoppod:eggs}
 # for instance
     plone.recipe.command
 
+[scripts]
+recipe  = zc.recipe.egg:scripts
+eggs    =
+# to install not only wendelin.core modules but also scripts
+    ${wendelin.core-dev:egg}
+
 
 # wendelin.core: latest not yet released
 [wendelin.core-repository]

software/powerdns/buildout.hash.cfg

@@ -18,15 +18,15 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
 
 [template-powerdns]
 filename = instance-powerdns.cfg
-md5sum = 9cd4e436fa432f37b9f8f4de8350581b
+md5sum = 0920200cb05a68b1b4a161a927d9488f
 
 [template-pdns-configuration]
 _update_hash_filename_ = template/pdns.conf.jinja2
-md5sum = e45d72de87b4adb89c195ba463be4077
+md5sum = 20c37ea06a8fa405bc02470d5115fd11
 
 [template-dns-replicate]
 _update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
-md5sum = a23e241a236f90ae1afbb5bd5ba0b32d
+md5sum = c2bd424f588ad57d37f4cf1329734fb6
 
 [iso-list]
 _update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -43,6 +43,7 @@ context =
 {%   set dns_name = 'ns%s' % i -%}
 {%   set dns_domain = dns_name_template_string % i %}
 {%   set request_section_title = 'request-%s' % dns_name -%}
+{%   set promise_section_title = 'promise-listen-port-%s' % dns_name -%}
 {%   set sla_key = "-sla-%s-" % i -%}
 {%   set sla_key_length = sla_key | length %}
 {%   set sla_parameters = [] %}
@@ -55,6 +56,7 @@ context =
 {%   do dns_domain_list.append(dns_domain) -%}
 {%   do dns_section_list.append(request_section_title) -%}
 {%   do part_list.append(request_section_title) -%}
+{%   do part_list.append(promise_section_title) -%}
 
 [{{request_section_title}}]
 <= replicate
@@ -69,6 +71,15 @@ config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 {%   endfor -%}
 
+[{{promise_section_title}}]
+<= monitor-promise-base
+module = check_port_listening
+name = pdns-port-listening.py
+{%   set ipv6 = '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' -%}
+config-hostname = {{ipv6}}
+{%   set port = '${' ~ request_section_title ~ ':connection-powerdns-port}' -%}
+config-port = {{port}}
+
 {%  do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
 {% endfor -%}
 
@@ -101,7 +112,7 @@ software-url = {{ slapparameter_dict.pop(dns_software_url_key) }}
 software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{dns_type}}
-return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
+return = slave-instance-information-list monitor-base-url powerdns-ipv6 powerdns-port powerdns-ipv4
 config-server-admin = {{ server_admin }}
 config-ns-record = {{ ns_record }}
 {% for parameter, value in slapparameter_dict.items() -%}
@@ -114,12 +125,17 @@ config-monitor-password = ${monitor-htpasswd:passwd}
 
 [publish-information]
 recipe = slapos.cookbook:publish
-domain = {{ slapparameter_dict.get('domain') }}
 slave-amount = {{ slave_instance_list | length }}
 ns-record = {{ ns_record }}
 monitor-url = ${monitor-publish-parameters:monitor-url}
 monitor-user = ${monitor-publish-parameters:monitor-user}
 monitor-password = ${monitor-publish-parameters:monitor-password}
+{% for dns_name, dns_section in zip(dns_list, dns_section_list) -%}
+{%   set dns_ipv6 = '${' ~ dns_section ~ ':connection-powerdns-ipv6}' -%}
+{%   set dns_port = '${' ~ dns_section ~ ':connection-powerdns-port}' -%}
+{{ dns_name }}-ipv6 = {{ dns_ipv6 }}
+{{ dns_name }}-port = {{ dns_port }}
+{% endfor -%}
 {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 {% if monitor_interface_url -%}
 monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}

software/powerdns/instance-powerdns.cfg

@@ -55,7 +55,7 @@ context =
 #
 [pdns]
 configuration = $${pdns-directory:configuration}/pdns.conf
-local-ipv4 = $${instance-parameter:ipv4-random}
+ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
 port = 5353
 socket-directory = $${pdns-directory:socket-directory}
@@ -139,7 +139,7 @@ extra-context =
 <= monitor-promise-base
 module = check_port_listening
 name = pdns-port-listening.py
-config-hostname = $${pdns:local-ipv4}
+config-hostname = $${pdns:ipv4}
 config-port = $${pdns:port}
 
 [publish-connection-informations]
@@ -147,6 +147,9 @@ recipe = slapos.cookbook:publish
 monitor-url = $${monitor-publish-parameters:monitor-url}
 monitor-user = $${monitor-publish-parameters:monitor-user}
 monitor-password = $${monitor-publish-parameters:monitor-password}
+powerdns-ipv4 = $${pdns:ipv4}
+powerdns-ipv6 = $${pdns:ipv6}
+powerdns-port = $${pdns:port}
 
 [buildout]
 parts =

software/powerdns/software.cfg

@@ -55,7 +55,6 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 4.4
-dnspython = 1.15.0
 passlib = 1.7.1
 GitPython = 2.1.11
 lockfile = 0.12.2

software/powerdns/template/pdns.conf.jinja2

 # -------------------------------------------------------------------------
 
 # Configure ip/port binding
-local-address={{ pdns.get('local-ipv4') }}
+local-address={{ pdns.get('ipv4') }}
 local-ipv6={{ pdns.get('ipv6') }}
 local-port={{ pdns.get('port') }}
 
@@ -10,7 +10,7 @@ socket-dir={{ pdns.get('socket-directory') }}
 
 # Monitoring
 webserver=yes
-webserver-address={{ pdns.get('local-ipv4') }}
+webserver-address={{ pdns.get('ipv4') }}
 webserver-port={{ pdns.get('webserver-port') }}
 
 # These totally disable query+packet caching for all zones.  This is necessary

software/powerdns/test/setup.py

@@ -47,6 +47,7 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
+          'dnspython',
       ],
       zip_safe=True,
       test_suite='test',

software/powerdns/test/test.py

@@ -25,7 +25,12 @@
 #
 ##############################################################################
 
+import dns.edns
+import dns.message
+import dns.query
+import http.client
 import os
+import requests
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -35,10 +40,64 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
+DNS_PORT = 5353
+
+AFRICAN_SUBNET = '41.0.0.0'
+CHINA_TELECOM_SUBNET = '1.0.32.0'
+CHINA_UNICOM_SUBNET = '116.181.0.0'
+CHINA_MOBILE_SUBNET = '112.21.42.5'
+EAST_ASIAN_SUBNET = '1.11.0.0'
+EUROPEAN_SUBNET = '5.42.160.0'
+HONG_KONG_SUBNET = '1.32.192.0'
+JAPANESE_SUBNET = '1.0.16.0'
+NORTH_AMERICAN_SUBNET = '3.0.0.0'
+OCEANIAN_SUBNET = '1.120.0.0'
+SOUTH_AMERICAN_SUBNET = '45.70.188.0'
+WEST_ASIAN_SUBNET = '46.70.0.0'
+
 
 class PowerDNSTestCase(SlapOSInstanceTestCase):
   # power dns uses sockets and need shorter paths on test nodes.
   __partition_reference__ = 'pdns'
+  default_zone = 'domain.com'
+
+  # focus to test connexion parameters only depending on PowerDNS
+  def getPowerDNSParameterDict(self, parameter_dict):
+    new_parameter_dict = {}
+    for key, value in parameter_dict.items():
+      if key in [
+        'ns-record',
+        'ns1-port',
+        'ns1-ipv6',
+        'slave-amount',
+      ]:
+        new_parameter_dict[key] = value
+    return new_parameter_dict
+
+  def getPowerDNSConnexionParameterDict(self):
+    return self.getPowerDNSParameterDict(
+      self.requestDefaultInstance().getConnectionParameterDict()
+    )
+
+  def _test_parameter_dict(self, zone=None, dns_quantity=1, slave_amount=0):
+    if zone is None:
+      zone = self.default_zone
+
+    parameter_dict = self.getPowerDNSConnexionParameterDict()
+    expected_dict = {
+      'ns-record': '',
+    }
+
+    ns_record = ''
+    for replicate_nb in range(1, dns_quantity + 1):
+      prefix = 'ns%s' % replicate_nb
+      ns_record += prefix + '.%s' % zone
+      expected_dict[prefix + '-port'] = str(DNS_PORT)
+      expected_dict[prefix + '-ipv6'] = self._ipv6_address
+    expected_dict['ns-record'] = ns_record
+    expected_dict['slave-amount'] = str(slave_amount)
+
+    self.assertEqual(expected_dict, parameter_dict)
 
 
 class ServicesTestCase(PowerDNSTestCase):
@@ -63,3 +122,194 @@ class ServicesTestCase(PowerDNSTestCase):
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)
+
+
+class TestMonitorAccess(PowerDNSTestCase):
+
+  def test(self):
+    connection_parameter_dict = self.requestDefaultInstance()\
+      .getConnectionParameterDict()
+    monitor_base_url = connection_parameter_dict.get('monitor-url')
+    result = requests.get(
+      monitor_base_url, verify=False, auth=(
+        connection_parameter_dict.get('monitor-user'),
+        connection_parameter_dict.get('monitor-password')
+      )
+    )
+
+    self.assertEqual(
+      http.client.OK,
+      result.status_code
+    )
+
+
+class TestMasterRequest(PowerDNSTestCase):
+
+  def test(self):
+    self._test_parameter_dict()
+
+
+class TestMasterRequestDomain(PowerDNSTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'zone': 'toto.example.com',
+    }
+
+  def test(self):
+    self._test_parameter_dict(zone=self.getInstanceParameterDict()['zone'])
+
+
+class PowerDNSSlaveTestCase(PowerDNSTestCase):
+
+  @classmethod
+  def requestDefaultInstance(cls):
+    default_instance = super(PowerDNSSlaveTestCase, cls)\
+        .requestDefaultInstance()
+    cls.requestSlaves()
+    return default_instance
+
+  @classmethod
+  def requestSlaves(cls):
+    software_url = cls.getSoftwareURL()
+    software_type = cls.getInstanceSoftwareType()
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      cls.logger.debug(
+        'requesting slave "%s" software:%s parameters:%s',
+        slave_reference, software_url, partition_parameter_kw)
+      cls.slap.request(
+        software_release=software_url,
+        software_type=software_type,
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      )
+
+  @classmethod
+  def getSlaveConnectionParameterDictList(cls):
+    parameter_dict_list = []
+
+    for slave_reference, partition_parameter_kw in cls\
+            .getSlaveParameterDictDict().items():
+      parameter_dict_list.append(cls.slap.request(
+        software_release=cls.getSoftwareURL(),
+        software_type=cls.getInstanceSoftwareType(),
+        partition_reference=slave_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        shared=True
+      ).getConnectionParameterDict())
+    return parameter_dict_list
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'slave-pdns1': {
+        'record': 'test1',
+        'origin': 'nexedi.com',
+        'default': 'test1.com.',
+        'africa': 'test1africa.com.',
+        'china-telecom': 'test1china-telecom.com.',
+        'china-unicom': 'test1china-unicom.com.',
+        'china-mobile': 'test1china-mobile.com.',
+        'east-asia': 'test1east-asia.com.',
+        'europe': 'test1europe.com.',
+        'hong-kong': 'test1hong-kong.com.',
+        'japan': 'test1japan.com.',
+        'north-america': 'test1north-america.com.',
+        'oceania': 'test1oceania.com.',
+        'south-america': 'test1south-america.com.',
+        'west-asia': 'test1west-asia.com.',
+      },
+      'slave-pdns2': {
+        'record': 'test2',
+        'origin': 'nexedi.com',
+        'default': 'test2.com.',
+        'china-telecom': 'test2china-telecom.com.',
+        'europe': 'test2europe.com.',
+        'japan': 'test2japan.com.',
+      }
+    }
+
+  def dns_query(self, domain_name, subnet):
+    message = dns.message.make_query(domain_name, 'A')
+    client_subnet_option = dns.edns.ECSOption(subnet)
+    message.use_edns(options=[client_subnet_option])
+    answer = dns.query.udp(message, self._ipv6_address, port=DNS_PORT)
+    return answer.find_rrset(
+            dns.message.ANSWER,
+            dns.name.from_text(domain_name),
+            dns.rdataclass.IN,
+            dns.rdatatype.CNAME
+          ).to_text().split()[-1]
+
+  def _test_dns_resolver(self, zone):
+    slave_parameter_dict_dict = self.getSlaveParameterDictDict()
+    subnet_dict = {
+      'africa': AFRICAN_SUBNET,
+      'china-telecom': CHINA_TELECOM_SUBNET,
+      'china-unicom': CHINA_UNICOM_SUBNET,
+      'china-mobile': CHINA_MOBILE_SUBNET,
+      'east-asia': EAST_ASIAN_SUBNET,
+      'europe': EUROPEAN_SUBNET,
+      'hong-kong': HONG_KONG_SUBNET,
+      'japan': JAPANESE_SUBNET,
+      'north-america': NORTH_AMERICAN_SUBNET,
+      'oceania': OCEANIAN_SUBNET,
+      'south-america': SOUTH_AMERICAN_SUBNET,
+      'west-asia': WEST_ASIAN_SUBNET,
+    }
+    default_rr_dict = {
+      'europe': 'eu',
+      'africa': 'af',
+      'south-america': 'sa',
+      'north-america': 'na',
+      'china-telecom': 'cn-t',
+      'china-unicom': 'cn-u',
+      'china-mobile': 'cn-m',
+      'japan': 'jp',
+      'hong-kong': 'hk',
+      'east-asia': 'as',
+      'west-asia': 'eu',
+      'oceania': 'oc',
+    }
+
+    for slave_name in slave_parameter_dict_dict:
+      slave_parameter_dict = slave_parameter_dict_dict[slave_name]
+      domain_name = '%s.%s' % (slave_parameter_dict['record'], zone)
+      for region in subnet_dict:
+        self.assertEqual(
+          slave_parameter_dict.pop(
+            region,
+            '%s.%s.' % (default_rr_dict[region], slave_parameter_dict['origin'])
+          ),
+          self.dns_query(domain_name, subnet_dict[region])
+        )
+
+  def _test(self, zone=None):
+    if zone is None:
+      zone = self.default_zone
+    self._test_parameter_dict(
+      zone=zone,
+      slave_amount=len(self.getSlaveParameterDictDict())
+    )
+    self._test_dns_resolver(zone)
+
+
+class TestSlaveRequest(PowerDNSSlaveTestCase):
+
+  def test(self):
+    self._test()
+
+
+class TestSlaveRequestDomain(PowerDNSSlaveTestCase):
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'zone': 'toto.example.com',
+    }
+
+  def test(self):
+    self._test(zone=self.getInstanceParameterDict()['zone'])

software/slapos-sr-testing/README.md

@@ -31,15 +31,15 @@ slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
 # and load this script to set environment variables
 source ( environment-script from step above )
 
-# Clone a working copy somewhere
-cd ~/srv/runner/project/
-git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
+# The source code is a git clone working copy on the instance
+cd ~/srv/runner/instance/slappartXXX/parts/slapos/
 
 # change directory to the directory containing test for this software
-cd ~/srv/runner/project/slapos_work/software/helloworld/test/
+cd ./software/helloworld/test/
+# make changes to test code or profile
 
-# run test (with debugging features activated)
-SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test
+# run test for helloworld software release (with debugging features activated)
+SLAPOS_TEST_DEBUG=1 python_for_test -m unittest discover -v
 ```
 
 ## Environment variables

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 4246cde0a27138e057ba1635cc621edf
+md5sum = 9639d0c0e161c094454808fb95fc9781

software/slapos-sr-testing/instance.cfg

 [buildout]
+extends = {{ nxdtest_instance }}
 parts =
-  slapos-test-runner
+  runTestSuite
   publish
 
 eggs-directory = {{ buildout['eggs-directory'] }}
@@ -26,46 +27,53 @@ repository = {{ slapos_location }}
 [directory]
 recipe = slapos.cookbook:mkdirectory
 bin = ${buildout:directory}/bin
-working-dir = ${buildout:directory}/tmp
 etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+test-working-dir = ${buildout:directory}/tmp
+nxdtest-working-dir = ${:var}/nxdtest
 
-[slapos-test-runner]
-recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:bin}/runTestSuite
-command-line =
-  {{ buildout['bin-directory'] }}/runTestSuite
-  --python_interpreter={{ buildout['bin-directory'] }}/{{ interpreter }}
-  --source_code_path_list={{ ','.join(tests.splitlines()) }}
+[slapos-test-runner-nxdtest-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:etc}/${:_buildout_section_name_}
+template = inline:
+  export PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
+  export SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
+  export SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
+  export SLAPOS_TEST_WORKING_DIR=${directory:test-working-dir}
 
-environment =
-  PATH=${slapos-test-runner-environment:PATH}
-  SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
-  SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
-  SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
+[slapos-test-runner-dot-nxdtest]
+recipe = slapos.recipe.template:jinja2
+rendered = ${:workdir}/.nxdtest
+workdir = ${directory:nxdtest-working-dir}
+template = inline:
+  {% for test in tests.splitlines() %}
+    TestCase(
+      {{ repr(test.split()[0]) }},
+      [ {{ repr(interpreter) }}, '-m', 'unittest', 'discover', '-v'],
+      cwd={{ repr(test.split()[1]) }},
+      summaryf=UnitTest.summary,
+    )
+  {% endfor %}
 
-[slapos-test-runner-environment]
-PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:{{ openssl_location }}/bin/:/usr/bin/:/bin
-SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
-SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
-SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
+[runTestSuite]
+env.sh = ${slapos-test-runner-nxdtest-environment.sh:rendered}
+workdir = ${slapos-test-runner-dot-nxdtest:workdir}
 
 [slapos-local-development-environment.sh]
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:etc}/${:_buildout_section_name_}
 template = inline:
-  export PATH=${slapos-test-runner-environment:PATH}:$PATH
-  export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
-  export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
-  export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
+  source ${slapos-test-runner-nxdtest-environment.sh:rendered}
   {% set shared_part_list = [] %}
   {% for shared_part in buildout['shared-part-list'].splitlines() -%}
     {% do shared_part_list.append(shared_part) %}
   {%- endfor %}
   export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
   echo "Environment loaded."
+  echo "The embedded SlapOS is in $SLAPOS_TEST_WORKING_DIR"
   echo "To work on a test, execute:"
-  echo "   SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test"
-  echo "from test folder"
+  echo "   SLAPOS_TEST_DEBUG=1 {{ interpreter }} -m unittest discover -v"
+  echo "from a folder containing software release test."
   echo
 
 [publish]

software/slapos-sr-testing/software-py3.cfg

@@ -12,9 +12,9 @@ eggs +=
 
 [template]
 extra =
-  ${slapos.test.helloworld-setup:setup}
-  ${slapos.test.monitor-setup:setup}
-  ${slapos.test.plantuml-setup:setup}
-  ${slapos.test.powerdns-setup:setup}
-  ${slapos.test.proftpd-setup:setup}
-  ${slapos.test.repman-setup:setup}
+  helloworld ${slapos.test.helloworld-setup:setup}
+  monitor ${slapos.test.monitor-setup:setup}
+  plantuml ${slapos.test.plantuml-setup:setup}
+  powerdns ${slapos.test.powerdns-setup:setup}
+  proftpd ${slapos.test.proftpd-setup:setup}
+  repman ${slapos.test.repman-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -13,11 +13,12 @@ extends =
   ../../component/python-backports-lzma/buildout.cfg
 
   ../../stack/slapos.cfg
+  ../../stack/nxdtest.cfg
 
   ./buildout.hash.cfg
 
 parts =
-  eggs
+  eggs/scripts
   slapos-cookbook
   template
 
@@ -167,8 +168,8 @@ egg = slapos.core
 setup = ${slapos.core-repository:location}
 
 [eggs]
-recipe = zc.recipe.egg
-eggs =
+<= python-interpreter
+eggs +=
   ${lxml-python:egg}
   ${slapos.core-setup:egg}
   ${pillow-python:egg}
@@ -202,21 +203,24 @@ eggs =
   ${slapos.test.dream-setup:egg}
   ${slapos.test.metabase-setup:egg}
   ${slapos.test.repman-setup:egg}
-entry-points =
-  runTestSuite=erp5.util.testsuite:runTestSuite
-scripts =
-  runTestSuite
-  slapos
-  supervisorctl
-  supervisord
-interpreter=
-  python_for_test
+
+# We don't name this interpreter `python`, so that when we run slapos node
+# software, installation scripts running `python` use a python without any
+# custom eggs pre-installed, not our special python interpreter.
+interpreter = python_for_test
 
 # patches for eggs
 patch-binary = ${patch:location}/bin/patch
 PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/0001-Custom-implementation-of-warnings.formatwarning-remo.patch#d25bb0f5dde7f3337a0a50c2f986f5c8
 PyPDF2-patch-options = -p1
 
+[eggs/scripts]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
+scripts =
+  slapos
+  supervisord
+
 [git-clone-repository]
 recipe = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
@@ -236,6 +240,7 @@ mode = 640
 context =
   section buildout buildout
   import os os
+  key nxdtest_instance nxdtest-instance.cfg:rendered
   key git_location git:location
   key slapos_location slapos-repository:location
   key interpreter eggs:interpreter
@@ -243,29 +248,31 @@ context =
   key openssl_location openssl-output:bin
   key faketime_location faketime:location
   key tests :tests
+
 tests =
-  ${slapos.test.kvm-setup:setup}
-  ${slapos.test.slaprunner-setup:setup}
-  ${slapos.test.metabase-setup:setup}
+  json-schemas ${slapos.cookbook-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
+  slaprunner ${slapos.test.slaprunner-setup:setup}
+  metabase ${slapos.test.metabase-setup:setup}
   ${:extra}
 extra =
-  ${slapos.cookbook-setup:setup}
-  ${slapos.test.backupserver-setup:setup}
-  ${slapos.test.caddy-frontend-setup:setup}
-  ${slapos.test.erp5-setup:setup}
-  ${slapos.test.htmlvalidatorserver-setup:setup}
-  ${slapos.test.slapos-master-setup:setup}
-  ${slapos.test.re6stnet-setup:setup}
-  ${slapos.test.seleniumserver-setup:setup}
-  ${slapos.test.jstestnode-setup:setup}
-  ${slapos.test.jupyter-setup:setup}
-  ${slapos.test.nextcloud-setup:setup}
-  ${slapos.test.turnserver-setup:setup}
-  ${slapos.test.theia-setup:setup}
-  ${slapos.test.grafana-setup:setup}
-  ${slapos.test.gitlab-setup:setup}
-  ${slapos.test.cloudooo-setup:setup}
-  ${slapos.test.dream-setup:setup}
+  backupserver ${slapos.test.backupserver-setup:setup}
+  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
+  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
+  slapos-master ${slapos.test.slapos-master-setup:setup}
+  re6stnet ${slapos.test.re6stnet-setup:setup}
+  seleniumserver ${slapos.test.seleniumserver-setup:setup}
+  jstestnode ${slapos.test.jstestnode-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
+  nextcloud ${slapos.test.nextcloud-setup:setup}
+  turnserver ${slapos.test.turnserver-setup:setup}
+  theia ${slapos.test.theia-setup:setup}
+  grafana ${slapos.test.grafana-setup:setup}
+  gitlab ${slapos.test.gitlab-setup:setup}
+  cloudooo ${slapos.test.cloudooo-setup:setup}
+  dream ${slapos.test.dream-setup:setup}
+  repman ${slapos.test.repman-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/slapos-testing/README.md

@@ -2,13 +2,11 @@
 
 This software release is used to run unit test of slapos eggs.
 
-The approach is to use setuptools' integrated test runner, `python setup.py test`, to run tests.
+The approach is to use nxdtest test runner, which will run tests for each
+projects, as described in `.nxdtest` file.
 
-The `python` used in this command will be a `zc.recipe.egg` interpreter with
-all eggs pre-installed by this software release.
-
-The results of this test suite running on Nexedi ERP5 are published as `SlapOS.Eggs.UnitTest-Master.Python3`
-and `SlapOS.Eggs.UnitTest-Master.Python2`.
+The results of this test suite running on Nexedi ERP5 are published as 
+`SlapOS.Eggs.UnitTest-Master.Python3` and `SlapOS.Eggs.UnitTest-Master.Python2`.
 
 
 Here's an example session of how a developer could use this software release in
@@ -23,21 +21,27 @@ INSTANCE_NAME=$COMP
 
 slapos supply $SR $COMP
 slapos node software
-slapos request --node=node=$COMP $INSTANCE_NAME $SR
+slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
 slapos node instance
 
+# The path of a an environment script was published by slapos parameters, as
+# "environment-script"
+slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
+
+# sourcing the script in the shell configure all environment variables and
+# print a message explaining how to run tests
+source ( environment script from step above )
+
 # The source code is a git clone working copy on the instance
-cd ~/srv/runner/instance/slappart0/parts/slapos.core/
+cd ~/srv/runner/instance/slappartXXX/parts/slapos.core/
 
 # make some changes to the code
 vim slapos/tests/client.py
 
-# run tests, using bundled python intepreter with pre-installed eggs dependencies
-SLAPOS_TEST_IPV6=::1 \
-SLAPOS_TEST_IPV4=127.0.0.1 \
-SLAPOS_TEST_VERBOSE=1 \
-SLAPOS_TEST_DEBUG=1 \
-~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
+# run slapos.core tests
+runTestSuite --run slapos.core
+# ... or run all eggs tests
+runTestSuite
 
 # when satified, commit changes
 git add -p && git commit

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5
+md5sum = 3b1b386f6ad4c9ac50ab1f1e1384e751

software/slapos-testing/instance.cfg

 [buildout]
+extends = ${nxdtest-instance.cfg:rendered}
 parts =
-  phantomjs-wrapper
-  slapos-test-runner
+  runTestSuite
+  publish
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -21,6 +22,8 @@ bin = $${buildout:directory}/bin
 etc = $${buildout:directory}/etc
 services = $${:etc}/run
 srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+nxdtest-working-dir = $${:var}/nxdtest
 
 [download-source]
 recipe = slapos.recipe.build:gitclone
@@ -75,30 +78,127 @@ repository = ${slapos.rebootstrap-repository:location}
 repository = ${rubygemsrecipe-repository:location}
 
 
-[slapos-test-runner]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${create-directory:bin}/runTestSuite
-command-line =
-  ${buildout:bin-directory}/runTestSuite
-  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
-  --source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location}
-
-# Notes about environment:
-# * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests
-#   needs ping, which is a setuid binary that cannot be installed via slapos
-#   way of building software without root access, so we keep "standard"
-#   /usr/bin and /bin in $PATH
-# * SLAPOS_TEST_environment is safe to be used by tests, but there is no
-#   guarantee about free ports on IPV4 and IPV6
-# * LOCAL_IPV4 is backward compatible, to be migrated, SLAPOS_TEST_IPV4
-environment =
-  PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:/usr/bin/:/bin/
-  LOCAL_IPV4=$${slap-configuration:ipv4-random}
-  SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
-  SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
-
-
-[phantomjs-wrapper]
-recipe = slapos.cookbook:wrapper
-command-line = ${phantomjs:location}/phantomjs-slapos
-wrapper-path = $${create-directory:bin}/phantomjs
+[slapos-test-runner-nxdtest-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = $${create-directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  export PATH=${coreutils:location}/bin:${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${socat:location}/bin:${lmsensors:location}/bin:${rsync:location}/bin/:${buildout:bin-directory}:$PATH
+  export SLAPOS_TEST_IPV4=$${slap-configuration:ipv4-random}
+  export SLAPOS_TEST_IPV6=$${slap-configuration:ipv6-random}
+
+
+[slapos-test-runner-dot-nxdtest]
+recipe = slapos.recipe.template:jinja2
+rendered = $${:workdir}/.nxdtest
+workdir = $${create-directory:nxdtest-working-dir}
+
+template = inline:
+  TestCase(
+      "kedifa",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${kedifa:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "caucase",
+      # XXX caucase uses 2to3 dynamically in setup.py, so it only supports
+      # runnning tests with python setup.py test
+      ['python', 'setup.py', 'test'],
+      cwd="""$${caucase:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "erp5.util",
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'erp5/tests/'],
+      cwd="""$${erp5.util:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.cookbook",
+      # slapos/test contain both tests for recipes and tests for
+      # json schemas, we only care about recipe tests here, json
+      # schemas are tested in software/slapos-sr-testing
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test/recipe'],
+      cwd="""$${slapos.cookbook:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.core",
+      # ['python', '-m', 'unittest', 'discover', '-v'],
+      # XXX some test fail when running with unittest (slapos*.cfg.example
+      # files cannot be found with pkg_resources.resource_string), so we keep
+      # using setup.py test for now.
+      ['python', 'setup.py', 'test'],
+      cwd="""$${slapos.core:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.build",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${slapos.recipe.build:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.cmmi",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${slapos.recipe.cmmi:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.recipe.template",
+      # ['python', '-m', 'unittest', 'slapos.recipe.template.tests.test_suite'],
+      # XXX some test fail when running with unittest, so we keep using setup.py test
+      ['python', 'setup.py', 'test'],
+      cwd="""$${slapos.recipe.template:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.toolbox",
+      # ['python', '-m', 'unittest', 'discover', '-v'],
+      # XXX We can not just run unittest discover here, since slapos/lamp
+      # imports MySQLDb and we currently don't have installed
+      # slapos.toolbox[lampconfigure] and on python3 discovery make a
+      # failing test for this import error.
+      # Currently slapos/lamp is not tested, but if it it is still used,
+      # the TODO seem to also install slapos.toolbox[lampconfigure] here.
+      ['python', '-m', 'unittest', 'discover', '-v', '--start', 'slapos/test', '--top-level-directory', '.'],
+      cwd="""$${slapos.toolbox:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.libnetworkcache",
+      ['python', '-m', 'unittest', '-v', 'slapos.libnetworkcachetests'],
+      cwd="""$${slapos.libnetworkcache:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "slapos.rebootstrap",
+      ['python', '-m', 'unittest', '-v', 'slapos.rebootstrap.tests.test_suite'],
+      cwd="""$${slapos.rebootstrap:location}""",
+      summaryf=UnitTest.summary,
+  )
+  TestCase(
+      "rubygemsrecipe",
+      ['python', '-m', 'unittest', 'discover', '-v'],
+      cwd="""$${rubygemsrecipe:location}""",
+      summaryf=UnitTest.summary,
+  )
+
+[runTestSuite]
+env.sh = $${slapos-test-runner-nxdtest-environment.sh:rendered}
+workdir = $${slapos-test-runner-dot-nxdtest:workdir}
+
+[slapos-local-development-environment.sh]
+recipe = slapos.recipe.template:jinja2
+rendered = $${create-directory:etc}/$${:_buildout_section_name_}
+template = inline:
+  source $${slapos-test-runner-nxdtest-environment.sh:rendered}
+  echo "Environment loaded."
+  echo "To work on a test, execute:"
+  echo "   $${runTestSuite:wrapper-path} -k test_name"
+  echo "replacing test_name by the name of the test."
+  echo
+
+[publish]
+recipe = slapos.cookbook:publish
+environment-script = $${slapos-local-development-environment.sh:rendered}

software/slapos-testing/software.cfg

@@ -14,11 +14,12 @@ extends =
   ../../component/lmsensors/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../stack/slapos.cfg
+  ../../stack/nxdtest.cfg
   ./buildout.hash.cfg
 
 parts =
   bootstrap-slapos.recipe.cmmi
-  eggs
+  eggs/scripts
   phantomjs
   template
 
@@ -103,8 +104,8 @@ egg = rubygemsrecipe[test]
 setup = ${rubygemsrecipe-repository:location}
 
 [eggs]
-recipe = zc.recipe.egg
-eggs =
+<= python-interpreter
+eggs +=
   ${lxml-python:egg}
   ${python-cryptography:egg}
   ${backports.lzma:egg}
@@ -126,14 +127,13 @@ eggs =
   ${rubygemsrecipe-setup:egg}
   zope.testing
   supervisor
-entry-points =
-  runTestSuite=erp5.util.testsuite:runTestSuite
+
+[eggs/scripts]
+recipe = zc.recipe.egg
+eggs = ${eggs:eggs}
 scripts =
-  runTestSuite
   slapos
   supervisord
-interpreter=
-  python_for_test
 
 [git-clone-repository]
 recipe = slapos.recipe.build:gitclone

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 6e279c46b07bf56b7b037a8ee2c6587e
+md5sum = 2582723c31166244ff25cb3d8c839ffa
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -46,6 +46,7 @@ common-runner-parts =
   custom-frontend-promise
 {% endif %}
 {% endif %}
+  project-link
 ## Monitoring part
   monitor-base
   monitor-check-webrunner-internal-instance
@@ -159,6 +160,11 @@ software-test = $${:test}/software
 instance-test = $${:test}/instance
 sessions = $${buildout:directory}/.sessions
 
+[project-link]
+recipe = slapos.cookbook:symbolic.link
+target-directory = $${directory:srv}
+link-binary = $${runnerdirectory:project}
+
 [slaprunner]
 slaprunner = ${buildout:bin-directory}/slaprunner
 slapos = ${buildout:bin-directory}/slapos

software/theia/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 2ceb9389281c00261abd864fc8ed566f
+md5sum = 1c60191f8724854f979a17d2624e65d8
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
+md5sum = 5a89742266a9f9d4115efa6d641fd5bb
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt

software/theia/instance.cfg.in

@@ -173,7 +173,7 @@ template =
         ],
         "options": {
           "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
             "GIT_EXEC_PATH": ""
           }
         },
@@ -197,7 +197,7 @@ template =
         ],
         "options": {
           "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-activate:slapos-configuration}",
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
             "GIT_EXEC_PATH": ""
           }
         },
@@ -227,6 +227,7 @@ template =
   export TMP=$${directory:tmp}
   export TEMP=$TMP
   export LC_ALL=C.UTF-8
+  export TERMINFO=${ncurses:location}/lib/terminfo/
   export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
   exec ${theia-wrapper:rendered} $@
 ip =  $${instance-parameter:ipv4-random}
@@ -241,7 +242,7 @@ hash-existing-files =
   ${yarn.lock:output}
   ${theia-wrapper:rendered}
 ip =  $${instance-parameter:ipv4-random}
-hostname =  $${:ip}
+hostname = $${:ip}
 port = $${theia-service:port}
 base-url = $${theia-service:base-url}
 
@@ -256,36 +257,76 @@ template = inline:
   import time
   args = sys.argv[1:]
   # when running interactively, activate slapos configuration and reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
-  if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash", ]
+  if not args: args = ["-c", ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash --rcfile $${theia-bashrc:rendered}", ]
   # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
   else: time.sleep(1)
   os.execv('${bash:location}/bin/bash', ['${bash:location}/bin/bash']  + args)
 
+[theia-bashrc]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:etc}/$${:_buildout_section_name_}
+template =
+  inline:
+  # enable bash completion
+  . ${bash-completion:location}/etc/profile.d/bash_completion.sh
+  # source user's .bashrc
+  [ -f ~/.bashrc ] && . ~/.bashrc
+depends =
+  $${shell-setup-completion:recipe}
+
+[shell-setup-completion]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  ${buildout:bin-directory}/slapos complete > $${directory:bash-completions}/slapos
+  ${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
+
+
+[slapos-standalone-config]
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+port = 4000
+slapos-configuration = $${directory:runner}/etc/slapos.cfg
+computer-id = slaprunner
+
 [slapos-standalone-activate]
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:bin}/$${:_buildout_section_name_}
 mode = 0700
-# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
 template =
   inline:#!/bin/sh
   export PATH=${buildout:bin-directory}:$PATH
-  ${slapos-standalone:script-path} \
-      $${directory:slapos} \
-      $${:ipv4} \
-      $${:ipv6} \
-      $${:port} \
+  export SLAPOS_CONFIGURATION=$${slapos-standalone-config:slapos-configuration}
+  export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
+  echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
+
+[slapos-standalone]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  export PATH=${buildout:bin-directory}:$PATH
+  exec ${slapos-standalone:script-path} \
+      $${directory:runner} \
+      $${slapos-standalone-config:ipv4} \
+      $${slapos-standalone-config:ipv6} \
+      $${slapos-standalone-config:port} \
+      $${slapos-standalone-config:computer-id} \
       $${slap-connection:server-url} \
       $${slap-connection:computer-id} \
       $${slap-connection:partition-id} \
       --key='$${slap-connection:key-file}' \
       --cert='$${slap-connection:cert-file}'
-  export SLAPOS_CONFIGURATION=$${:slapos-configuration}
-  export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
 
-ipv4 = $${instance-parameter:ipv4-random}
-ipv6 = $${instance-parameter:ipv6-random}
-port = 4000
-slapos-configuration = $${directory:slapos}/etc/slapos.cfg
+[slapos-standalone-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line = $${slapos-standalone:rendered}
+hash-files =
+  $${slapos-standalone:rendered}
+hostname = $${slapos-standalone-config:ipv4}
+port = $${slapos-standalone-config:port}
 
 [promises]
 recipe =
@@ -293,6 +334,7 @@ instance-promises =
   $${theia-listen-promise:name}
   $${frontend-listen-promise:name}
   $${apache-frontend-url-available-promise:name}
+  $${slapos-standalone-listen-promise:name}
 
 [theia-listen-promise]
 <= monitor-promise-base
@@ -315,6 +357,14 @@ name = $${:_buildout_section_name_}.py
 config-url = $${apache-frontend:connection-secure_access}
 config-check-secure = 1
 
+[slapos-standalone-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+# XXX promise plugins can not contain "slapos" in their names
+name = standalone-listen-promise.py
+config-hostname = $${slapos-standalone-instance:hostname}
+config-port = $${slapos-standalone-instance:port}
+
 [apache-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -353,8 +403,11 @@ dot-theia = $${buildout:directory}/.theia/
 pidfiles = $${:var}/run
 
 services = $${:etc}/service
+runner = $${:srv}/runner
 project = $${:srv}/project
-slapos = $${:srv}/slapos
 frontend-static = $${:srv}/frontend-static
 frontend-static-public = $${:frontend-static}/public
 frontend-static-css = $${:frontend-static}/css
+
+bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
+fish-completions = $${buildout:directory}/.config/fish/completions/

software/theia/software.cfg

@@ -4,6 +4,7 @@ extends =
    ../../component/caddy/buildout.cfg
    ../../component/git/buildout.cfg
    ../../component/bash/buildout.cfg
+   ../../component/bash-completion/buildout.cfg
    ../../component/fish-shell/buildout.cfg
    ../../component/tmux/buildout.cfg
    ../../component/tig/buildout.cfg
@@ -55,9 +56,11 @@ entry-points =
   ${:scripts}=not_used:main
 initialization =
   import argparse
+  import glob
   import os.path
   import sys
-  import glob
+  import signal
+  import time
 
   import slapos.slap.standalone
 
@@ -66,6 +69,7 @@ initialization =
   parser.add_argument('ipv4')
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
+  parser.add_argument('computer_id')
   forwarded_arguments = parser.add_argument_group('forwarded')
   forwarded_arguments.add_argument('master_url')
   forwarded_arguments.add_argument('computer')
@@ -91,7 +95,10 @@ initialization =
       args.base_directory,
       args.ipv4,
       args.server_port,
+      computer_id=args.computer_id,
       shared_part_list=shared_part_list,
+      software_root="%s/software" % args.base_directory,
+      instance_root="%s/instance" % args.base_directory,
       partition_forward_configuration=partition_forward_configuration,
   )
   standalone.start()
@@ -104,7 +111,26 @@ initialization =
         args.ipv4,
         args.ipv6
     )
-  print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
+  print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
+  # Run instance at least once, to start the supervisor managing instances.
+  try:
+    standalone.waitForInstance(max_retry=0)
+  except slapos.slap.standalone.SlapOSNodeCommandError as e:
+    print("Error instanciating: {}".format(e))
+
+  quit_requested = []
+  def signal_handler(signum, frame):
+    print("Signal {signum} received".format(signum=signum))
+    quit_requested.append(True)
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  print("Standalone SlapOS ready")
+  while not quit_requested:
+    time.sleep(.1)
+
+  print("Stopping standalone subsystem")
+  standalone.stop()
+  print("Exiting")
   sys.exit(0)
 
 needs-these-eggs-scripts-in-path =
@@ -343,7 +369,7 @@ command =
 
 
 [cli-utilities]
-PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/
+PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
 
 [theia-wrapper]
 recipe = slapos.recipe.template:jinja2

software/theia/test/test.py

@@ -36,9 +36,13 @@ import re
 from six.moves.urllib.parse import urlparse, urljoin
 
 import pexpect
+import psutil
 import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.grid.svcbackend import getSupervisorRPC
+from slapos.grid.svcbackend import _getSupervisordSocketPath
+
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
@@ -110,12 +114,11 @@ class TestTheia(SlapOSInstanceTestCase):
         pass
     process.logfile = DebugLogFile()
 
-    process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
-    process.expect_exact('Standalone SlapOS for computer `local` activated')
+    process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
 
     # try to supply and install a software to check that this slapos is usable
     process.sendline(
-        'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local'
+        'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg slaprunner'
     )
     process.expect(
         'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
@@ -135,12 +138,6 @@ class TestTheia(SlapOSInstanceTestCase):
     # interrupt this, we don't want to actually wait for software installation
     process.sendcontrol('c')
 
-    # shutdown this slapos
-    process.sendline(
-        'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
-            self.computer_partition_root_path))
-    process.expect('Shut down')
-
     process.terminate()
     process.wait()
 
@@ -153,3 +150,37 @@ class TestTheia(SlapOSInstanceTestCase):
         'touch "{}"'.format(test_file)
     ])
     self.assertTrue(os.path.exists(test_file))
+
+
+class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for sockets in included slapos
+
+  def test_stopping_instance_stops_embedded_slapos(self):
+    embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'slapos',
+            'inst',
+        ), self.logger)
+
+    # Wait a bit for this supervisor to be started.
+    for _ in range(20):
+      if os.path.exists(embedded_slapos_supervisord_socket):
+        break
+      time.sleep(1)
+
+    # get the pid of the supervisor used to manage instances
+    with getSupervisorRPC(embedded_slapos_supervisord_socket) as embedded_slapos_supervisor:
+      embedded_slapos_process = psutil.Process(embedded_slapos_supervisor.getPID())
+
+    # Stop theia's services
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      process_info, = [
+          p for p in instance_supervisor.getAllProcessInfo()
+          if p['name'].startswith('slapos-standalone-instance-')
+      ]
+      instance_supervisor.stopProcessGroup(process_info['group'])
+
+    # the supervisor controlling instances is also stopped
+    self.assertFalse(embedded_slapos_process.is_running())

stack/nxdtest.cfg

@@ -32,7 +32,7 @@
 [buildout]
 extends =
     slapos.cfg
-    ../component/git/buildout.cfg
+    ../component/nxdtest/buildout.cfg
     nxdtest/buildout.hash.cfg
 
 [jinja2-template]
@@ -52,25 +52,5 @@ context =
   section nxdtest   nxdtest
 
 
-[nxdtest]
-recipe  = zc.recipe.egg:scripts
-eggs    = ${nxdtest-egg:egg}
-scripts = nxdtest
-# convenience for nxdtest users
-exe     = ${buildout:bin-directory}/nxdtest
-
-[nxdtest-egg]
-recipe  = zc.recipe.egg:develop
-setup   = ${nxdtest-repository:location}
-egg     = nxdtest
-
-[nxdtest-repository]
-recipe  = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = bd91f6f1579a
-location = ${buildout:parts-directory}/nxdtest
-git-executable = ${git:location}/bin/git
-
-
 [versions]
 slapos.recipe.template = 4.4

stack/slapos.cfg

@@ -164,6 +164,7 @@ dateparser = 0.7.6
 decorator = 4.3.0
 funcsigs = 1.0.2
 gevent = 20.9.0
+geventmp = 0.0.1
 greenlet = 0.4.17
 idna = 2.9
 importlib-metadata = 1.7.0
@@ -191,7 +192,7 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.167
-slapos.core = 1.6.2
+slapos.core = 1.6.3
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
@@ -247,7 +248,7 @@ croniter = 0.3.25
 
 # Required by:
 # slapos.toolbox==0.94
-dnspython = 1.15.0
+dnspython = 1.16.0
 
 # Required by:
 # cryptography==1.8.1