Update Release Candidate

component/bcrypt/buildout.cfg

 [buildout]
 extends =
   ../libffi/buildout.cfg
+  ../python-cffi/buildout.cfg
 
 parts =
   bcrypt
@@ -16,3 +17,5 @@ egg = bcrypt
 rpath =
   ${libffi:location}/lib/
 environment = bcrypt-env
+setup-eggs =
+  ${python-cffi:egg}

component/cyrus-sasl/0001-db_gdbm-fix-gdbm_errno-overlay-from-gdbm_close.patch

+From af48f6fec9a7b6374d4153c5db894d4a1f349645 Mon Sep 17 00:00:00 2001
+From: Jonas Jelten <jj@sft.mx>
+Date: Sat, 2 Feb 2019 20:53:37 +0100
+Subject: [RFC] db_gdbm: fix gdbm_errno overlay from gdbm_close
+
+`gdbm_close` also sets gdbm_errno since version 1.17.
+This leads to a problem in `libsasl` as the `gdbm_close` incovation overlays
+the `gdbm_errno` value which is then later used for the error handling.
+---
+ sasldb/db_gdbm.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/sasldb/db_gdbm.c b/sasldb/db_gdbm.c
+index ee56a6b..c908808 100644
+--- a/sasldb/db_gdbm.c
++++ b/sasldb/db_gdbm.c
+@@ -107,9 +107,11 @@ int _sasldb_getdata(const sasl_utils_t *utils,
+   gkey.dptr = key;
+   gkey.dsize = key_len;
+   gvalue = gdbm_fetch(db, gkey);
++  int fetch_errno = gdbm_errno;
++
+   gdbm_close(db);
+   if (! gvalue.dptr) {
+-      if (gdbm_errno == GDBM_ITEM_NOT_FOUND) {
++      if (fetch_errno == GDBM_ITEM_NOT_FOUND) {
+           utils->seterror(conn, SASL_NOLOG,
+ 			  "user: %s@%s property: %s not found in %s",
+ 			  authid, realm, propName, path);
+-- 
+2.30.1
+

component/cyrus-sasl/buildout.cfg

@@ -15,6 +15,9 @@ shared = true
 url = ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz
 md5sum = a7f4e5e559a0e37b3ffc438c9456e425
 location = @@LOCATION@@
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/0001-db_gdbm-fix-gdbm_errno-overlay-from-gdbm_close.patch#eef9bb30101c29f045ed68db45d5e5b3
 pre-configure =
   "${libtool:location}/bin/libtoolize" -c -f &&
   "${automake:location}/bin/aclocal" -I "${libtool:location}/share/aclocal" -I config -I cmulocal &&

component/dbus/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 [dbus]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://dbus.freedesktop.org/releases/dbus/dbus-1.11.6.tar.gz
-md5sum = fdb03ad2f2861a171fe7d0caad3b415f 
+url = https://dbus.freedesktop.org/releases/dbus/dbus-1.12.20.tar.gz
+md5sum = dfe8a71f412e0b53be26ed4fbfdc91c4
 location = @@LOCATION@@
 configure-options =
   --disable-static
@@ -21,14 +21,14 @@ make-targets = install && ${:location}/bin/dbus-uuidgen > ${:location}/var/lib/d
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig
-  CPPFLAGS=-I${libexpat:location}/include
-  LDFLAGS=-L${libexpat:location}/lib
+  EXPAT_CFLAGS=-I${libexpat:location}/include
+  EXPAT_LIBS=-L${libexpat:location}/lib -lexpat
 
 [dbus-glib]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.104.tar.gz
-md5sum = 5497d2070709cf796f1878c75a72a039
+url = https://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.110.tar.gz
+md5sum = d7cebf1d69445cbd28b4983392145192
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/firewalld/buildout.cfg

@@ -10,22 +10,34 @@ extends =
   ../python-slip/buildout.cfg
   ../dbus/buildout.cfg
   ../flex/buildout.cfg
+  ../nftables/buildout.cfg
 
 [firewalld]
 recipe = slapos.recipe.cmmi
-url = https://github.com/t-woerner/firewalld/archive/v0.3.14.2.tar.gz
-md5sum = b8f81b536ede502721f69300d374447b
+url = https://github.com/firewalld/firewalld/releases/download/v0.8.6/firewalld-0.8.6.tar.gz
+md5sum = a120286a2c2bc12ece24d719ef2879b5
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
+location = @@LOCATION@@
+sbin-dir = sbin-orig
 pre-configure =
   sed -i -E 's,(^SUBDIRS *=.*) doc(\b.*),\1\2,' Makefile.am
   aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal
   ./autogen.sh
+configure-options =
+  --sbindir=${:location}/${:sbin-dir}
+  --disable-docs
+  --with-iptables=${iptables:location}/sbin/iptables-legacy
+  --with-iptables-restore=${iptables:location}/sbin/iptables-legacy-restore
+  --with-ip6tables=${iptables:location}/sbin/ip6tables-legacy
+  --with-ip6tables-restore=${iptables:location}/sbin/ip6tables-legacy-restore
+  --with-ebtables=${iptables:location}/sbin/ebtables-legacy
+  --with-ebtables-restore=${iptables:location}/sbin/ebtables-legacy-restore
+  --with-ipset=/bin/false
 
 environment =
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${perl:location}/bin:%(PATH)s
   CPPFLAGS=-I${gettext:location}/include
-  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
   M4=${m4:location}/bin/m4
   ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
   FIREWALLD_DEVEL_ENV=${buildout:parts-directory}/${:_buildout_section_name_}/src
@@ -33,9 +45,9 @@ environment =
 
 [dbus-python]
 recipe = slapos.recipe.cmmi
-version = 1.2.4
+version = 1.2.16
 url = http://dbus.freedesktop.org/releases/dbus-python/dbus-python-${:version}.tar.gz
-md5sum = 7372a588c83a7232b4e08159bfd48fe5 
+md5sum = 51a45c973d82bedff033a4b57d69d5d8
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
 environment =
   PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
@@ -83,8 +95,6 @@ environment =
   FFI_LIBS=-L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
   CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include
   LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
-#  GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
-#  GLIB_LIBS=-L${glib:location}/lib -lgobject-2.0 -lgthread-2.0 -pthread -lrt -lglib-2.0
   GIO_LIBS=-L${glib:location}/lib -lgio-2.0
   GI_CFLAGS=-I${gobject-introspection:location}/include/gobject-introspection-1.0
   GI_LIBS=-L${gobject-introspection:location}/lib -lgirepository-1.0
@@ -101,17 +111,29 @@ md5sum = 893752ba2e93a1e96334dfee19f884ad
 recipe = plone.recipe.command
 python = ${buildout:bin-directory}/${firewalld-eggs:interpreter}
 command =
-  sed -i 's#/usr/bin/python -Es#${:python}#' ${firewalld:location}/bin/firewall-cmd
-  sed -i 's#/usr/bin/python -Es#${:python}#' ${firewalld:location}/sbin/firewalld
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-applet
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-cmd
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-config
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-offline-cmd
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/${firewalld:sbin-dir}/firewalld
+
+  sed -i 's#python2.7 /usr/bin/firewall-config#${:python} ${firewalld:location}/bin/firewall-config#' ${firewalld:location}/etc/firewalld/lockdown-whitelist.xml
+
   sed -i 's#DefaultZone=public#DefaultZone=trusted#' ${firewalld:location}/etc/firewalld/firewalld.conf
+
+  sed -i 's#/etc/firewalld#${firewalld:location}/etc/firewalld#'     ${firewalld:python-egg}/firewall/config/__init__.py
   sed -i 's#/usr/lib/firewalld#${firewalld:location}/lib/firewalld#' ${firewalld:python-egg}/firewall/config/__init__.py
-  sed -i 's#/usr/share/#${firewalld:location}/share#' ${firewalld:python-egg}/firewall/config/__init__.py
+  sed -i 's#/usr/share/#${firewalld:location}/share#'                ${firewalld:python-egg}/firewall/config/__init__.py
+
   sed -i "s#import sys#import sys, os\n\nos.environ['GI_TYPELIB_PATH'] = '${gobject-introspection:location}/lib/girepository-1.0/'#" ${:python}
   sed -i 's#<syslog/>#<!-- no syslog -->#' ${dbus:location}/etc/dbus-1/system.conf
   sed -i 's#<user>messagebus</user>#<user>slapsoft</user>#' ${dbus:location}/share/dbus-1/system.conf
   cp -f ${firewalld:location}/lib/firewalld/zones/trusted.xml ${firewalld:location}/etc/firewalld/zones/
-  cp -f ${firewalld:location}/etc/dbus-1/system.d/FirewallD.conf ${dbus:location}/share/dbus-1/system.d/
-  mkdir -p ${buildout:directory}/sbin
+  cp -f ${firewalld:location}/share/dbus-1/system.d/FirewallD.conf ${dbus:location}/share/dbus-1/system.d/
+  mkdir -p ${firewalld:location}/sbin
+  echo -n '#!/bin/sh\nLD_LIBRARY_PATH=${nftables:location}/lib ${firewalld:location}/${firewalld:sbin-dir}/firewalld "$@"' > ${firewalld:location}/sbin/firewalld
+  chmod a+x ${firewalld:location}/sbin/firewalld
+
 update-command = ${:command}
 stop-on-error = true
 
@@ -127,3 +149,4 @@ extra-paths =
   ${dbus-python:python-egg}
   ${firewalld:python-egg}
   ${pygobject3:python-egg}
+  ${nftables:python-egg}

component/gmp/buildout.cfg

@@ -9,10 +9,12 @@ parts =
 [gmp]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 6.1.1
+version = 6.2.1
 url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
-md5sum = e70e183609244a332d80529e7e155a35
+md5sum = 0b82665c4a92fd2ade7440c13fcaa42b
 configure-options =
+  --disable-assembly
+  --host=none-pc-linux-gnu
   --enable-cxx
   --disable-static
 environment =

component/golang/buildout.cfg

@@ -52,8 +52,8 @@ environment-extra =
 
 [golang1.14]
 <= golang-common
-url = https://golang.org/dl/go1.14.12.src.tar.gz
-md5sum = ae124560e0185a82883b57cc9662aa73
+url = https://golang.org/dl/go1.14.15.src.tar.gz
+md5sum = 1065b705a0d330136084f4e1628baf82
 
 # go1.14 needs go1.4 to bootstrap
 environment-extra =
@@ -61,13 +61,22 @@ environment-extra =
 
 [golang1.15]
 <= golang-common
-url = https://golang.org/dl/go1.15.5.src.tar.gz
-md5sum = 12a60d319ff58c03c323e8dcc125ae81
+url = https://golang.org/dl/go1.15.8.src.tar.gz
+md5sum = 7e4b8580bfe25ffc51501b1dc3d2d8ce
 
 # go1.15 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
+[golang1.16]
+<= golang-common
+url = https://golang.org/dl/go1.16.src.tar.gz
+md5sum = b767f7e81d886b4208874ea953db2ce3
+
+# go1.16 needs go1.4 to bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}
+
 
 # ---- infrastructure to build Go workspaces / projects ----
 
@@ -113,7 +122,7 @@ bin	= ${:directory}/bin
 depends = ${gowork.goinstall:recipe}
 
 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.14:location}
+golang  = ${golang1.15:location}
 
 # no special build flags by default
 buildflags =
@@ -133,7 +142,7 @@ recipe	= slapos.recipe.template
 url     = ${:_profile_base_location_}/goenv.sh.in
 output	= ${gowork:directory}/env.sh
 depends = ${gowork.mkdir:recipe}
-md5sum	= f89553711ea95ad1f6727b76747a62bf
+md5sum	= 6efdf5d63381c240c7d206d7939a63f7
 
 [gowork.mkdir]
 # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)

component/golang/goenv.sh.in

@@ -10,11 +10,18 @@ export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_
 # ---- 8< ----
 
 export GOPATH=$X:$GOPATH
-export PATH=$X/bin:$PATH
-export PS1="(`basename $X`) $PS1"
+export GOBIN=$X/bin
 export GOCACHE=$X/cache
+export GOMODCACHE=$X/pkg/mod
 export GOENV=$X/goenv
 
+export PATH=$GOBIN:$PATH
+export PS1="(`basename $X`) $PS1"
+
+# by default make go run in module-aware mode only if go.mod is present, and in GOPATH-mode otherwise.
+# (we still have some GOPATH uses; TODO -> default to "on" after GOPATH-mode usage is gone)
+export GO111MODULE=auto
+
 # strip trailing : from $GOPATH, $PKG_CONFIG_PATH
 GOPATH=$${GOPATH%:}
 PKG_CONFIG_PATH=$${PKG_CONFIG_PATH%:}

component/groonga/buildout.cfg

@@ -15,12 +15,12 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/groonga/groonga-10.0.5.tar.gz
-md5sum = 12f5c7b522afd72d0be2caa1e5f254da
+url = https://packages.groonga.org/source/groonga/groonga-11.0.0.tar.gz
+md5sum = 2ea277c19f21b9d45410547173ff41c2
 groonga-plugin-dir =  @@LOCATION@@/lib/groonga/plugins/
 # temporary patch to respect more tokens in natural language mode.
 patches =
-  ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
+  ${:_profile_base_location_}/groonga.patch#5d831331ddfdcd87e91b68949c339a1d
 patch-options = -p1
 configure-options =
   --disable-static

component/groonga/groonga.patch

-diff --git a/lib/ii.c b/lib/ii.c
-index d10b84d..e20bdce 100644
 --- a/lib/ii.c
 +++ b/lib/ii.c
-@@ -5695,7 +5695,9 @@ grn_ii_similar_search(grn_ctx *ctx, grn_ii *ii,
-     ? (optarg->similarity_threshold > GRN_HASH_SIZE(h)
+@@ -10387,7 +10387,9 @@
+     ? (data->optarg->similarity_threshold > GRN_HASH_SIZE(h)
         ? GRN_HASH_SIZE(h)
-        : optarg->similarity_threshold)
+        : data->optarg->similarity_threshold)
 -    : (GRN_HASH_SIZE(h) >> 3) + 1;
 +    : (GRN_HASH_SIZE(h) < 8
 +       ? GRN_HASH_SIZE(h)

component/h5py/buildout.cfg

@@ -17,6 +17,8 @@ egg = h5py
 setup-eggs = 
   ${cython:egg}
   ${numpy:egg}
+  pkgconfig
+  six
 include-dirs =
   ${hdf5:location}/include
 library-dirs =

component/helloweb/buildout.cfg

@@ -25,7 +25,7 @@ parts =
 install =
     lab.nexedi.com/nexedi/helloweb/go/...
 
-golang  = ${golang1.15:location}
+golang  = ${golang1.16:location}
 
 # -*- go -*-
 [helloweb-go]

component/mariadb/buildout.cfg

@@ -30,8 +30,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
-version = 10.4.17
-md5sum = e8193b9cd008b6d7f177f5a5c44c7a9f
+version = 10.4.18
+md5sum = 22e429d532fcfcb0e4beba64e1c296f0
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -96,8 +96,8 @@ post-install =
 # as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/mroonga/mroonga-10.05.tar.gz
-md5sum = a690811be5cc0593acc12ecf4ac9c40f
+url = https://packages.groonga.org/source/mroonga/mroonga-11.00.tar.gz
+md5sum = 0aefcbb3c0dc6e29c53c6ee6287114b8
 pre-configure = 
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source
@@ -127,7 +127,7 @@ environment =
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
 
 
-### XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### XXX keep using mariadb 10.3 because 10.4 causes crash
 ### (we just override here for easier revert)
 [mariadb-10.3]
 <= mariadb-10.4
@@ -143,4 +143,4 @@ post-install =
 
 [mariadb]
 location = ${mariadb-10.3:location}
-### /XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### /XXX keep using mariadb 10.3 because 10.4 causes crash

component/nftables/buildout.cfg

+[buildout]
+parts =
+  nftables
+
+extends =
+  ../defaults.cfg
+  ../gmp/buildout.cfg
+
+
+[libnml]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/libmnl/files/libmnl-1.0.4.tar.bz2
+md5sum = be9b4b5328c6da1bda565ac5dffadb2d
+
+[libnftnl]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/libnftnl/files/libnftnl-1.1.9.tar.bz2
+md5sum = e03cefd53f4b076d959abe36de5c38f8
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+
+[jansson]
+recipe = slapos.recipe.cmmi
+url = https://digip.org/jansson/releases/jansson-2.13.1.tar.gz
+md5sum = 570af45b8203e95876d71fecd56cee20
+
+[iptables]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/iptables/files/iptables-1.8.7.tar.bz2
+md5sum = 602ba7e937c72fbb7b1c2b71c3b0004b
+location = @@LOCATION@@
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig:${libnftnl:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${:location}/lib
+
+[nftables]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/nftables/files/nftables-0.9.8.tar.bz2
+md5sum = 77bf0bd43e65e92212fc73139a2e47fc
+python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig:${libnftnl:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  LDFLAGS = -L${gmp:location}/lib -L${jansson:location}/lib
+  CFLAGS = -I${gmp:location}/include -I${jansson:location}/include
+configure-options =
+  --enable-python
+  --with-json
+  --without-cli
+

component/nxdtest/buildout.cfg

@@ -20,6 +20,6 @@ egg     = nxdtest
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = 40e2c4ab
+revision = b5a74214
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git

component/python-pynacl/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../libffi/buildout.cfg
   ../pkgconfig/buildout.cfg
+  ../python-cffi/buildout.cfg
 
 parts =
   python-pynacl
@@ -14,6 +15,8 @@ library-dirs =
   ${libffi:location}/lib/
 rpath =
   ${libffi:location}/lib/
+setup-eggs =
+  ${python-cffi:egg}
 
 [python-pynacl-env]
 PATH = ${pkgconfig:location}/bin:%(PATH)s

component/slapos/obs.cfg

@@ -184,6 +184,8 @@ configure-command =
 environment +=
   DESTDIR=${buildout:destdir}
 
-[versions]
-# More recently version of cliff uses pbr which will break package build on OBS.
-cliff = 1.4.5
+[iptables]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+environment +=
+  DESTDIR=${buildout:destdir}

component/statsmodels/buildout.cfg

 [buildout]
 extends =
   ../pandas/buildout.cfg
+  ../scipy/buildout.cfg
 
 parts =
   statsmodels
@@ -14,3 +15,4 @@ egg = statsmodels
 environment = statsmodels-env
 setup-eggs =
   ${pandas:egg}
+  ${scipy:egg}

slapos/test/test_json_schema.py

@@ -25,14 +25,16 @@
 #
 ##############################################################################
 
-import unittest
-import os
+import collections
 import glob
 import json
-import collections
-import slapos.test
+import os
+import unittest
+
 import jsonschema
+import six
 
+import slapos.test
 
 def createInstanceParameterSchemaValidatorTest(path):
   # Test that json is a valid json schema, supports several
@@ -65,7 +67,7 @@ def createSoftwareCfgValidatorTest(path, software_cfg_schema):
       # also make sure request and response schemas can be resolved
       schema.setdefault('$id', 'file://' + path)
       resolver = jsonschema.RefResolver.from_schema(schema)
-      for software_type_definition in schema['software-type'].itervalues():
+      for software_type_definition in six.itervalues(schema['software-type']):
         resolver.resolve(software_type_definition['request'])
         resolver.resolve(software_type_definition['response'])
 

software/caddy-frontend/README.caddy_frontend.rst

@@ -6,6 +6,8 @@ Frontend system using Caddy, based on apache-frontend software release, allowing
 
 Caddy Frontend works using the master instance / slave instance design. It means that a single main instance of Caddy will be used to act as frontend for many slaves.
 
+This documentation covers only specific scenarios. Most of the parameters are described in `software.cfg.json <software.cfg.json>`_.
+
 Software type
 =============
 
@@ -129,7 +131,7 @@ Example sessions is::
 
   cat certificate.pem ca.pem key.pem > bundle.pem
 
-  curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem master-key-upload-url+authtoken
+  curl -g --upload-file bundle.pem --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" master-key-upload-url+authtoken
 
 This replaces old request parameters:
 
@@ -157,7 +159,7 @@ Example sessions is::
 
   cat certificate.pem ca.pem key.pem > bundle.pem
 
-  curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem key-upload-url+authtoken
+  curl -g --upload-file bundle.pem --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" key-upload-url+authtoken
 
 This replaces old request parameters:
 
@@ -215,29 +217,15 @@ Example of value: "/erp5/web_site_module/hosting/"
 
 url
 ~~~
-Necessary to activate cache. ``url`` of backend to use.
-
-``url`` is an optional parameter.
+URL of the backend to use, optional but will result with non functioning slave.
 
 Example: http://mybackend.com/myresource
 
-domain
-~~~~~~
-
-Necessary to activate cache.
-
-The frontend will be accessible from this domain.
-
-``domain`` is an optional parameter.
-
-Example: www.mycustomdomain.com
-
 enable_cache
 ~~~~~~~~~~~~
 
-Necessary to activate cache.
+Enables HTTP cache, optional.
 
-``enable_cache`` is an optional parameter.
 
 health-check-*
 ~~~~~~~~~~~~~~

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = de69a8c408ce4f228fc22eacb7e96657
+md5sum = 04015a7a552285984d091293ef573fb9
 
 [profile-common]
 filename = instance-common.cfg.in
@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 9cc78e7ce1960691e37f103855ff0dc9
+md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = eb98ffd96b2768cc6a5cf664b23aabd3
+md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -21,7 +21,6 @@
         'ciphers',
         'request-timeout',
         'authenticate-to-backend',
-        'strict-transport-security',
     ]
 %}
 {% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}

software/caddy-frontend/instance-caddy-input-schema.json

@@ -19,7 +19,7 @@
       "type": "string"
     },
     "domain": {
-      "description": "Base Domain for create subdomains (ie.: example.com).",
+      "description": "Domain used to generate automatic hostnames for slaves. For example 'example.com' will result with slave hostname 'slaveref.example.com'.",
       "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
       "title": "Domain",
       "type": "string"
@@ -107,12 +107,6 @@
       ],
       "title": "Authenticate to backend",
       "type": "string"
-    },
-    "strict-transport-security": {
-      "title": "Strict Transport Security",
-      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
-      "default": "0",
-      "type": "integer"
     }
   },
   "title": "Input Parameters",

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -288,7 +288,7 @@
     },
     "strict-transport-security": {
       "title": "Strict Transport Security",
-      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
+      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
       "default": "0",
       "type": "integer"
     },

software/caddy-frontend/instance.cfg.in

@@ -106,4 +106,3 @@ configuration.backend-haproxy-statistic-port = 21444
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
 configuration.slave-introspection-https-port = 22443
-configuration.strict-transport-security = 0

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -68,7 +68,7 @@ context =
 {%-   for key in ['disabled-cookie-list'] %}
 {%-     do slave_instance.__setitem__(key, slave_instance.get(key, '').split()) %}
 {%-   endfor %}
-{%-   for key, default in [('virtualhostroot-http-port', '80'), ('virtualhostroot-https-port', '443')] %}
+{%-   for key, default in [('virtualhostroot-http-port', '80'), ('virtualhostroot-https-port', '443'), ('strict-transport-security', '0')] %}
 {%-     do slave_instance.__setitem__(key, int(slave_instance.get(key, default))) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('default-path', slave_instance.get('default-path', '').strip('/') | urlencode) %}
@@ -128,7 +128,7 @@ context =
 {%-   do part_list.extend([slave_section_title]) %}
 {%-   set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
 {#-   Pass backend timeout values #}
-{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend', 'strict-transport-security'] %}
+{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
 {%-     if slave_instance.get(key, '') == '' %}
 {%-       do slave_instance.__setitem__(key, configuration[key]) %}
 {%-     endif %}

software/caddy-frontend/test/test.py

@@ -6814,7 +6814,6 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
       'ciphers': 'ciphers',
       'request-timeout': 100,
       'authenticate-to-backend': True,
-      'strict-transport-security': 200,
       # specific parameters
       '-frontend-config-1-ram-cache-size': '512K',
       '-frontend-config-2-ram-cache-size': '256K',
@@ -6912,8 +6911,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
         u'ram-cache-size': u'512K',
         u're6st-verification-url': u're6st-verification-url',
         u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
       },
       'caddy-frontend-2': {
         'X-software_release_url': self.frontend_2_sr,
@@ -6940,8 +6938,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
         u'ram-cache-size': u'256K',
         u're6st-verification-url': u're6st-verification-url',
         u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
       },
       'caddy-frontend-3': {
         'X-software_release_url': self.frontend_3_sr,
@@ -6967,8 +6964,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
         u'port': u'11443',
         u're6st-verification-url': u're6st-verification-url',
         u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
       },
       'kedifa': {
         'X-software_release_url': self.kedifa_sr,
@@ -7014,8 +7010,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
         'request-timeout': '100',
         'root_instance_title': 'testing partition 0',
         'slap_software_type': 'RootSoftwareInstance',
-        'slave_instance_list': [],
-        'strict-transport-security': '200'
+        'slave_instance_list': []
       }
     }
     self.assertEqual(

software/galene/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [instance-cfg]
 filename = instance.cfg.in
-md5sum = 8341d1d8c1839a1c623dd0fd771ef104
+md5sum = e54af429aba9b395c13471de254c4621

software/galene/instance.cfg.in

@@ -20,7 +20,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
-configuration.ice-servers.json = [{"urls":["stun:turn.api.nexedi.net:3478"]}]
+configuration.ice-servers.json = [{"urls":["stun:turn-paris-rapidspace.api.nexedi.net:443"]}]
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -50,28 +50,29 @@ storage-path = $${directory:data}/.passwd
 username = admin
 
 [ice-servers.json]
-recipe = collective.recipe.template
-input = inline:
+recipe = slapos.recipe.template:jinja2
+template = inline:
   $${slap-configuration:configuration.ice-servers.json}
-output = $${directory:data}/ice-servers.json
+rendered = $${directory:data}/ice-servers.json
 
 [groups-json]
-recipe = collective.recipe.template
-input = inline:{
+recipe = slapos.recipe.template:jinja2
+template = inline:{
     "public":true,
     "op": [{"username":"$${admin-password:username}","password":"$${admin-password:passwd}"}],
     "other": [],
     "presenter": [{"username": "", "password": "nexedi"}],
     "max-users":100
   }
-output = $${directory:groups}/public.json
+rendered = $${directory:groups}/public.json
 
 [galene-wrapper]
-recipe = collective.recipe.template
+recipe = slapos.recipe.template:jinja2
 port = 8443
 ip = $${slap-configuration:ipv6-random}
-input = inline:
-  #!/bin/bash
+template =
+  inline:#!/bin/sh
+  
   ulimit -n $(ulimit -Hn)
   exec ${galene:location}/bin/galene \
     -static ${galene-repository:location}/static \
@@ -80,8 +81,7 @@ input = inline:
     -data $${directory:data} \
     -http [$${:ip}]:$${:port} \
     -turn ""
-output = $${directory:services}/galene
-mode = 0755
+rendered = $${directory:services}/galene
 depends =
   $${ice-servers.json:recipe}
   $${groups-json:recipe}

software/html5as-base/software.cfg

@@ -16,6 +16,8 @@ parts =
 # Call creation of instance.cfg file that will be called for deployment of
 # instance
     template-cfg
+# Add extra egg
+    extra-eggs
 
 # Download instance.cfg.in (buildout profile used to deployment of instance),
 # replace all {{ foo_bar }} parameters by real values
@@ -55,7 +57,7 @@ mode = 0644
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 _update_hash_filename_ = templates/launcher.in
-md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+md5sum = 6cb0d64905ae7fc67277c1bf76b86875
 mode = 0644
 
 [template_mime_types]
@@ -65,6 +67,12 @@ _update_hash_filename_ = templates/mime_types.in
 md5sum = 4ef94a7b458d885cd79ba0b930a5727e
 mode = 0644
 
+[extra-eggs]
+recipe  = zc.recipe.egg
+eggs    =
+    plone.recipe.command
+
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
+plone.recipe.command = 1.1

software/html5as/buildout.hash.cfg

@@ -29,7 +29,7 @@ md5sum = 61dc4c82bf48563228ce4dea6c5c6319
 
 [template_launcher]
 _update_hash_filename_ = templates/launcher.in
-md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+md5sum = 6cb0d64905ae7fc67277c1bf76b86875
 
 [template_mime_types]
 _update_hash_filename_ = templates/mime_types.in

software/jupyter/software.cfg.json

@@ -8,7 +8,7 @@
       "software-type": "default",
       "description": "Default",
       "request": "instance-jupyter-input-schema.json",
-      "response": "instance-jupyter-ouput-schema.json",
+      "response": "instance-jupyter-output-schema.json",
       "index": 1
     }
   }

software/slapos-sr-testing/software.cfg

@@ -2,6 +2,7 @@
 
 extends =
   ../../component/bcrypt/buildout.cfg
+  ../../component/python-cliff/buildout.cfg
   ../../component/curl/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/git/buildout.cfg
@@ -185,6 +186,7 @@ setup = ${slapos.core-repository:location}
 [python-interpreter]
 eggs += ${:extra-eggs}
 extra-eggs =
+  ${python-cliff:egg}
   ${lxml-python:egg}
   ${slapos.core-setup:egg}
   ${pillow-python:egg}

software/slapos-testing/software.cfg

@@ -14,6 +14,7 @@ extends =
   ../../component/lmsensors/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../stack/slapos.cfg
+  ../../stack/caucase/buildout.cfg
   ../../stack/nxdtest.cfg
   ./buildout.hash.cfg
 
@@ -235,12 +236,8 @@ httmock = 1.2.6
 manuel = 1.9.0
 mock = 2.0.0
 testfixtures = 6.11.0
-pem = 18.1.0
 pycurl = 7.43.0.2
 pyflakes = 2.0.0
 zope.testing = 4.6.2
 urllib3 = 1.24.1
 pathlib = 1.0.1
-# Required by:
-# caucase
-PyJWT = 1.6.4

software/theia/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 397a2e0ac0d7f8813e0b594af6785300
+md5sum = 4f119df2b2d71e5ec1fd04051fd8fff8
 
 [yarn.lock]
 filename = yarn.lock

software/theia/instance.cfg.in

 [buildout]
-parts =
-  monitor-base
-  promises
-  frontend-reload
+extends =
+  ${monitor-template:rendered}
+
+theia-environment-parts =
   tasks.json
-  publish-connection-parameter
   slapos-repository
   runner-link
   settings.json
 
-extends = ${monitor-template:rendered}
+theia-parts =
+  frontend-reload
+  promises
+
+parts =
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+  publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+configuration.autorun = running
+configuration.embedded-sr =
+configuration.embedded-sr-type =
+
+
+[publish-connection-parameter]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+url = $${apache-frontend:connection-secure_access}
+username = $${frontend-instance-password:username}
+password = $${frontend-instance-password:passwd}
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+dot-theia = $${buildout:directory}/.theia/
+pidfiles = $${:var}/run
+
+services = $${:etc}/service
+runner = $${:srv}/runner
+project = $${:srv}/project
+frontend-static = $${:srv}/frontend-static
+frontend-static-public = $${:frontend-static}/public
+frontend-static-css = $${:frontend-static}/css
+
+bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
+fish-completions = $${buildout:directory}/.config/fish/completions/
+
+
+# Promises
+# --------
+
+[promises]
+recipe =
+instance-promises =
+  $${theia-listen-promise:name}
+  $${frontend-listen-promise:name}
+  $${apache-frontend-url-available-promise:name}
+  $${slapos-standalone-listen-promise:name}
+  $${slapos-autorun-promise:name}
+
+[theia-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = $${:_buildout_section_name_}.py
+config-hostname = $${theia-instance:ip}
+config-port = $${theia-instance:port}
+
+[frontend-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = $${:_buildout_section_name_}.py
+config-hostname = $${frontend-instance:ip}
+config-port = $${frontend-instance:port}
+
+[apache-frontend-url-available-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${apache-frontend:connection-secure_access}
+config-check-secure = 1
+
+[slapos-standalone-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+# XXX promise plugins can not contain "slapos" in their names
+name = standalone-listen-promise.py
+config-hostname = $${slapos-standalone-instance:hostname}
+config-port = $${slapos-standalone-instance:port}
+
+[slapos-autorun-promise]
+<= monitor-promise-base
+module = check_service_state
+# XXX promise plugins can not contain "slapos" in their names
+name = autorun-state-promise.py
+config-service = $${slapos-autorun:service-name}
+config-expect = $${slapos-autorun:autorun}
+
+
+# Remote Apache Frontend
+# ----------------------
+
+[apache-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Theia Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = $${frontend-instance:url}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /services /file-upload
+return = domain secure_access
+
+
+# Local Caddy Frontend
+# --------------------
+
 [frontend-instance-password]
 recipe = slapos.cookbook:generate.password
 username = node
 bytes = 12
 
+[frontend-instance-port]
+recipe = slapos.cookbook:free_port
+minimum = 3000
+maximum = 3100
+ip = $${instance-parameter:ipv6-random}
+
 [frontend-instance-certificate]
 recipe = plone.recipe.command
 command =
@@ -70,9 +194,9 @@ template = inline:
       /
     }
   }
-ip = $${instance-parameter:ipv6-random}
+ip = $${frontend-instance-port:ip}
 hostname = [$${:ip}]
-port = 3001
+port = $${frontend-instance-port:port}
 
 [frontend-instance]
 recipe = slapos.cookbook:wrapper
@@ -152,67 +276,15 @@ install =
 location = $${directory:frontend-static}/$${:filename}
 filename = $${:_buildout_section_name_}
 
-[tasks.json]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:dot-theia}/tasks.json
-template =
-  inline:
-  {
-    // See https://go.microsoft.com/fwlink/?LinkId=733558
-    // for the documentation about the tasks.json format
-    "version": "2.0.0",
-    "tasks": [
-      {
-        "label": "slapos node software",
-        "detail": "Build all software supplied to the node",
-        "type": "shell",
-        "command": "${buildout:bin-directory}/slapos",
-        "args": [
-          "node",
-          "software",
-          // debug mode can be enabled by commenting out this line:
-          // "--buildout-debug",
-          "--all"
-        ],
-        "options": {
-          "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
-            "GIT_EXEC_PATH": ""
-          }
-        },
-        "group": {
-          "kind": "build",
-          "isDefault": true
-        },
-        "problemMatcher": []
-      },
-      {
-        "label": "slapos node instance",
-        "detail": "Create all instances requested on the node",
-        "type": "shell",
-        "command": "${buildout:bin-directory}/slapos",
-        "args": [
-          "node",
-          "instance",
-          // debug mode can be enabled by commenting out this line:
-          // "--buildout-debug",
-          "--all"
-        ],
-        "options": {
-          "env": {
-            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
-            "GIT_EXEC_PATH": ""
-          }
-        },
-        "problemMatcher": [],
-        "group": {
-          "kind": "build",
-          "isDefault": true
-        }
-      }
-    ]
-  }
 
+# Theia Backend
+# -------------
+
+[theia-service-port]
+recipe = slapos.cookbook:free_port
+minimum = 3500
+maximum = 3600
+ip = $${instance-parameter:ipv4-random}
 
 [theia-service]
 recipe = slapos.recipe.template:jinja2
@@ -233,8 +305,8 @@ template =
   export TERMINFO=${ncurses:location}/lib/terminfo/
   export EDITOR="${python-language-server:location}/bin/python -m theia_open --wait"
   exec ${theia-wrapper:rendered} $@
-ip =  $${instance-parameter:ipv4-random}
-port = 3000
+ip =  $${theia-service-port:ip}
+port = $${theia-service-port:port}
 base-url = http://$${:ip}:$${:port}/
 
 [theia-instance]
@@ -285,10 +357,19 @@ command =
   ${buildout:bin-directory}/slapos complete --shell fish > $${directory:fish-completions}/slapos.fish
 
 
+# SlapOS Standalone
+# -----------------
+
+[slapos-standalone-port]
+recipe = slapos.cookbook:free_port
+minimum = 4000
+maximum = 4100
+ip = $${instance-parameter:ipv4-random}
+
 [slapos-standalone-config]
 ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
-port = 4000
+port = $${slapos-standalone-port:port}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
 
@@ -333,82 +414,9 @@ hash-files =
 hostname = $${slapos-standalone-config:ipv4}
 port = $${slapos-standalone-config:port}
 
-[promises]
-recipe =
-instance-promises =
-  $${theia-listen-promise:name}
-  $${frontend-listen-promise:name}
-  $${apache-frontend-url-available-promise:name}
-  $${slapos-standalone-listen-promise:name}
-  $${slapos-autorun-promise:name}
-
-[theia-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = $${:_buildout_section_name_}.py
-config-hostname = $${theia-instance:ip}
-config-port = $${theia-instance:port}
-
-[frontend-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = $${:_buildout_section_name_}.py
-config-hostname = $${frontend-instance:ip}
-config-port = $${frontend-instance:port}
-
-[apache-frontend-url-available-promise]
-<= monitor-promise-base
-module = check_url_available
-name = $${:_buildout_section_name_}.py
-config-url = $${apache-frontend:connection-secure_access}
-config-check-secure = 1
-
-[slapos-standalone-listen-promise]
-<= monitor-promise-base
-module = check_port_listening
-# XXX promise plugins can not contain "slapos" in their names
-name = standalone-listen-promise.py
-config-hostname = $${slapos-standalone-instance:hostname}
-config-port = $${slapos-standalone-instance:port}
 
-[slapos-autorun-promise]
-<= monitor-promise-base
-module = check_service_state
-# XXX promise plugins can not contain "slapos" in their names
-name = autorun-state-promise.py
-config-service = $${slapos-autorun:service-name}
-config-expect = $${slapos-autorun:autorun}
-
-[apache-frontend]
-<= slap-connection
-recipe = slapos.cookbook:requestoptional
-name = Theia Frontend
-# XXX We have hardcoded SR URL here.
-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-slave = true
-config-url = $${frontend-instance:url}
-config-https-only = true
-config-type = websocket
-config-websocket-path-list = /services /file-upload
-return = domain secure_access
-
-[publish-connection-parameter]
-<= monitor-publish
-recipe = slapos.cookbook:publish
-url = $${apache-frontend:connection-secure_access}
-username = $${frontend-instance-password:username}
-password = $${frontend-instance-password:passwd}
-
-[instance-parameter]
-recipe = slapos.cookbook:slapconfiguration
-computer = $${slap-connection:computer-id}
-partition = $${slap-connection:partition-id}
-url = $${slap-connection:server-url}
-key = $${slap-connection:key-file}
-cert = $${slap-connection:cert-file}
-configuration.autorun = running
-configuration.embedded-sr =
-configuration.embedded-sr-type =
+# Slapos Standalone Autoprocessing
+# --------------------------------
 
 [slapos-autorun]
 recipe = plone.recipe.command
@@ -422,6 +430,71 @@ service-name = slapos-node-auto
 supervisor-conf = $${directory:runner}/etc/supervisord.conf
 autorun = $${instance-parameter:configuration.autorun}
 
+
+# Theia Local Environment Setup
+# -----------------------------
+
+[tasks.json]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:dot-theia}/tasks.json
+template =
+  inline:
+  {
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "slapos node software",
+        "detail": "Build all software supplied to the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "software",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        },
+        "problemMatcher": []
+      },
+      {
+        "label": "slapos node instance",
+        "detail": "Create all instances requested on the node",
+        "type": "shell",
+        "command": "${buildout:bin-directory}/slapos",
+        "args": [
+          "node",
+          "instance",
+          // debug mode can be enabled by commenting out this line:
+          // "--buildout-debug",
+          "--all"
+        ],
+        "options": {
+          "env": {
+            "SLAPOS_CONFIGURATION": "$${slapos-standalone-config:slapos-configuration}",
+            "GIT_EXEC_PATH": ""
+          }
+        },
+        "problemMatcher": [],
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        }
+      }
+    ]
+  }
+
 [slapos-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/slapos.git
@@ -451,23 +524,3 @@ template =
 recipe = slapos.cookbook:symbolic.link
 target-directory = $${directory:project}
 link-binary = $${directory:runner}
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-dot-theia = $${buildout:directory}/.theia/
-pidfiles = $${:var}/run
-
-services = $${:etc}/service
-runner = $${:srv}/runner
-project = $${:srv}/project
-frontend-static = $${:srv}/frontend-static
-frontend-static-public = $${:frontend-static}/public
-frontend-static-css = $${:frontend-static}/css
-
-bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
-fish-completions = $${buildout:directory}/.config/fish/completions/

software/theia/software.cfg.json

@@ -8,7 +8,7 @@
       "software-type": "default",
       "description": "Default",
       "request": "instance-input-schema.json",
-      "response": "instance-ouput-schema.json",
+      "response": "instance-output-schema.json",
       "index": 1
     }
   }

stack/erp5/buildout.cfg

@@ -378,6 +378,31 @@ initialization =
   repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
   sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
 
+[zope-product-with-eggtestinfo]
+recipe = zc.recipe.egg:custom
+setup-eggs =
+  eggtestinfo
+egg = ${:_buildout_section_name_}
+
+[Products.CMFUid]
+<= zope-product-with-eggtestinfo
+[Products.CMFActionIcons]
+<= zope-product-with-eggtestinfo
+[Products.CMFCalendar]
+<= zope-product-with-eggtestinfo
+[Products.CMFCore]
+<= zope-product-with-eggtestinfo
+[Products.CMFDefault]
+<= zope-product-with-eggtestinfo
+[Products.CMFTopic]
+<= zope-product-with-eggtestinfo
+[Products.DCWorkflow]
+<= zope-product-with-eggtestinfo
+Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
+Products.DCWorkflow-patch-options = -p1
+[Products.GenericSetup]
+<= zope-product-with-eggtestinfo
+
 [eggs]
 <= neoppod
 eggs = ${neoppod:eggs}
@@ -472,14 +497,14 @@ eggs = ${neoppod:eggs}
   Products.PluginRegistry
 
 # CMF 2.2
-  Products.CMFActionIcons
-  Products.CMFCalendar
-  Products.CMFCore
-  Products.CMFDefault
-  Products.CMFTopic
-  Products.CMFUid
-  Products.DCWorkflow
-  Products.GenericSetup
+  ${Products.CMFActionIcons:egg}
+  ${Products.CMFCalendar:egg}
+  ${Products.CMFCore:egg}
+  ${Products.CMFDefault:egg}
+  ${Products.CMFTopic:egg}
+  ${Products.CMFUid:egg}
+  ${Products.DCWorkflow:egg}
+  ${Products.GenericSetup:egg}
   five.localsitemanager
 
 # Other products
@@ -545,8 +570,6 @@ PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/00
 PyPDF2-patch-options = -p1
 Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7
 Acquisition-patch-options = -p1
-Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
-Products.DCWorkflow-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1
 
@@ -773,3 +796,5 @@ beautifulsoup4 = 4.8.2
 # WSGIProxy2==0.4.6
 WebOb = 1.8.5
 soupsieve = 1.9.5
+
+eggtestinfo = 0.3

stack/slapos.cfg

@@ -121,6 +121,7 @@ depends = ${slapos-toolbox-dependencies:eggs}
 # slapos.toolbox dependencies.
 recipe = zc.recipe.egg
 eggs =
+  ${python-cliff:egg}
   ${lxml-python:egg}
   ${pycurl:egg}
   ${python-cryptography:egg}
@@ -135,6 +136,7 @@ setup-eggs = setuptools_scm
 [slapos-command]
 recipe = zc.recipe.egg
 eggs =
+  ${python-cliff:egg}
   ${lxml-python:egg}
   slapos.core
   slapos.libnetworkcache
@@ -198,13 +200,13 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.171
-slapos.core = 1.6.4
+slapos.core = 1.6.5
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
-slapos.recipe.cmmi = 0.16
+slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.5
 slapos.toolbox = 0.117
 stevedore = 1.21.0
@@ -354,6 +356,8 @@ chardet = 3.0.4
 # requests==2.24.0
 urllib3 = 1.25.9
 
+pkgconfig = 1.5.1
+
 [networkcache]
 download-cache-url = http://shacache.nxdcdn.com
 download-dir-url = http://shadir.nxdcdn.com