Merge branch 'master' of https://lab.nexedi.com/nexedi/slapos into replication-manager-2.1

component/apache/buildout.cfg

@@ -38,9 +38,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
 shared = false
-version = 2.4.39
+version = 2.4.41
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 930e217ba2d71e708a3f1521ecae7ec0
+md5sum = dfc674f8f454e3bc2d4ccd73ad3b5f1e
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util

component/babeld/buildout.cfg

@@ -6,7 +6,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/babeld.git
 branch = master
 git-executable = ${git:location}/bin/git
-revision = hmac-nxd1
+revision = hmac-nxd2
 
 [babeld]
 recipe = slapos.recipe.cmmi

component/coreutils/buildout.cfg

@@ -11,6 +11,9 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://ftp.gnu.org/gnu/coreutils/coreutils-8.31.tar.xz
 md5sum = 0009a224d8e288e8ec406ef0161f9293
+configure-options = 
+  --disable-libcap
+  --prefix=@@LOCATION@@
 environment =
   PATH=${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
   LDFLAGS=-Wl,--as-needed -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/curl/buildout.cfg

@@ -15,10 +15,11 @@ parts =
 [curl]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://curl.haxx.se/download/curl-7.60.0.tar.xz
-md5sum = a889cd11b4ae8794b7030472cb4df0a0
+url = http://curl.haxx.se/download/curl-7.67.0.tar.xz
+md5sum = d55351b88dec558dd3a24dabb2c2d899
 configure-options =
   --disable-static
+  --disable-esni
   --disable-ldap
   --disable-ldaps
   --disable-rtsp
@@ -29,22 +30,28 @@ configure-options =
   --disable-imap
   --disable-smtp
   --disable-gopher
+  --disable-manual
   --enable-ipv6
   --disable-sspi
+  --disable-alt-svc
   --with-zlib=${zlib:location}
   --with-ssl=${openssl:location}
   --without-gnutls
   --without-polarssl
   --without-mbedtls
-  --without-cyassl
+  --without-mesalink
   --without-nss
-  --without-axtls
   --without-libpsl
   --without-libmetalink
   --without-libssh2
   --without-librtmp
-  --without-libidn
+  --without-libidn2
   --with-nghttp2=${nghttp2:location}
+  --without-ngtcp2
+  --without-nghttp3
+  --without-quiche
+  --without-zsh-functions-dir
+  --without-fish-functions-dir
 
 environment =
   PATH=${perl:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/dcron/buildout.cfg

@@ -8,6 +8,7 @@ parts = dcron-output
 
 [dcron]
 recipe = slapos.recipe.cmmi
+shared = false
 url = http://www.jimpryor.net/linux/releases/dcron-4.4.tar.gz
 md5sum = 02d848ba043a9df5bf2102a9f4bc04bd
 configure-command = true

component/file/0001-fix-libmagic-correctly-detect-msooxml-files.patch

-From a975c66c81e45433a668b7daeb4c903a78cb9d33 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
-Date: Fri, 27 Sep 2019 01:56:24 +0200
-Subject: [PATCH] fix libmagic correctly detect msooxml files
-
-Coming from slapos patch on component:
-
-commit 7a24c4e4ff96a7d00072db891761cea8db7b9122
-Author: Boris Kocherov <bk@raskon.ru>
-Date:   Sun Feb 4 10:52:17 2018 +0300
-
-    component/file: fix libmagic correctly detect msooxml files
-
-that was reformatted as a patch so that component/file supports shared
-slapos.recipe.cmmi. Maybe file now include a proper fix.
----
- magic/Magdir/msooxml | 27 ++++++++++++++++++---------
- 1 file changed, 18 insertions(+), 9 deletions(-)
-
-diff --git a/magic/Magdir/msooxml b/magic/Magdir/msooxml
-index 059e729c..f8431dc8 100644
---- a/magic/Magdir/msooxml
-+++ b/magic/Magdir/msooxml
-@@ -1,4 +1,3 @@
--
- #------------------------------------------------------------------------------
- # $File: msooxml,v 1.5 2014/08/05 07:38:45 christos Exp $
- # msooxml:  file(1) magic for Microsoft Office XML
-@@ -13,24 +12,34 @@
- #   which can distinguish between the three types
- 
- # start by checking for ZIP local file header signature
--0		string		PK\003\004
-+0               string          PK\003\004
- !:strength +10
- # make sure the first file is correct
-->0x1E		regex		\\[Content_Types\\]\\.xml|_rels/\\.rels
-+>0x1E           regex           \\[Content_Types\\]\\.xml|_rels/\\.rels
- # skip to the second local file header
- # since some documents include a 520-byte extra field following the file
- # header, we need to scan for the next header
-->>(18.l+49)	search/2000	PK\003\004
-+>>(18.l+49)     search/2000     PK\003\004
- # now skip to the *third* local file header; again, we need to scan due to a
- # 520-byte extra field following the file header
-->>>&26		search/1000	PK\003\004
-+>>>&26          search/1000     PK\003\004
-+# and check the subdirectory name to determine which type of OOXML
-+# file we have.  Correct the mimetype with the registered ones:
-+# http://technet.microsoft.com/en-us/library/cc179224.aspx
-+>>>>&26         string          word/           Microsoft Word 2007+
-+!:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
-+>>>>&26         string          ppt/            Microsoft PowerPoint 2007+
-+!:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
-+>>>>&26         string          xl/             Microsoft Excel 2007+
-+!:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-+
-+>>1104          search/300      PK\003\004
- # and check the subdirectory name to determine which type of OOXML
- # file we have.  Correct the mimetype with the registered ones:
- # http://technet.microsoft.com/en-us/library/cc179224.aspx
-->>>>&26		string		word/		Microsoft Word 2007+
-+>>>&26          string          word/           Microsoft Word 2007+
- !:mime application/vnd.openxmlformats-officedocument.wordprocessingml.document
-->>>>&26		string		ppt/		Microsoft PowerPoint 2007+
-+>>>&26          string          ppt/            Microsoft PowerPoint 2007+
- !:mime application/vnd.openxmlformats-officedocument.presentationml.presentation
-->>>>&26		string		xl/		Microsoft Excel 2007+
-+>>>&26          string          xl/             Microsoft Excel 2007+
- !:mime application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-->>>>&26		default		x		Microsoft OOXML
--- 
-2.11.0
-

component/file/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 [file]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.icm.edu.pl/packages/file/file-5.32.tar.gz
-md5sum = 4f2503752ff041895090ed6435610435
+url = http://ftp.icm.edu.pl/packages/file/file-5.37.tar.gz
+md5sum = 80c29aca745466c6c24d11f059329075
 configure-options =
   --disable-static
 environment =
@@ -21,4 +21,4 @@ environment =
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/0001-fix-libmagic-correctly-detect-msooxml-files.patch#5e66a340d8ec7212d485e17d9af95f24
+  https://sources.debian.org/data/main/f/file/1:5.37-6/debian/patches/cherry-pick.FILE5_37-67-g46a8443f.limit-the-number-of-elements-in-a-vector-found-by-oss-fuzz.patch#fb6f7d32ce89573bf4b4b302c812e394

component/geoip2/buildout.cfg

@@ -17,5 +17,8 @@ rpath = ${:library-dirs}
 [geolite2-country]
 recipe = slapos.recipe.build:download-unpacked
 url = http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz#${:md5sum}
-md5sum = ff7bcb609291b9f5ab14397ccc506254
+md5sum = dc6224c648350d90f344a0c5c3ca5474
 strip-top-level-dir = true
+
+[versions]
+geoip2 = 2.9.0

component/ghostscript/buildout.cfg

@@ -32,5 +32,5 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs927/ghostscript-9.27.tar.xz
-md5sum = dd531503dbbc524f73528359e2ea145c
+url = https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs950/ghostscript-9.50.tar.xz
+md5sum = 6cea6bae4a7cdfac6ccb09f07f0caf8c

component/glib/buildout.cfg

@@ -10,13 +10,21 @@ extends =
 parts =
   glib
 
+# --with-python=${buildout:executable} is simpler but we may end up with
+# scripts whose shebang exceeds the kernel limit.
+# And ${buildout:executable}/.. is not a valid $PATH part.
+[glib-python]
+recipe = collective.recipe.shelloutput
+commands =
+  bin-directory = dirname ${buildout:executable}
+
 [glib]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/glib-2.50.2.tar.xz
-md5sum = 5eeb2bfaf78a07be59585e8b6e80b1d6
+url = https://ftp.gnome.org/pub/gnome/sources/glib/2.56/glib-2.56.4.tar.xz
+md5sum = 17c3dca43d99a4882384f1a7b530b80b
 configure-options =
-  --with-python=${buildout:executable}
+  --with-python=python
   --disable-libmount
   --disable-static
   --disable-selinux
@@ -24,7 +32,7 @@ configure-options =
   --disable-xattr
   --disable-man
 environment =
-  PATH=${gettext:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${glib-python:bin-directory}:${gettext:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${gettext:location}/include -I${zlib:location}/include
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   LIBFFI_CFLAGS=-I${libffi:location}/include

component/groonga/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://packages.groonga.org/source/groonga/groonga-9.0.9.tar.gz
-md5sum = 877bc3e2840e2935f606fb3d2d359a2c
+url = https://packages.groonga.org/source/groonga/groonga-9.1.0.tar.gz
+md5sum = 81722e0bc8cc0cfdddcafde2087b71df
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.4.tar.gz
-md5sum = 540cd21169e8828d5d11894b2fa74ab4
+url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.23.tar.gz
+md5sum = 6c35b83a9969449c4b79783a2119551e
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/libexpat/buildout.cfg

 [buildout]
+extends =
+  ../lunzip/buildout.cfg
 parts =
   libexpat
 
 [libexpat]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://downloads.sourceforge.net/project/expat/expat/2.2.1/expat-2.2.1.tar.bz2
-md5sum = d9c3baeab58774cefc2f04faf29f2cf8
+url = https://github.com/libexpat/libexpat/releases/download/R_2_2_9/expat-2.2.9.tar.lz
+md5sum = c356e4f2092df4f0b0ffef904f001842
 configure-options =
   --disable-static
+  --without-xmlwf
+  --without-examples
+  --without-tests
+  --without-docbook
+environment =
+  PATH=${lunzip:location}/bin:%(PATH)s

component/libpng/buildout.cfg

@@ -19,15 +19,10 @@ environment =
 
 [libpng12]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.2.58.tar.xz
-md5sum = 1fe68fa3cdab99dbcfd2a6b4de95645f
-
-[libpng15]
-<= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.29.tar.xz
-md5sum = b9e5452ee9681c313638efedb16c12a6
+url = http://download.sourceforge.net/libpng/libpng-1.2.59.tar.xz
+md5sum = e120f48f4e27e72255bc366c73aae1db
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.32.tar.xz
-md5sum = e01be057a9369183c959b793a685ad15
+url = http://download.sourceforge.net/libpng/libpng-1.6.37.tar.xz
+md5sum = 015e8e15db1eecde5f2eb9eb5b6e59e9

component/libzip/buildout.cfg

@@ -26,4 +26,4 @@ environment =
   PATH=${cmake:location}/bin:%(PATH)s
   CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
   CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
-  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib
+  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/mariadb/buildout.cfg

@@ -29,8 +29,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.3.20
-md5sum = a87a9532568c0ed50d25ce7e7c60d67d
+version = 10.3.21
+md5sum = c5b9ed98f2f57f7b809d4ec9c1515107
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 pre-configure =
   set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
@@ -84,8 +84,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = https://packages.groonga.org/source/mroonga/mroonga-9.09.tar.gz
-md5sum = 1b4ec3a8c6b4b459bd7a94f2bb8ad585
+url = https://packages.groonga.org/source/mroonga/mroonga-9.10.tar.gz
+md5sum = 30a3010fd5ef11a8bd440053f1624bdb
 pre-configure = set -e
   rm -rf fake_mariadb_source
   mkdir -p fake_mariadb_source

component/mosh/buildout.cfg

@@ -9,6 +9,7 @@ extends =
 
 [mosh]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://mosh.org/mosh-1.3.0.tar.gz
 md5sum = d961276995936953bf2d5a794068b076
 configure-options =

component/nano/buildout.cfg

@@ -8,8 +8,10 @@ extends =
 
 [nano]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://www.nano-editor.org/dist/v2.8/nano-2.8.4.tar.xz
 md5sum = 02ff28870194178595b287fc16fa611b
+location = @@LOCATION@@
 # The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
 # are installed on the system.
 environment=
@@ -17,7 +19,6 @@ environment=
   NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
   CPPFLAGS=-I${file:location}/include
   LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 post-install =
   cd ${:location} && mkdir etc &&
   echo include "${:location}/share/nano/*.nanorc" > etc/nanorc

component/nginx/buildout.cfg

@@ -12,11 +12,13 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = https://nginx.org/download/nginx-1.17.1.tar.gz
-md5sum = 51021f3e8204a5fc809f5e695a4508db
+shared = false
+url = https://nginx.org/download/nginx-1.17.6.tar.gz
+md5sum = 55022aa5715386c994f6773478822853
 
 [nginx]
 <= nginx-common
+shared = true
 configure-options=
   --with-http_ssl_module
   --with-http_v2_module

component/nodejs/buildout.cfg

@@ -15,9 +15,14 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-10.6.0]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v10.6.0
+md5sum = 9df233b86244ebda1ded1f91694fbe86
+
 [nodejs-8.9.4]
 <= nodejs-base
-recipe = slapos.recipe.cmmi
 version = v8.9.4
 md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 
@@ -26,24 +31,27 @@ md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 version = v8.6.0
 md5sum = 0c95e08220667d8a18b97ecec8218ac6
 
+
 [nodejs-base]
 # Server-side Javascript.
 version =
 md5sum =
+openssl_location = ${openssl-1.0:location}
 
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
 configure-options =
   --shared-openssl
-  --shared-openssl-includes=${openssl-1.0:location}/include
-  --shared-openssl-libpath=${openssl-1.0:location}/lib
+  --shared-openssl-includes=${:openssl_location}/include
+  --shared-openssl-libpath=${:openssl_location}/lib
 environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
+  HOME=@@LOCATION@@
   PATH=${gcc:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin/:%(PATH)s
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig/
+  PKG_CONFIG_PATH=${:openssl_location}/lib/pkgconfig/
   CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${openssl-1.0:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  LD_LIBRARY_PATH=${openssl-1.0:location}/lib
+  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${:openssl_location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LD_LIBRARY_PATH=${:openssl_location}/lib
 
 [nodejs-8.6.0-output]
 # Shared binary location to ease migration

component/openssh/buildout.cfg

@@ -15,7 +15,9 @@ parts =
 
 [openssh]
 recipe = slapos.recipe.cmmi
+shared = true
 md5sum = 68ba883aff6958297432e5877e9a0fe2
+location = @@LOCATION@@
 url = https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-7.7p1.tar.gz
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
@@ -25,9 +27,9 @@ environment =
   CPPFLAGS=-I${zlib:location}/include -I${openssl-1.0:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
 configure-options = 
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --exec-prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --with-privsep-path=${buildout:parts-directory}/${:_buildout_section_name_}/var/empty
+  --prefix=${:location}
+  --exec-prefix=${:location}
+  --with-privsep-path=${:location}/var/empty
 
 [openssh-output]
 # Shared binary location to ease migration

component/openssl/buildout.cfg

@@ -17,8 +17,8 @@ parts =
 [openssl]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.1.0j.tar.gz
-md5sum = b4ca5b78ae6ae79da80790b30dbedbdc
+url = https://www.openssl.org/source/openssl-1.1.1d.tar.gz
+md5sum = 3be209000dbc7e1b95bcdf47980a3baa
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with DESTDIR). Used by slapos.package.git/obs
@@ -57,8 +57,8 @@ openssl = ${openssl:location}/bin/openssl
 [openssl-1.0]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.0.2p.tar.gz
-md5sum = ac5eb30bf5798aa14b1ae6d0e7da58df
+url = https://www.openssl.org/source/openssl-1.0.2t.tar.gz
+md5sum = ef66581b80f06eae42f5268bc0b50c6d
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs

component/pkgconfig/buildout.cfg

@@ -4,30 +4,12 @@
 [buildout]
 parts =
   pkgconfig
-extends =
-  ../gettext/buildout.cfg
-  ../glib/buildout.cfg
-
-[pkg-config]
-<= pkgconfig
 
 [pkgconfig]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz
-md5sum = 77f27dce7ef88d0634d0d6f90e03a77f
-location = @@LOCATION@@
-# build pkg-config twice so that second configure can use pkg-config
-# to compute GLIB_CFLAGS and GLIB_LIBS.
-pre-configure =
-  ./configure --prefix=${:location} --with-installed-glib && make
+url = https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz
+md5sum = f6e931e319531b736fadc017f470e68a
 configure-options =
-  --with-installed-glib
+  --with-internal-glib
   --disable-host-tool
-environment =
-  PATH=.:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
-  CPPFLAGS=-I${glib:location}/include
-  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib
-  GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
-  GLIB_LIBS=-L${glib:location}/lib -lglib-2.0 -lintl

component/proftpd/buildout.cfg

@@ -22,25 +22,21 @@ recipe = collective.recipe.grp
 
 [proftpd]
 recipe = slapos.recipe.cmmi
-md5sum = 13270911c42aac842435f18205546a1b
-url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6.tar.gz
-# 1.3.6 is not compatible with openssl 1.1.x
-# https://github.com/proftpd/proftpd/issues/674
-# (fix is in master, we are waiting for next release)
+md5sum = 4040f6a6b86173e2a03f4ccdb9b9af6e
+url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6b.tar.gz
 configure-options =
   --enable-openssl
   --enable-nls
   --enable-ctrls
   --enable-dso
+  --disable-cap
   --with-modules=mod_sftp:mod_ban
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
 environment = 
-  CPPFLAGS=${:cppflags}
-  LDFLAGS=${:ldflags}
+  CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
   install_user=${proftpd-environment:USER}
   install_group=${proftpd-grp:GROUP}
-cppflags=-I${zlib:location}/include -I${openssl-1.0:location}/include
-ldflags=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl-1.0:location}/lib -Wl,-rpath=${openssl-1.0:location}/lib
 patch-binary = ${patch:location}/bin/patch
 patch-options = -p1
 patches =
@@ -59,14 +55,14 @@ git-executable = ${git:location}/bin/git
 [proftpd-mod_auth_web]
 recipe = plone.recipe.command
 stop-on-error = true
+# prxs does not support setting rpath, but we can "inject" -R that will be passed to libtool
 command =
   LIBTOOL=${libtool:location}/bin/libtool \
-  LDFLAGS="-Wl,-rpath=${curl:location}/lib ${proftpd:ldflags}" \
-  ${proftpd-output:prxs} -c -i \
-  -I ${curl:location}/include ${proftpd:cppflags} \
-  -L ${curl:location}/lib/ \
+  ${proftpd-output:prxs} -c -i -d \
+  -I ${curl:location}/include \
+  -L '${curl:location}/lib/ -R ${openssl:location}/lib -R ${zlib:location}/lib' \
   -l curl \
-  -d ${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
+  ${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
 location=${proftpd:location}/libexec/mod_auth_web.so
 
 

component/protobuf/buildout.cfg

@@ -3,10 +3,12 @@ parts = protobuf
 
 [protobuf]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-python-3.4.0.tar.gz
 md5sum = 0820cc2e56d71aef8e99794fcbd184cd
 
 [protobuf-cpp]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
 md5sum = 6d59dad503bea5ad420fd09ddad84481

component/pylint/buildout.cfg

+[buildout]
+parts =
+  pylint
+extends =
+  ../patch/buildout.cfg
+
+[pylint]
+recipe = zc.recipe.egg:custom
+egg = pylint
+patches =
+  ${:_profile_base_location_}/pylint-super_on_old_class.patch#cb0c3f8c091bf4980be395c917edc435
+patch-options = -p1
+patch-binary = ${patch:location}/bin/patch

component/pylint/pylint-super_on_old_class.patch

+commit 6cf07b3483ec0d921296aefa8a28a8491f13786e
+Author: Claudiu Popa <pcmanticore@gmail.com>
+Date:   Sun Dec 6 14:54:52 2015 +0200
+
+    Don't emit super-on-old-class on classes with unknown bases.
+    
+    
+    The change also removes the confidence handling for super-on-old-class,
+    which isn't used enough to merit its existence.
+    Closes issue #721.
+
+--- a/pylint/checkers/newstyle.py	2019-12-25 13:51:01.611586402 +0900
++++ b/pylint/checkers/newstyle.py	2019-12-25 13:50:31.861542768 +0900
+@@ -127,12 +127,9 @@
+             if isinstance(call, astroid.CallFunc) and \
+                isinstance(call.func, astroid.Name) and \
+                call.func.name == 'super':
+-                confidence = (INFERENCE if has_known_bases(klass)
+-                              else INFERENCE_FAILURE)
+-                if not klass.newstyle:
++                if not klass.newstyle and has_known_bases(klass):
+                     # super should not be used on an old style class
+-                    self.add_message('super-on-old-class', node=node,
+-                                     confidence=confidence)
++                    self.add_message('super-on-old-class', node=node)
+                 else:
+                     # super first arg should be the class
+                     if not call.args and sys.version_info[0] == 3:
+@@ -146,8 +143,7 @@
+                         continue
+ 
+                     if supcls is None:
+-                        self.add_message('missing-super-argument', node=call,
+-                                         confidence=confidence)
++                        self.add_message('missing-super-argument', node=call)
+                         continue
+ 
+                     if klass is not supcls:
+@@ -163,8 +159,7 @@
+                         if name is not None:
+                             self.add_message('bad-super-call',
+                                              node=call,
+-                                             args=(name, ),
+-                                             confidence=confidence)
++                                             args=(name, ))
+ 
+ 
+ def register(linter):

component/python-2.7/buildout.cfg

@@ -29,9 +29,9 @@ python = python2.7
 [python2.7]
 recipe = slapos.recipe.cmmi
 shared = true
-package_version = 2.7.15
+package_version = 2.7.17
 package_version_suffix =
-md5sum = a80ae3cc478460b922242f43a1b4094d
+md5sum = b3b6d2c92f42a60667814358ab9f0cfd
 
 # This is actually the default setting for prefix, but we can't use it in
 # other settings in this part if we don't set it explicitly here.
@@ -42,7 +42,7 @@ executable = ${:prefix}/bin/python${:version}
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
-  ${:_profile_base_location_}/pytracemalloc_pep445.patch#3dfad79654af9671325f988c36fb6be2
+  ${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
   ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
   ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
 url =
@@ -56,6 +56,16 @@ configure-options =
 # Profiled build:
 make-binary =
 make-targets = make profile-opt && make install
+# Mangle shebang to use installed Python. This is not only a convenience:
+# when building system packages (e.g. re6st-node using OBS), some OS like
+# recent RedHat would otherwise mangle them, either failing ('python' is
+# ambiguous and 'python2' is not supported anymore) or replacing with
+# something that's really wrong (/usr/bin/python...).
+post-install = cd '${:prefix}' &&
+  find -executable -type f -print0 |
+  xargs -0 grep -I -m 1 '' |
+  sed -n 's,:#! */usr/bin/env \+python2\?$,,p' |
+  xargs -d '\n' sed -i '1s,.*,#!${:executable},'
 
 extra-ldflags =
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,

component/python-2.7/pytracemalloc_pep445.patch

@@ -492,7 +492,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  #ifdef WITH_PYMALLOC
  
  #ifdef HAVE_MMAP
-@@ -214,7 +489,7 @@
+@@ -220,7 +495,7 @@
   * Arenas are allocated with mmap() on systems supporting anonymous memory
   * mappings to reduce heap fragmentation.
   */
@@ -501,7 +501,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  
  #ifdef WITH_MEMORY_LIMITS
  #define MAX_ARENAS              (SMALL_MEMORY_LIMIT / ARENA_SIZE)
-@@ -581,7 +856,7 @@
+@@ -587,7 +862,7 @@
              return NULL;                /* overflow */
  #endif
          nbytes = numarenas * sizeof(*arenas);
@@ -510,7 +510,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
          if (arenaobj == NULL)
              return NULL;
          arenas = arenaobj;
-@@ -785,9 +1060,8 @@
+@@ -791,9 +1066,8 @@
   * Unless the optimizer reorders everything, being too smart...
   */
  
@@ -522,7 +522,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  {
      block *bp;
      poolp pool;
-@@ -802,15 +1076,6 @@
+@@ -808,15 +1082,6 @@
  #endif
  
      /*
@@ -538,7 +538,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
       * This implicitly redirects malloc(0).
       */
      if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
-@@ -981,17 +1246,13 @@
+@@ -987,17 +1252,13 @@
       * last chance to serve the request) or when the max memory limit
       * has been reached.
       */
@@ -559,7 +559,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  {
      poolp pool;
      block *lastfree;
-@@ -1211,7 +1472,7 @@
+@@ -1217,7 +1478,7 @@
  redirect:
  #endif
      /* We didn't allocate this address. */
@@ -568,7 +568,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  }
  
  /* realloc.  If p is NULL, this acts like malloc(nbytes).  Else if nbytes==0,
-@@ -1219,10 +1480,8 @@
+@@ -1225,10 +1486,8 @@
   * return a non-NULL result.
   */
  
@@ -581,7 +581,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  {
      void *bp;
      poolp pool;
-@@ -1232,16 +1491,7 @@
+@@ -1238,16 +1497,7 @@
  #endif
  
      if (p == NULL)
@@ -599,7 +599,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  
  #ifdef WITH_VALGRIND
      /* Treat running_on_valgrind == -1 the same as 0 */
-@@ -1269,10 +1519,10 @@
+@@ -1275,10 +1525,10 @@
              }
              size = nbytes;
          }
@@ -612,7 +612,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
          }
          return bp;
      }
-@@ -1290,40 +1540,17 @@
+@@ -1296,40 +1546,17 @@
       * at p.  Instead we punt:  let C continue to manage this block.
       */
      if (nbytes)
@@ -655,7 +655,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  #endif /* WITH_PYMALLOC */
  
  #ifdef PYMALLOC_DEBUG
-@@ -1343,10 +1570,6 @@
+@@ -1349,10 +1576,6 @@
  #define DEADBYTE       0xDB    /* dead (newly freed) memory */
  #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
  
@@ -666,7 +666,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  static size_t serialno = 0;     /* incremented on each debug {m,re}alloc */
  
  /* serialno is always incremented via calling this routine.  The point is
-@@ -1429,58 +1652,18 @@
+@@ -1467,58 +1690,18 @@
  p[2*S+n: 2*S+n+S]
      Copies of FORBIDDENBYTE.  Used to catch over- writes and reads.
  p[2*S+n+S: 2*S+n+2*S]
@@ -722,7 +722,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
 -
 -/* generic debug memory api, with an "id" to identify the API in use */
 -void *
--_PyObject_DebugMallocApi(char id, size_t nbytes)
+-_PyObject_DebugMallocApi(char api, size_t nbytes)
 +static void *
 +_PyMem_DebugMallocCtx(void *ctx, size_t nbytes)
  {
@@ -730,25 +730,27 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
      uchar *p;           /* base address of malloc'ed block */
      uchar *tail;        /* p + 2*SST + nbytes == pointer to tail pad bytes */
      size_t total;       /* nbytes + 4*SST */
-@@ -1491,14 +1674,14 @@
-         /* overflow:  can't represent total as a size_t */
+@@ -1530,7 +1713,7 @@
          return NULL;
  
--    p = (uchar *)PyObject_Malloc(total);
-+    p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
-     if (p == NULL)
-         return NULL;
+     if (api == _PYMALLOC_OBJ_ID) {
+-        p = (uchar *)PyObject_Malloc(total);
++        p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
+     }
+     else {
+         p = (uchar *)_PyMem_Malloc(total);
+@@ -1540,8 +1723,8 @@
  
-     /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
+     /* at p, write size (SST bytes), api (1 byte), pad (SST-1 bytes) */
      write_size_t(p, nbytes);
--    p[SST] = (uchar)id;
+-    p[SST] = (uchar)api;
 -    memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
 +    p[SST] = (uchar)api->api_id;
 +    memset(p + SST + 1, FORBIDDENBYTE, SST-1);
  
      if (nbytes > 0)
          memset(p + 2*SST, CLEANBYTE, nbytes);
-@@ -1516,35 +1699,37 @@
+@@ -1559,40 +1742,42 @@
     Then fills the original bytes with DEADBYTE.
     Then calls the underlying free.
  */
@@ -769,8 +771,13 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
      nbytes += 4*SST;
      if (nbytes > 0)
          memset(q, DEADBYTE, nbytes);
--    PyObject_Free(q);
-+    api->alloc.free(api->alloc.ctx, q);
+     if (api == _PYMALLOC_OBJ_ID) {
+-        PyObject_Free(q);
++        api->alloc.free(api->alloc.ctx, q);
+     }
+     else {
+         _PyMem_Free(q);
+     }
  }
  
 -void *
@@ -795,7 +802,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
      bumpserialno();
      original_nbytes = read_size_t(q - 2*SST);
      total = nbytes + 4*SST;
-@@ -1552,16 +1737,12 @@
+@@ -1600,17 +1785,13 @@
          /* overflow:  can't represent total as a size_t */
          return NULL;
  
@@ -808,13 +815,14 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
       * case we didn't get the chance to mark the old memory with DEADBYTE,
       * but we live with that.
       */
--    q = (uchar *)PyObject_Realloc(q - 2*SST, total);
-+    oldq = q;
-+    q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total);
-     if (q == NULL) {
-         if (nbytes <= original_nbytes) {
-             /* bpo-31626: the memset() above expects that realloc never fails
-@@ -1571,11 +1752,17 @@
+     if (api == _PYMALLOC_OBJ_ID) {
+-        q = (uchar *)PyObject_Realloc(q - 2*SST, total);
++        oldq = q;
++        q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total);
+     }
+     else {
+         q = (uchar *)_PyMem_Realloc(q - 2*SST, total);
+@@ -1624,11 +1805,17 @@
          return NULL;
      }
  
@@ -833,7 +841,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
      tail = q + nbytes;
      memset(tail, FORBIDDENBYTE, SST);
      write_size_t(tail + SST, serialno);
-@@ -1594,8 +1781,8 @@
+@@ -1647,8 +1834,8 @@
   * and call Py_FatalError to kill the program.
   * The API id, is also checked.
   */
@@ -844,7 +852,7 @@ diff -Naru a/Objects/obmalloc.c b/Objects/obmalloc.c
  {
      const uchar *q = (const uchar *)p;
      char msgbuf[64];
-@@ -1941,3 +2128,44 @@
+@@ -1994,3 +2181,44 @@
             arenas[arenaindex_temp].address != 0;
  }
  #endif

component/qemu-kvm/buildout.cfg

@@ -10,6 +10,8 @@ extends =
   ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
   ../zlib/buildout.cfg
+  ../pcre/buildout.cfg
+  ../glib/buildout.cfg
 
 # XXX Change all reference to kvm section to qemu section, then
 # use qemu as main name section.

component/rsync/buildout.cfg

@@ -7,9 +7,10 @@ parts =
 
 [rsync]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://download.samba.org/pub/rsync/src/rsync-3.1.3.tar.gz
 md5sum = 1581a588fde9d89f6bc6201e8129afaf
 make-options =
-  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+  PREFIX=@@LOCATION@@
 environment =
   PATH=${perl:location}/bin:%(PATH)s

component/screen/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 
 [screen]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
 version = 4.5.1
 md5sum = a8c5da2f42f8a18fa4dada2419d1549b

component/snappy/buildout.cfg

@@ -6,8 +6,8 @@ parts =
 
 [snappy]
 recipe = slapos.recipe.cmmi
-url = https://github.com/google/snappy/archive/1.1.7.tar.gz
-md5sum = ee9086291c9ae8deb4dac5e0b85bf54a
+url = https://github.com/google/snappy/archive/1.1.8.tar.gz
+md5sum = 70e48cba7fecf289153d009791c9977f
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = ${cmake:location}/bin/cmake
 configure-options =

component/sqlite3/buildout.cfg

@@ -6,11 +6,13 @@ parts =
 [sqlite3]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://sqlite.org/2018/sqlite-autoconf-3220000.tar.gz
-md5sum = 96b5648d542e8afa6ab7ffb8db8ddc3d
+url = https://sqlite.org/2019/sqlite-autoconf-3300100.tar.gz
+md5sum = 51252dc6bc9094ba11ab151ba650ff3c
 configure-options =
   --disable-static
   --enable-readline
+# Increase MAX_VARIABLE_NUMBER like many os. For example:
+# https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/sqlite
 environment =
-  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include
+  CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include -DSQLITE_MAX_VARIABLE_NUMBER=250000
   LDFLAGS=-L@@LOCATION@@ -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib

component/surykatka/buildout.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../python3/buildout.cfg
+
+parts =
+  surykatka
+
+[surykatka-requirements]
+recipe = slapos.recipe.build:download 
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[surykatka]
+recipe = plone.recipe.command
+command =
+  bash -c "${python3.7:executable} -m venv ${:location} && \
+    . ${:location}/bin/activate && \
+    pip install -r ${surykatka-requirements:target}"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+executable = ${:location}/bin/surykatka

component/surykatka/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[surykatka-requirements]
+_update_hash_filename_ = requirements.txt
+md5sum = 2dfe4f8b8c5b6f5d3478b70e53c49201

component/surykatka/requirements.txt

+certifi==2019.11.28
+chardet==3.0.4
+Click==7.0
+dnspython==1.16.0
+forcediphttpsadapter==1.0.1
+idna==2.8
+miniupnpc==2.0.2
+peewee==3.13.1
+requests==2.22.0
+surykatka==0.2.0
+urllib3==1.25.7

component/texinfo/buildout.cfg

@@ -9,6 +9,7 @@ extends =
 
 [texinfo4]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://ftp.gnu.org/gnu/texinfo/texinfo-4.13.tar.gz
 md5sum = 71ba711519209b5fb583fed2b3d86fcb
 configure-options =

component/tig/buildout.cfg

@@ -7,6 +7,7 @@ extends =
 
 [tig]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://jonas.nitro.dk/tig/releases/tig-2.1.tar.gz
 md5sum = d6c237aba2c03d85897da79789fd6104
 environment =

component/tomcat/buildout.cfg

@@ -20,8 +20,8 @@ md5sum = 3dde098fd0b3a08d3f2867e4a95591ba
 recipe = hexagonit.recipe.download
 ignore-existing = true
 strip-top-level-dir = true
-url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.96/bin/apache-tomcat-7.0.96.tar.gz
-md5sum = 0669aa2996b67c61662a5a4f993767b8
+url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.99/bin/apache-tomcat-7.0.99.tar.gz
+md5sum = ab39c15461f2a99493528b4a5819bc56
 
 [tomcat9]
 recipe = hexagonit.recipe.download

component/vim/buildout.cfg

@@ -8,6 +8,7 @@ extends =
 
 [vim]
 recipe = slapos.recipe.cmmi
+shared = true
 url = ftp://ftp.vim.org/pub/vim/unix/vim-8.0.586.tar.bz2
 md5sum = b35e794140c196ff59b492b56c1e73db
 environment=

component/xorg/buildout.cfg

@@ -563,9 +563,9 @@ url = https://www.x.org/releases/individual/app/xwd-1.0.7.tar.gz
 md5sum = 3ebd74f7a1980305e5e19ec8ff7aa794
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
   CPPFLAGS=-I${libXt:location}/include
-  LDFLAGS=-L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib
+  LDFLAGS=-L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${libxkbfile:location}/lib -Wl,-rpath=${libxkbfile:location}/lib
 
 [xserver]
 # Adds Xvfb functionnality

component/zip/buildout.cfg

@@ -3,8 +3,8 @@ parts = zip
 
 [zip]
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://downloads.sourceforge.net/project/infozip/Zip%203.x%20%28latest%29/3.0/zip30.tar.gz
 md5sum = 7b74551e63f8ee6aab6fbc86676c0d37
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = true
-make-options = prefix=${:location} NO_BZIP2_SUPPORT=1 -f unix/Makefile generic
+make-options = prefix=@@LOCATION@@ NO_BZIP2_SUPPORT=1 -f unix/Makefile generic

component/zstd/buildout.cfg

 [buildout]
+extends = ../zlib/buildout.cfg
 parts =
   zstd
 
@@ -10,4 +11,7 @@ md5sum = 487f7ee1562dee7c1c8adf85e2a63df9
 shared = true
 location = @@LOCATION@@
 configure-command = :
-make-options = PREFIX=${:location}
+environment =
+  PREFIX=${:location}
+  LD_FLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  CFLAGS=-I${zlib:location}/include

software/backupserver/test/README.md

+Tests for backupserver software release

software/backupserver/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.backupserver'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' backupserver",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/backupserver/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+import httplib
+import json
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestBackupServer(InstanceTestCase):
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+
+    # Check that there is a RSS feed
+    self.assertTrue('rss' in parameter_dict)
+    self.assertTrue(parameter_dict['rss'].startswith(
+      'https://[%s]:9443/' % (self._ipv6_address, )
+    ))
+
+    result = requests.get(
+      parameter_dict['rss'], verify=False, allow_redirects=False)
+
+    # XXX crontab not triggered yet
+    self.assertEqual(
+      [httplib.NOT_FOUND, False],
+      [result.status_code, result.is_redirect]
+    )
+
+    # Check monitor
+    self.assertTrue('monitor-base-url' in parameter_dict)
+    self.assertTrue('monitor-setup-url' in parameter_dict)
+
+    result = requests.get(
+      parameter_dict['monitor-base-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [httplib.UNAUTHORIZED, False],
+      [result.status_code, result.is_redirect]
+    )

software/caddy-frontend/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = c6d78b2856d9d0ec63728e668e3395d8
+md5sum = 378f6da53a02b2bfe7777a493fc95585
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -320,32 +320,36 @@ extra-context =
 # BBB: SlapOS Master non-zero knowledge END
 
 [caddy-wrapper]
-recipe = slapos.cookbook:wrapper
-environment =
-  CADDYPATH=${directory:frontend_cluster}
-command-line = {{ parameter_dict['caddy'] }}
-  -conf ${dynamic-caddy-frontend-template:rendered}
-  -log ${caddy-configuration:error-log}
-  -log-roll-mb 0
+recipe = slapos.recipe.template:jinja2
+template = inline:
+  #!/bin/sh
+  export CADDYPATH=${directory:frontend_cluster}
+  ulimit -n $(ulimit -Hn)
+  exec {{ parameter_dict['caddy'] }} \
+  -conf ${dynamic-caddy-frontend-template:rendered} \
+  -log ${caddy-configuration:error-log} \
+  -log-roll-mb 0 \
 {% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
-  -http2=false
+  -http2=false \
 {% else %}
-  -http2=true
+  -http2=true \
 {% endif %}
 {% if instance_parameter['configuration.enable-quic'].lower() in TRUE_VALUES %}
-  -quic
+  -quic \
 {% endif %}
-  -grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s
-  -disable-http-challenge
-  -disable-tls-alpn-challenge
-wrapper-path = ${directory:bin}/caddy-wrapper
+  -grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s \
+  -disable-http-challenge \
+  -disable-tls-alpn-challenge \
+   "$@"
+rendered = ${directory:bin}/caddy-wrapper
+mode = 0755
 
 [caddy-frontend]
 recipe = slapos.cookbook:wrapper
-command-line = ${caddy-wrapper:wrapper-path} -pidfile ${caddy-configuration:pid-file}
+command-line = ${caddy-wrapper:rendered} -pidfile ${caddy-configuration:pid-file}
 wrapper-path = ${directory:service}/frontend_caddy
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-hash-files = ${caddy-wrapper:wrapper-path}
+hash-files = ${caddy-wrapper:rendered}
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -594,7 +598,7 @@ rendered = ${directory:bin}/frontend-caddy-validate
 mode = 0700
 last_state_file = ${directory:run}/caddy_configuration_last_state
 extra-context =
-    key wrapper caddy-wrapper:wrapper-path
+    key wrapper caddy-wrapper:rendered
     key caddy_configuration_state frontend-caddy-configuration-state-validate:rendered
     key last_state_file :last_state_file
 

software/caddy-frontend/software.cfg

@@ -27,4 +27,3 @@ smmap = 0.9.0
 numpy = 1.16.4
 
 websockify = 0.8.0
-collective.recipe.shelloutput = 0.1

software/caddy-frontend/test/test.py

@@ -534,7 +534,7 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
       cls.startServerProcess()
     except BaseException:
       cls.logger.exception("Error during setUpClass")
-      cls._cleanup()
+      cls._cleanup("{}.{}.setUpClass".format(cls.__module__, cls.__name__))
       cls.setUp = lambda self: self.fail('Setup Class failed.')
       raise
 
@@ -559,8 +559,9 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
       cls.waitForCaddy()
     except BaseException:
       cls.logger.exception("Error during setUpClass")
-      cls._storeSnapshot("{}.setUpClass".format(cls.__name__))
-      cls._cleanup()
+      # "{}.{}.setUpClass".format(cls.__module__, cls.__name__) is already used by SlapOSInstanceTestCase.setUpClass
+      # so we use another name for snapshot, to make sure we don't store another snapshot in same directory.
+      cls._cleanup("{}.SlaveHttpFrontendTestCase.{}.setUpClass".format(cls.__module__, cls.__name__))
       cls.setUp = lambda self: self.fail('Setup Class failed.')
       raise
 
@@ -854,9 +855,9 @@ class SlaveHttpFrontendTestCase(SlapOSInstanceTestCase):
         break
 
   @classmethod
-  def _cleanup(cls):
+  def _cleanup(cls, snapshot_name):
     cls.stopServerProcess()
-    super(SlaveHttpFrontendTestCase, cls)._cleanup()
+    super(SlaveHttpFrontendTestCase, cls)._cleanup(snapshot_name)
 
   def patchRequests(self):
     HTTPResponse = requests.packages.urllib3.response.HTTPResponse

software/erp5/instance-erp5-input-schema.json

@@ -4,9 +4,7 @@
   "additionalProperties": false,
   "definitions": {
     "tcpv4port": {
-      "minimum": 0,
-      "maximum": 65535,
-      "type": "integer"
+      "$ref": "./schemas-definitions.json#/tcpv4port"
     }
   },
   "properties": {
@@ -194,16 +192,34 @@
     },
     "kumofs": {
       "description": "Persistent memcached service",
-      "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-kumofs-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2000
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "memcached": {
       "description": "Volatile memcached service",
-      "additionalProperties": {
-        "$ref": "./instance-kumofs-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-kumofs-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2010
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "cloudooo-url": {
@@ -218,16 +234,34 @@
     },
     "smtp": {
       "description": "Mail queuing and relay service",
-      "additionalProperties": {
-        "$ref": "./instance-smtp-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-smtp-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2010
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "mariadb": {
       "description": "Relational database service",
-      "additionalProperties": {
-        "$ref": "./instance-mariadb-schema.json"
-      },
+      "allOf": [
+        {
+          "$ref": "./instance-mariadb-schema.json"
+        },
+        {
+          "properties": {
+            "tcpv4-port": {
+              "default": 2099
+            }
+          }
+        }
+      ],
       "type": "object"
     },
     "zodb-zeo": {
@@ -257,7 +291,7 @@
       "type": "object"
     },
     "zodb": {
-      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/3.10/src/ZODB/component.xml for extra options.",
+      "description": "Zope Object DataBase mountpoints. See https://github.com/zopefoundation/ZODB/blob/4/src/ZODB/component.xml for extra options.",
       "items": {
         "required": [
           "type"
@@ -282,8 +316,13 @@
                 "type": "boolean"
               }
             },
+            "patternProperties": {
+              ".!$": {
+                "$ref": "#/properties/zodb/items/patternProperties/.!$"
+              }
+            },
             "additionalProperties": {
-              "type": "string"
+              "$ref": "#/properties/zodb/items/additionalProperties"
             },
             "type": "object"
           },
@@ -313,13 +352,38 @@
               },
               "server": {
                 "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. Partitions references are 'neo-0', 'neo-1', ...",
-                "$ref": "../neoppod/instance-neo-input-schema.json"
+                "$ref": "../neoppod/instance-neo-input-schema.json#/definitions/neo-cluster"
               }
             }
           }
         ],
+        "patternProperties": {
+          ".!$": {
+            "description": "Override with the value of the first item whose zope id matches against the pattern.",
+            "items": {
+              "items": [
+                {
+                  "description": "Override pattern (Python regular expression).",
+                  "type": "string"
+                },
+                {
+                  "description": "Override value (parameter for maching nodes).",
+                  "type": [
+                    "integer",
+                    "string"
+                  ]
+                }
+              ],
+              "type": "array"
+            },
+            "type": "array"
+          }
+        },
         "additionalProperties": {
-          "type": "string"
+          "type": [
+            "integer",
+            "string"
+          ]
         },
         "type": "object"
       },

software/erp5/instance-kumofs-schema.json

@@ -8,7 +8,7 @@
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going upward"

software/erp5/instance-mariadb-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "required": [
-    "tcpv4-port"
-  ],
   "type": "object",
   "properties": {
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going downward"

software/erp5/instance-smtp-schema.json

 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "extends": "./schema-definitions.json#",
   "required": [
     "tcpv4-port"
   ],
@@ -9,7 +8,7 @@
     "tcpv4-port": {
       "allOf": [
         {
-          "$ref": "#/definitions/tcpv4port"
+          "$ref": "./schemas-definitions.json#/tcpv4port"
         },
         {
           "description": "Start allocating ports at this value, going upward"

software/erp5/schemas-definitions.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "tcpv4port": {
+    "minimum": 0,
+    "maximum": 65535,
+    "type": "integer"
+  }
+}

software/erp5/test/test/__init__.py

@@ -46,3 +46,13 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase):
     """Return the output paramters from the root partition"""
     return json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
+
+  def getComputerPartition(self, partition_reference):
+    for computer_partition in self.slap.computer.getComputerPartitionList():
+      if partition_reference == computer_partition.getInstanceParameter(
+          'instance_title'):
+        return computer_partition
+
+  def getComputerPartitionPath(self, partition_reference):
+    partition_id = self.getComputerPartition(partition_reference).getId()
+    return os.path.join(self.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -195,8 +195,8 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     """
     # self.computer_partition_root_path is the path of root partition.
     # we want to assert that no scripts exist in any partition.
-    bin_programs = [os.path.basename(path) for path in
-      glob.glob("{}/../*/bin/*".format(self.computer_partition_root_path))]
+    bin_programs = map(os.path.basename,
+      glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
 
     self.assertTrue(bin_programs) # just to check the glob was correct.
     self.assertNotIn('runUnitTest', bin_programs)
@@ -210,8 +210,99 @@ class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMix
     apache_process = psutil.Process(process_info['pid'])
     self.assertEqual(
         sorted([socket.AF_INET, socket.AF_INET6]),
-        sorted([
+        sorted(
             c.family
             for c in apache_process.connections()
             if c.status == 'LISTEN'
-        ]))
+        ))
+
+class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test override zope node parameters
+  """
+  __partition_reference__ = 'override'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # The following example includes the most commonly used options,
+    # but not necessarily in a meaningful way.
+    return {'_': json.dumps({
+      "zodb": [{
+        "type": "zeo",
+        "server": {},
+        "cache-size-bytes": "20MB",
+        "cache-size-bytes!": [
+          ("bb-0", 1<<20),
+          ("bb-.*", "500MB"),
+        ],
+        "pool-timeout": "10m",
+        "storage-dict": {
+          "cache-size!": [
+            ("a-.*", "50MB"),
+          ],
+        },
+      }],
+      "zope-partition-dict": {
+          "a": {
+              "instance-count": 3,
+          },
+          "bb": {
+              "instance-count": 5,
+              "port-base": 2300,
+          },
+      },
+    })}
+
+  def test_zope_conf(self):
+    zeo_addr = json.loads(
+        self.getComputerPartition('zodb').getConnectionParameter('_')
+      )["storage-dict"]["root"]["server"]
+
+    def checkParameter(line, kw):
+      k, v = line.split()
+      self.assertFalse(k.endswith('!'), k)
+      try:
+        expected = kw.pop(k)
+      except KeyError:
+        if k == 'server':
+          return
+      self.assertIsNotNone(expected)
+      self.assertEqual(str(expected), v)
+
+    def checkConf(zodb, storage):
+      zodb["mount-point"] = "/"
+      zodb["pool-size"] = 4
+      zodb["pool-timeout"] = "10m"
+      storage["storage"] = "root"
+      storage["server"] = zeo_addr
+      with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
+        conf = map(str.strip, f.readlines())
+      i = conf.index("<zodb_db root>") + 1
+      conf = iter(conf[i:conf.index("</zodb_db>", i)])
+      for line in conf:
+        if line == '<zeoclient>':
+          for line in conf:
+            if line == '</zeoclient>':
+              break
+            checkParameter(line, storage)
+          for k, v in storage.iteritems():
+            self.assertIsNone(v, k)
+          del storage
+        else:
+          checkParameter(line, zodb)
+      for k, v in zodb.iteritems():
+        self.assertIsNone(v, k)
+
+    partition = self.getComputerPartitionPath('zope-a')
+    for zope in xrange(3):
+      checkConf({
+          "cache-size-bytes": "20MB",
+        }, {
+          "cache-size": "50MB",
+        })
+    partition = self.getComputerPartitionPath('zope-bb')
+    for zope in xrange(5):
+      checkConf({
+          "cache-size-bytes": "500MB" if zope else 1<<20,
+        }, {
+          "cache-size": None,
+        })

software/erp5testnode/testsuite/deploy-test/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [deploy-script-controller-script]
 filename = deploy-script-controller
-md5sum = f0f5dd379361eb37f84e0bc7639f645f
+md5sum = 8288e59eb442c662544daffbf446a033
 
 [template-deploy-test]
 filename = instance.cfg.in

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

@@ -72,6 +72,8 @@ function add_checks ()
   echo 'iptables-save' >> $LOG_FILE 2>&1
   iptables-save >> $LOG_FILE 2>&1
   for f in /tmp/playbook-* ; do echo $f ; cat $f; echo; done >> $LOG_FILE 2>&1
+  echo 'slapos node status' >> $LOG_FILE 2>&1
+  slapos node status >> $LOG_FILE 2>&1
 }
 function upload ()
 {

software/erp5testnode/testsuite/deploy-test/software.cfg

 [buildout]
 extends =
   buildout.hash.cfg
-  https://lab.nexedi.com/nexedi/slapos/raw/1.0.112/software/kvm/software.cfg
-
-# PyPi servers from new host since some time and change is not incorporated in extended KVM SR
-allow-hosts +=
-  files.pythonhosted.org
+  https://lab.nexedi.com/nexedi/slapos/raw/1.0.134/software/kvm/software.cfg
 
 parts =
   eggs

software/gitlab/buildout.hash.cfg

@@ -66,7 +66,7 @@ md5sum = a56a44e96f65f5ed20211bb6a54279f4
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = abcc5eda03e10b26c74619f299a7f6a8
+md5sum  = e74695aa1be60f0ffac64ddbe1c8eaf1
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -121,6 +121,13 @@ server {
     # Set CORS header
     add_header 'Access-Control-Allow-Origin' {{ cfg('nginx_header_allow_origin') }};
     add_header 'Access-Control-Allow-Credentials' true;
+    if ($request_method = OPTIONS ) {
+        add_header Allow "GET, OPTIONS";
+        add_header Content-Type text/plain;
+        add_header 'Access-Control-Allow-Origin' $http_origin;
+        add_header Access-Control-Allow-Headers "Origin, X-Requested-With, Authorization, Content-Type, Accept";
+        return 200;
+    }
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
     {{ 'gzip off;' if cfg_https else ''}}

software/helloworld/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [instance-profile]
 filename = instance.cfg.in
-md5sum = cc8902e44c1d50804b570775633b8c2a
+md5sum = dcb9d2f540e0e397c9346c8b0c05f233

software/helloworld/instance.cfg.in

@@ -26,11 +26,11 @@ extends = {{ template_monitor }}
 # Always the same. Just copy/paste.
 # See docstring of slapos.cookbook:slapconfiguration for more information.
 recipe = slapos.cookbook:slapconfiguration
-computer = ${slap_connection:computer_id}
-partition = ${slap_connection:partition_id}
-url = ${slap_connection:server_url}
-key = ${slap_connection:key_file}
-cert = ${slap_connection:cert_file}
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
 
 # Define default parameter(s) that will be used later, in case user didn't
 # specify it.

software/htmlvalidatorserver/test/README.md

+Tests for htmlvalidator software release

software/htmlvalidatorserver/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.htmlvalidatorserver'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' htmlvalidatorserver",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/htmlvalidatorserver/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+import httplib
+import json
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestHtmlValidatorServer(InstanceTestCase):
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+
+    # Check that there is a RSS feed
+    self.assertTrue('vnu-url' in parameter_dict)
+    self.assertEqual(
+      'https://[%s]:8899/' % (self._ipv6_address, ),
+      parameter_dict['vnu-url']
+    )
+
+    result = requests.get(
+      parameter_dict['vnu-url'], verify=False, allow_redirects=False)
+
+    self.assertEqual(
+      [httplib.OK, False, 'Apache-Coyote/1.1'],
+      [result.status_code, result.is_redirect, result.headers['Server']]
+    )
+
+    # Check monitor
+    self.assertTrue('monitor-base-url' in parameter_dict)
+    self.assertTrue('monitor-setup-url' in parameter_dict)
+
+    result = requests.get(
+      parameter_dict['monitor-base-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [httplib.UNAUTHORIZED, False],
+      [result.status_code, result.is_redirect]
+    )

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 9f22db89a2679534aa8fd37dbca86782
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = b44268d46a41042a879f47babb66c922
+md5sum = 5cac160fd6f14cd69cc8d63f87cc9726

software/jstestnode/runTestSuite.in

@@ -92,7 +92,7 @@ def main():
         firefox_capabilities['marionette'] = True
         browser = webdriver.Firefox(
             capabilities=firefox_capabilities,
-            firefox_binary='${firefox-wrapper:location}',
+            firefox_binary='${firefox-wrapper-68:location}',
             executable_path='${geckodriver:location}')
       else:
         assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])

software/jstestnode/software.cfg

@@ -18,7 +18,7 @@ parts =
   git
   eggs
   xserver
-  firefox
+  firefox-68
   xwd
   renderjs-install
   jio-install

software/kvm/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 2cbfd6b08c65369c1d45cf3ba2ff335a
+md5sum = a236b719aaac61ac342ada0ce569151a
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 36e7a8656e52d3aace8d9e52dcb3864e
+md5sum = d604d8696815716ac51af770164e86d5
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -27,7 +27,7 @@ md5sum = 2e743132ba4e001f784791311df9ba6a
 
 [template-kvm-resilient]
 filename = instance-kvm-resilient.cfg.jinja2
-md5sum = e50e45c3097ed5a7115817fbc967f173
+md5sum = 7de5756f59ef7d823cd8ed33e6d15230
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
@@ -46,8 +46,8 @@ filename = template/kvm-export.sh.jinja2
 md5sum = b617d64de73de1eed518185f310bbc82
 
 [template-nbd]
-filename = instance-nbd.cfg.in
-md5sum = a05b581d65768ac55faf3b06d4aec447
+filename = instance-nbd.cfg.jinja2
+md5sum = 003112c44ef8d536861a46e3dfc67a37
 
 [template-ansible-promise]
 filename = template/ansible-promise.in
@@ -55,7 +55,7 @@ md5sum = 2036bf145f472f62ef8dee5e729328fd
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 71e0c6c8a0b9a46498f25b67a3ab6894
+md5sum = 08af4ed0e2a53e76c844e3d7325aac09
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -384,8 +384,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": "internet-address",
-              "default": "debian.nbd.vifib.net"
+              "format": "internet-address"
             },
             "nbd-port": {
               "title": "NBD port",

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -64,8 +64,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": "internet-address",
-              "default": "debian.nbd.vifib.net"
+              "format": "internet-address"
             },
             "nbd-port": {
               "title": "NBD port",

software/kvm/instance-kvm-input-schema.json

@@ -172,8 +172,7 @@
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",
       "type": "string",
-      "format": "internet-address",
-      "default": "debian.nbd.vifib.net"
+      "format": "internet-address"
     },
     "nbd-port": {
       "title": "NBD port",

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -13,6 +13,7 @@
 {% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return} -%}
 {% endif -%}
 {% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
+{% set additional_frontend = (slapparameter_dict.get('frontend-additional-instance-guid', '').strip() != '') %}
 
 [buildout]
 eggs-directory = {{ eggs_directory }}
@@ -66,6 +67,9 @@ return =
   url ssh-public-key resilient-ssh-url notification-id ip {{ monitor_return | join(' ') }}
 # KVM related parameters
 # XXX: return ALL parameters (like nat rules), through jinja
+{% if additional_frontend %}
+  url-additional
+{% endif %}
   backend-url url ip
 {{ '  ' }}ipv6-network-info
 
@@ -81,6 +85,9 @@ monitor-user = ${monitor-publish-parameters:monitor-user}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${publish-early:monitor-password}
 {% endif -%}
+{% if additional_frontend %}
+url-additional = ${request-kvm:connection-url-additional}
+{% endif %}
 
 [kvm-frontend-url-bin]
 recipe = collective.recipe.template

software/kvm/instance-kvm.cfg.jinja2

@@ -88,6 +88,7 @@ nbd2-host = ${slap-parameter:nbd2-host}
 nbd2-port = ${slap-parameter:nbd2-port}
 
 tap-interface = {{ slap_configuration.get('tap-name', '') }}
+tap-ipv6-addr = {{ slap_configuration.get('tap-ipv6-addr', '') }}
 
 disk-size = ${slap-parameter:disk-size}
 disk-type = ${slap-parameter:disk-type}
@@ -532,7 +533,7 @@ key_info = Get the publick key file in your VM with the command: wget {{ kvm_htt
 
 {% if use_tap == 'true' and slap_configuration.get('tap-ipv4-addr') -%}
 ipv4-network-info =
-  Use this configuration in /etc/network/interfaces to configure IPv4 on interface {{ iface }} in your VM.
+  PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
   iface {{ iface }} inet static
   	address {{ slap_configuration.get('tap-ipv4-addr') }}
@@ -540,13 +541,14 @@ ipv4-network-info =
   	gateway {{ slap_configuration.get('tap-ipv4-gateway') }}
 {% if enable_http == 'true' %}
   ${helper:blank-line}
-  Or run in your VM the command: wget -O- {{ kvm_http }}/${network-config-ipv4:filename} | /bin/sh -
+  TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv4:filename} | /bin/sh -"
+  (the configuration will be gone after the next reboot)
 {% endif %}
 {% endif %}
 
 ipv6-network-info =
 {% if use_tap == 'true' and slap_configuration.get('tap-ipv6-addr') %}
-  Use this configuration in /etc/network/interfaces to configure IPv6 on interface {{ iface }} in your VM.
+  PERMANENT SOLUTION: in your VM, add the lines below in /etc/network/interfaces and then run: "ifup {{ iface }}"
   auto {{ iface }}
   iface {{ iface }} inet6 static
   	address {{ slap_configuration.get('tap-ipv6-gateway') }}
@@ -554,7 +556,8 @@ ipv6-network-info =
   	gateway {{ slap_configuration.get('tap-ipv6-addr') }}
 {% if enable_http == 'true' %}
   ${helper:blank-line}
-  Or run in your VM the command: wget -O- {{ kvm_http }}/${network-config-ipv6:filename} | /bin/sh -
+  TEMPORARY SOLUTION: run in your VM the command: "wget -O- {{ kvm_http }}/${network-config-ipv6:filename} | /bin/sh -"
+  (the configuration will be gone after the next reboot)
 {% endif %}
 {% endif %}
 
@@ -820,6 +823,14 @@ template = inline:
 rendered = ${buildout:directory}/.slapos-disk-permission
 context =
   raw disk_device_path {{disk_device_path}}
+
+{%   do part_list.append('wipe-disk-device-wrapper') -%}
+[wipe-disk-device-wrapper]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:prerm}/slapos_wipe_device_disk
+command-line =
+  dd if=/dev/zero of={{disk_device_path}} bs=4096 count=500k
+
 {% endif -%}
 
 [buildout]

software/kvm/instance-nbd.cfg.in

-#############################
-#
-# Instanciate nbdserver
-#
-#############################
-[buildout]
-parts =
-  nbd-promise
-  onetimeupload-promise
-  publish-connection-information
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-srv = $${buildout:directory}/srv
-log = $${buildout:directory}/log
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-services = $${rootdirectory:etc}/run
-watched-services = $${rootdirectory:etc}/service
-
-[nbd-instance]
-recipe = slapos.cookbook:nbdserver
-ip = $${slap-network-information:global-ipv6}
-port = 1024
-image-path = $${onetimeupload-instance:image-path}
-qemu-path = ${kvm:location}/bin/qemu-nbd
-shell-path = ${dash:location}/bin/dash
-# XXX TODO: Wait for the iso to be uploaded (execute_wait)
-path = $${basedirectory:services}/nbdserver
-
-[nbd-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = nbd_promise.py
-config-hostname = $${nbd-instance:ip}
-config-port = $${nbd-instance:port}
-
-[gen-passwd]
-recipe = slapos.cookbook:generate.password
-storage-path = $${rootdirectory:srv}/passwd
-bytes = 24
-
-[onetimeupload-instance]
-recipe = slapos.cookbook:onetimeupload
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-image-path = $${rootdirectory:srv}/cdrom.iso
-log-path = $${rootdirectory:log}/onetimeupload.log
-shell-path = ${dash:location}/bin/dash
-onetimeupload-path = ${buildout:bin-directory}/onetimeupload
-path = $${basedirectory:watched-services}/onetimeupload
-key = $${gen-passwd:passwd}
-
-[onetimeupload-promise]
-<= monitor-promise-base
-module = check_port_listening
-name = onetimeupload_promise.py
-config-hostname = $${onetimeupload-instance:ip}
-config-port = $${onetimeupload-instance:port}
-
-[publish-connection-information]
-recipe = slapos.cookbook:publish
-nbd_url = nbd://[$${nbd-instance:ip}]:$${nbd-instance:port}
-upload_url = http://[$${onetimeupload-instance:ip}]:$${onetimeupload-instance:port}/
-upload_key = $${onetimeupload-instance:key}

software/kvm/instance-nbd.cfg.jinja2

+#############################
+#
+# Instanciate nbdserver
+#
+#############################
+[buildout]
+parts =
+  nbd-promise
+  onetimeupload-promise
+  publish-connection-information
+extends = {{ template_monitor }}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+log = ${buildout:directory}/log
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = ${rootdirectory:etc}/run
+watched-services = ${rootdirectory:etc}/service
+
+[nbd-instance]
+recipe = slapos.cookbook:nbdserver
+ip = ${slap-network-information:global-ipv6}
+port = 1024
+image-path = ${onetimeupload-instance:image-path}
+qemu-path =  {{ qemu_nbd_executable_location }}
+shell-path = {{ dash_executable_location }}
+# XXX TODO: Wait for the iso to be uploaded (execute_wait)
+path = ${basedirectory:services}/nbdserver
+
+[nbd-checker-bin]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  [ ! -f ${onetimeupload-instance:image-path} ] ||
+    ${buildout:executable} -c 'import socket ; socket.create_connection(("${nbd-instance:ip}","${nbd-instance:port}")).close()'
+rendered = ${rootdirectory:bin}/check-nbd-running.sh
+
+[nbd-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = nbd_promise.py
+config-command = ${nbd-checker-bin:rendered}
+
+[gen-passwd]
+recipe = slapos.cookbook:generate.password
+storage-path = ${rootdirectory:srv}/passwd
+bytes = 24
+
+[onetimeupload-instance]
+recipe = slapos.cookbook:onetimeupload
+ip = ${slap-network-information:global-ipv6}
+port = {{ slapparameter_dict.get('otu-port', 8080) }}
+image-path = ${rootdirectory:srv}/cdrom.iso
+log-path = ${rootdirectory:log}/onetimeupload.log
+shell-path = {{ dash_executable_location }}
+onetimeupload-path = {{ onetimeupload_executable_location }}
+path = ${basedirectory:watched-services}/onetimeupload
+key = ${gen-passwd:passwd}
+
+[onetimeupload-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = onetimeupload_promise.py
+config-hostname = ${onetimeupload-instance:ip}
+config-port = ${onetimeupload-instance:port}
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+nbd_hostname = ${nbd-instance:ip}
+nbd_port = ${nbd-instance:port}
+upload_url = http://[${onetimeupload-instance:ip}]:${onetimeupload-instance:port}
+upload_key = ${onetimeupload-instance:key}
+status_message = ${detect-if-cdrom-present:status}
+
+[detect-if-cdrom-present]
+recipe = collective.recipe.shelloutput
+commands =
+  status = [ -f ${onetimeupload-instance:image-path} ] && echo "image already uploaded, you can't upload it again" || echo "WARNING: no image yet, the NBD server doesn't work"

software/kvm/instance.cfg.in

@@ -12,7 +12,7 @@ recipe = slapos.cookbook:softwaretype
 default = $${:kvm}
 kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
 kvm = $${dynamic-template-kvm:rendered}
-nbd = ${template-nbd:output}
+nbd = $${dynamic-template-nbd:rendered}
 
 kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
 kvm-import = $${dynamic-template-kvm-import:rendered}
@@ -149,3 +149,16 @@ context =
     raw gzip_binary ${gzip:location}/bin/gzip
 mode = 0644
 
+[dynamic-template-nbd]
+<= jinja2-template-base
+template = ${template-nbd:location}/instance-nbd.cfg.jinja2
+filename = template-nbd.cfg
+context =
+  section slap_configuration slap-configuration
+  key slapparameter_dict slap-configuration:configuration
+  key eggs_directory buildout:eggs-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  raw qemu_nbd_executable_location ${kvm:location}/bin/qemu-nbd
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw onetimeupload_executable_location ${buildout:bin-directory}/onetimeupload
+  raw template_monitor ${monitor2-template:rendered}

software/kvm/software.cfg

@@ -144,8 +144,8 @@ filename = kvm-export.sh.jinja2
 mode = 0755
 
 [template-nbd]
-<= template-file-base
-output = ${buildout:directory}/template-nbd.cfg
+<= download-file-base
+on-update = true
 
 [template-ansible-promise]
 <= download-template-base
@@ -196,7 +196,6 @@ context =
 websockify = 0.5.1
 
 collective.recipe.environment = 0.2.0
-collective.recipe.shelloutput = 0.1
 gitdb = 0.6.4
 pycurl = 7.43.0
 slapos.recipe.template = 4.3

software/kvm/template/template-kvm-run.in

@@ -44,6 +44,7 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
 listen_ip = '{{ parameter_dict.get("ipv4") }}'
 mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
+tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
 numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
 ram_size = {{ parameter_dict.get("ram-size") }}
 ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
@@ -278,6 +279,8 @@ if use_nat == 'true':
                                            cluster_doc_host, cluster_doc_port)
   if set_nat_restrict == 'true':
     rules += ',restrict=on'
+  if use_tap == 'true' and tap_ipv6_addr != '':
+    rules += ',ipv6=off'
   nat_network_parameter = ['-netdev', rules,
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, mac_address)]
 if use_tap == 'true':

software/kvm/test/test.py

@@ -308,3 +308,63 @@ class TestInstanceResilient(InstanceTestCase):
         'takeover-kvm-1-password',
         'takeover-kvm-1-url',
         'url']))
+
+@unittest.skipIf(not sanityCheck(), 'missing kvm_intel module')
+class TestAccessResilientAdditional(InstanceTestCase):
+  __partition_reference__ = 'ara'
+  expected_partition_with_monitor_base_url_count = 1
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-resilient'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'frontend-additional-instance-guid': 'SOMETHING'
+    }
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+
+    result = requests.get(connection_parameter_dict['url'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
+
+    result = requests.get(
+      connection_parameter_dict['url-additional'], verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>noVNC</title>', result.text)
+
+class TestInstanceNbdServer(InstanceTestCase):
+  __partition_reference__ = 'ins'
+  instance_max_retry = 5
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'nbd'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # port 8080 is used by testnode, use another one
+    return {
+      'otu-port': '8090'
+    }
+
+  def test(self):
+    connection_parameter_dict = self.computer_partition\
+      .getConnectionParameterDict()
+    result = requests.get(connection_parameter_dict['upload_url'].strip(), verify=False)
+    self.assertEqual(
+      httplib.OK,
+      result.status_code
+    )
+    self.assertIn('<title>Upload new File</title>', result.text)
+    self.assertIn("WARNING", connection_parameter_dict['status_message'])

software/monitor/README.edgetest.rst

+============
+Edge testing
+============
+
+``edgetest`` is a special software type of monitor software release used for website monitoring by using bots.
+
+It uses `surykatka <https://lab.nexedi.com/nexedi/surykatka>`_ and `check_surykatka_json <https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_surykatka_json.py>`_ to monitor websites.
+
+``surykatka`` provides a bot to query list of hosts and a JSON reporting system.
+
+``check_surykatka_json`` is used in promises to provide monitoring information about the websites.
+
+In order to monitor an url one need to:
+
+ * request a monitor software release with ``edgetest`` software type, configured as described in ``instance-edgetest-input-schema.json``,
+ * request a slave to monitor with ``edgetest`` software type, configured as described in ``instance-edgetest-slave-input-schema.json``.

software/monitor/buildout.hash.cfg

@@ -14,16 +14,32 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 1b7d2d097f208f6641bf98a17df079c8
+md5sum = d778b6f436ae6864819eb2ff2d12a86f
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
-md5sum = 373c79480e6425c20480fc911a56c3fd
-
-[template-monitor-distributor]
-_update_hash_filename_ = instance-monitor-distributor.cfg.jinja2
-md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
+md5sum = 165a15672fc85981f68b9af2d6253254
 
 [json-test-template]
 _update_hash_filename_ = json-test-template.json.in.jinja2
 md5sum = 2eb5596544d9c341acf653d4f7ce2680
+
+[template-monitor-edgetest]
+_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
+md5sum = 9e237dbdda59e788202f0da194a57d41
+
+[template-monitor-edgebot]
+_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
+md5sum = 8786e4245db0d27dfa4815222d970e52
+
+[network-bench-cfg]
+filename = network_bench.cfg.in
+md5sum = cfcbf2002b8eff5153e2bf68ed24b720
+
+[monitor-collect-csv-dump]
+filename = script/collect_csv_dump.py
+md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
+
+[template-surykatka-ini]
+_update_hash_filename_ = surykatka.ini.jinja2
+md5sum = 40870921e05d93b5843ab34abd7e3902

software/monitor/instance-default-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "dummy": {
+      "title": "dummy",
+      "description": "Dummy",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-default-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Re6st Master instanciation",
+  "properties": {
+    "re6stry-url": {
+      "description": "ipv6 url to access your re6st registry service",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/monitor/instance-edgetest-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "nameserver": {
+      "default": "",
+      "title": "Nameserver",
+      "description": "Space separated list of name servers to use.",
+      "type": "string"
+    },
+    "check-status-code": {
+      "default": "200",
+      "title": "Default Check HTTP Code",
+      "description": "Default HTTP code to check against (default: 200).",
+      "type": "string"
+    },
+    "check-frontend-ip": {
+      "default": "",
+      "title": "Default space separated list of Frontend IPs to check",
+      "description": "Default list of Frontend IPs to check, if empty no constraint is used.",
+      "type": "string"
+    },
+    "check-certificate-expiration-days": {
+      "default": "15",
+      "title": "Default certificate expiration days check",
+      "description": "Default amount of days to consider certitifcate as being to-be-expired (default: 15).",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-edgetest-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "url": {
+      "title": "URL to check",
+      "description": "URL to check, like https://example.com",
+      "type": "string"
+    },
+    "check-status-code": {
+      "default": "Master default",
+      "title": "Default Check HTTP Code.",
+      "description": "HTTP code to check against (default: comes from master partition).",
+      "type": "string"
+    },
+    "check-frontend-ip": {
+      "default": "Master default",
+      "title": "Space separated list of Frontend IPs to check",
+      "description": "List of Frontend IPs to check, if empty no constraint is used (default: comes from master partition).",
+      "type": "string"
+    },
+    "check-certificate-expiration-days": {
+      "default": "Master default",
+      "title": "Certificate expiration days check",
+      "description": "Default amount of days to consider certitifcate as being to-be-expired (default: comes from master partition).",
+      "type": "string"
+    }
+  }
+}

software/monitor/instance-monitor-distributor.cfg.jinja2

-
-[jinja2-template-base]
-recipe = slapos.recipe.template:jinja2
-rendered = ${buildout:directory}/${:filename}
-extra-context =
-context =
-    import json_module json
-    ${:extra-context}
-
-[slave-test-configuration]
-<=jinja2-template-base
-template = {{ template_json_distributor_test }}
-filename = srv/monitor/private/test.json
-extensions = jinja2.ext.do 
-extra-context = 
-  section slave_information slap-configuration
-
-{% set part_list = [] -%}
-# Publish information for each slave
-
-{% set directory_list = [] -%}
-
-{% for slave_instance in slave_instance_list -%}
-{%   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
-{%   do part_list.append(publish_section_title) -%}
-[{{ publish_section_title }}]
-recipe = slapos.cookbook:publish
--slave-reference = {{ slave_instance.get('slave_reference') }}
-log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
-log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
-{% endfor %}
-
-
-{% set data_source_dict = slapparameter_dict.get('data-source', None)  -%}
-{% set cron_min_count = 0 -%}
-{% if data_source_dict %}
-{% for entry in data_source_dict -%}
-{% set cron_min_count = cron_min_count + 1 -%}
-{% set cron_min = cron_min_count%60 -%}
-{% do part_list.append('cron-crawl-' + entry) -%}
-{% do directory_list.append(entry) -%}
-[cron-crawl-{{ entry }}]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = cron-crawler-{{ entry }}
-frequency = * * * * *
-command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }} 
-{% endfor %}
-{% endif %}
-
-[monitor-directory]
-crawl-log = ${:srv}/crawlog
-network-user-logs = ${:private}/network-user-logs/
-{% for slave_instance in slave_instance_list -%}
-user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}
-user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
-user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
-
-{% endfor -%}
-{% for directory in directory_list %}
-{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
-{% endfor %}
-
-
-[crawler-bin]
-recipe = slapos.cookbook:wrapper
-command-line = 
-  {{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp  
-wrapper-path = ${monitor-directory:bin}/log-crawler
-
-[buildout]
-extends = {{ instance_base_monitor }}
-parts +=
-  slave-test-configuration
-{% for part in part_list %}
-{{ '  %s' % part }}
-{% endfor %}
-

software/monitor/instance-monitor-edgebot.cfg.jinja2

+{%- if slap_software_type == software_type %}
+{%- set CONFIGURATION = {} %}
+{%- for k, v in sorted(slap_configuration.items()) %}
+{%-   if k.startswith('configuration.') %}
+{%-     do CONFIGURATION.__setitem__(k[14:], v) %}
+{%-   endif %}
+{%- endfor %}
+{%- set slave_instance_list = [] %}
+{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
+{%- if extra_slave_instance_list %}
+{#- Create slaves to process with setting up defaults #}
+{%-   for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
+{%-   if 'check-status-code' not in slave %}
+{%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
+{%-   endif %}
+{%-   if 'check-certificate-expiration-days' not in slave %}
+{%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
+{%-   endif %}
+{%-   if 'check-frontend-ip' not in slave %}
+{%-     do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
+{%-   endif %}
+{%-   if 'url' in slave %}
+{%-     do slave_instance_list.append(slave) %}
+{%-   endif %}
+{%-   endfor %}
+{%- endif %}
+{%- set part_list = [] %}
+{%- for slave in sorted(slave_instance_list) %}
+{%-   set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
+{%-   do part_list.append(part_id) %}
+{%-   set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
+[{{part_id}}]
+<= monitor-promise-base
+module = check_surykatka_json
+name = {{ safe_name }}.py
+config-report = http_query
+config-url = {{ slave['url'] }}
+config-status-code = {{ slave['check-status-code'] }}
+config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
+config-ip-list = {{ slave['check-frontend-ip'] }}
+config-json-file = ${surykatka-config:json}
+{% endfor %}
+
+[surykatka-bot-promise]
+<= monitor-promise-base
+module = check_surykatka_json
+name = surykatka-bot-promise.py
+config-report = bot_status
+config-json-file = ${surykatka-config:json}
+
+[buildout]
+extends = {{ monitor_template_output }}
+
+parts =
+  cron
+  cron-entry-surykatka-status
+  monitor-base
+  publish-connection-information
+  surykatka
+  surykatka-bot-promise
+{% for part_id in sorted(part_list) %}
+  {{ part_id }}
+{% endfor %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[surykatka-config]
+recipe = slapos.recipe.template:jinja2
+db = ${directory:srv}/surykatka.db
+rendered = ${directory:etc}/surykatka.ini
+template = {{ template_surykatka_ini }}
+slave_instance_list = {{ dumps(slave_instance_list) }}
+nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
+json = ${directory:srv}/surykatka.json
+
+context =
+  import json_module json
+  key db :db
+  key nameserver :nameserver
+  key slave_instance_list :slave_instance_list
+
+[surykatka]
+recipe = slapos.cookbook:wrapper
+config = ${surykatka-config:rendered}
+command-line =
+  {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
+wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[surykatka-status-json]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
+    mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
+  else
+    rm -f ${surykatka-config:json}.tmp
+  fi
+rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
+mode = 0755
+
+[cron-entry-surykatka-status]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${directory:etc}/cron.d
+name = surykatka-status
+frequency = */2 * * * *
+command = ${surykatka-status-json:rendered}
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish.serialised
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
+
+[monitor-instance-parameter]
+monitor-httpd-port = {{ slapparameter_dict['monitor-httpd-port'] }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{% if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% endif -%}
+interface-url = {{ slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') }}
+
+[monitor-directory]
+service = ${buildout:directory}/etc/service
+var = ${buildout:directory}/var
+srv = ${buildout:directory}/srv
+server-log = ${:private}/server-log
+monitor-log = ${:private}/monitor-log
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-parameter]
+{%- endif %}

software/monitor/instance-monitor-edgetest.cfg.jinja2

+{%- if slap_software_type == software_type %}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extra-context =
+context =
+    import json_module json
+    ${:extra-context}
+
+[slave-test-configuration]
+<=jinja2-template-base
+template = {{ template_json_edgetest_test }}
+filename = srv/monitor/private/test.json
+extensions = jinja2.ext.do 
+extra-context = 
+  section slave_information slap-configuration
+
+{% set part_list = [] -%}
+# Publish information for each slave
+{%- set edgebot_software_type = 'edgebot' %}
+{%- set edgebot_quantity = slapparameter_dict.pop('edgebot-quantity', '1') | int %}
+{%- set edgebot_list = [] %}
+{%- set edgebot_section_list = [] %}
+{%- set slave_list_name = 'extra_slave_instance_list' %}
+{%- set request_dict = {} %}
+{%- set namebase = "edgebot" %}
+{%- set authorized_slave_list = [] %}
+{%- set monitor_base_url_dict = {} -%}
+{%- for slave in sorted(slave_instance_list) %}
+{%-   do authorized_slave_list.append(slave) %}
+{%- endfor %}
+{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
+{%- for i in range(1, edgebot_quantity + 1) %}
+{%-   set edgebot_name = "%s-%s" % (namebase, i) %}
+{%-   set request_section_title = 'request-%s' % edgebot_name %}
+{%-   do edgebot_list.append(edgebot_name) %}
+{%-   do edgebot_section_list.append(request_section_title) %}
+{%-   do part_list.append(request_section_title) %}
+{%-   do request_dict.__setitem__(request_section_title,
+                                  {
+                                  'config': {'monitor-httpd-port': monitor_base_port + i},
+                                  'name': edgebot_name,
+                                  'sla': {},
+                                  'state': 'started', 
+                                  }) %}
+{%- endfor %}
+
+[replicate]
+<= slap-connection
+recipe = slapos.cookbook:request.serialised
+config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+config-monitor-username = ${monitor-instance-parameter:username}
+config-monitor-password = ${monitor-htpasswd:passwd}
+software-url = ${slap-connection:software-release-url}
+software-type = {{edgebot_software_type}}
+return = monitor-base-url
+
+{% for section, edgebot_request in request_dict.iteritems() %}
+[{{section}}]
+<= replicate
+name = {{ edgebot_request.get('name') }}
+{%-   if edgebot_request.get('state') %}
+state = {{ edgebot_request.get('state') }}
+{%-   endif%}
+{%-   set slave_configuration_dict = slapparameter_dict %}
+{%-   do slave_configuration_dict.update(edgebot_request.get('config')) %}
+{%-   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
+{%-   for config_key, config_value in slave_configuration_dict.iteritems() %}
+config-{{ config_key }} = {{ dumps(config_value) }}
+{%   endfor -%}
+{%-   if edgebot_request.get('sla') %}
+{%-     for parameter, value in edgebot_request.get('sla').iteritems() %}
+sla-{{ parameter }} = {{ value }}
+{%-     endfor %}
+{%-   else %}
+# As no SLA was provided, by default it is requested on the same computer
+sla-computer_guid = ${slap-connection:computer-id}
+{%   endif %}
+{%-  do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{%- endfor %}
+{%- set directory_list = [] -%}
+{%- for slave_instance in slave_instance_list -%}
+{%-   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
+{%-   do part_list.append(publish_section_title) %}
+[{{ publish_section_title }}]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_instance.get('slave_reference') }}
+{% endfor %}
+
+[monitor-conf-parameters]
+monitor-title = Monitor
+password = ${monitor-htpasswd:passwd}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+
+[buildout]
+extends = {{ instance_base_monitor }}
+parts +=
+  slave-test-configuration
+{% for part in part_list %}
+{{ '  %s' % part }}
+{%- endfor %}
+{%- endif %}

software/monitor/instance-monitor.cfg.jinja2

@@ -57,14 +57,15 @@ recipe = slapos.cookbook:generate.password
 user = admin
 bytes = 16
 
+[monitor-instance-parameter]
+monitor-httpd-port = {{ slap_configuration['configuration.monitor-base-port'] }}
+
 [monitor-directory]
 service = ${buildout:directory}/etc/service
 var = ${buildout:directory}/var
 srv = ${buildout:directory}/srv
 server-log = ${:private}/server-log
 monitor-log = ${:private}/monitor-log
-cache = ${:var}/cache
-mod-ssl = ${:cache}/httpd_mod_ssl
 system-log = ${:private}/system-log
 consumption = ${:log}/consumption
 

software/monitor/instance.cfg

@@ -8,38 +8,65 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${instance-base-monitor:rendered}
-distributor = $${instance-base-distributor:rendered}
+edgetest = $${instance-base-edgetest:rendered}
+edgebot = $${instance-base-edgebot:rendered}
 
 [instance-base-monitor]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor:destination}
+template = ${template-monitor:target}
 rendered = $${buildout:directory}/template-base-monitor.cfg
 extensions = jinja2.ext.do
 context = key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
+          section slap_configuration slap-configuration
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}
           raw network_benck_cfg_output ${network-bench-cfg:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
 mode = 0644
 
-[instance-base-distributor]
+[instance-base-edgetest]
 recipe = slapos.recipe.template:jinja2
-template = ${template-monitor-distributor:destination}
-rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
+template = ${template-monitor-edgetest:target}
+rendered = $${buildout:directory}/template-monitor-base-edgetest.cfg
 extensions = jinja2.ext.do
 context = import json_module json
           key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
+          key slap_software_type slap-configuration:slap-software-type
+          section slap_configuration slap-configuration
+          raw software_type edgetest
           key instance_base_monitor instance-base-monitor:rendered
           key slave_instance_list slap-configuration:slave-instance-list 
           raw buildout_bin ${buildout:bin-directory}
-          raw template_json_distributor_test ${json-test-template:destination}
-          raw wget_bin ${wget:location}/bin/wget
+          raw template_json_edgetest_test ${json-test-template:target}
 mode = 0644
 
+[instance-base-edgebot]
+recipe = slapos.recipe.template:jinja2
+template = ${template-monitor-edgebot:target}
+rendered = $${buildout:directory}/template-monitor-edgebot.cfg
+extensions = jinja2.ext.do
+surykatka-binary = ${surykatka:executable}
+template-surykatka-ini = ${template-surykatka-ini:target}
+
+context = import json_module json
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          section slap_configuration slap-configuration
+          key slapparameter_dict slap-configuration:configuration
+          key slap_software_type slap-configuration:slap-software-type
+          raw software_type edgebot
+          key surykatka_binary :surykatka-binary
+          key template_surykatka_ini :template-surykatka-ini
+          raw buildout_bin ${buildout:bin-directory}
+          raw monitor_template_output ${monitor-template:output}
+          raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
+mode = 0644
+
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -47,3 +74,13 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+# Defaults
+configuration.check-status-code = 200
+configuration.nameserver =
+configuration.check-frontend-ip =
+configuration.check-certificate-expiration-days = 15
+# use monitor-base-port to have monitor listening on each instance
+# on different port and also on different port than other services
+# it makes it possible to instantiate it correctly on signle IP, for
+# example in case of webrunner
+configuration.monitor-base-port = 9700

software/monitor/software.cfg

@@ -4,17 +4,16 @@ extends =
   buildout.hash.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
-  ../../component/wget/buildout.cfg
+  ../../component/surykatka/buildout.cfg
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
 
 parts = 
-  wget
   slapos-cookbook
   network-bench-cfg
   json-test-template
   template
-  template-monitor-distributor
+  template-monitor-edgetest
   template-monitor
   monitor-collect-csv-dump
 
@@ -27,26 +26,31 @@ mode = 0644
 [template-monitor]
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/template-base-monitor.cfg
 mode = 0644
 
-[template-monitor-distributor]
+[template-monitor-edgetest]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/template-monitor-base-distributor.cfg
 mode = 0644
 
+[template-monitor-edgebot]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
+[template-surykatka-ini]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
 [json-test-template]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
-destination = ${buildout:directory}/json-test-template.json.in.jinja2
 mode = 0644
 
 [network-bench-cfg]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/network_bench.cfg.in
-md5sum  = cfcbf2002b8eff5153e2bf68ed24b720
-output = ${buildout:directory}/template-network-bench-cfg.in
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
 mode = 0644
 
 [monitor-collect-csv-dump]
@@ -54,7 +58,6 @@ mode = 0644
 url = ${:_profile_base_location_}/script/${:filename}
 filename = collect_csv_dump.py
 output = ${:destination}/${:filename}
-md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
 [extra-eggs]
 <= monitor-eggs
@@ -83,6 +86,5 @@ eggs +=
   hexagonit.recipe.download
   plone.recipe.command
 
-
 [versions]
 slapos.recipe.template = 4.3

software/monitor/software.cfg.json

+{
+  "name": "Monitor",
+  "description": "Software release for Monitoring purpose",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Standalone Monitor",
+      "request": "instance-default-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 0
+    },
+    "edgetest": {
+      "title": "Edge Test",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 1
+    },
+    "edgetest-slave": {
+      "title": "Edge Test Slave",
+      "software-type": "edgetest",
+      "description": "Cluster of bots to perform a distributed monitoring ",
+      "request": "instance-edgetest-slave-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "index": 2
+    }
+  }
+}

software/monitor/surykatka.ini.jinja2

+[SURYKATKA]
+INTERVAL = 120
+SQLITE = {{ db }}
+{%- set nameserver_list = nameserver.split() %}
+{%- if len(nameserver_list) > 0 %}
+NAMESERVER =
+{%-   for nameserver_entry in sorted(nameserver_list) %}
+  {{ nameserver_entry }}
+{%-   endfor %}
+{% endif %}
+URL =
+{%- for slave in sorted(slave_instance_list) %}
+{%-   if 'url' in slave %}
+  {{ slave['url'] }}
+{%-   endif -%}
+{% endfor %}
+

software/monitor/test/test.py

@@ -25,10 +25,13 @@
 #
 ##############################################################################
 
+import json
 import os
-
+import re
+import requests
+import subprocess
+import xml.etree.ElementTree as ET
 from slapos.recipe.librecipe import generateHashFromFiles
-
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -59,3 +62,431 @@ class ServicesTestCase(SlapOSInstanceTestCase):
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)
+
+
+class MonitorTestMixin(object):
+  monitor_setup_url_key = 'monitor-setup-url'
+
+  def test_monitor_setup(self):
+    connection_parameter_dict = self\
+      .computer_partition.getConnectionParameterDict()
+    self.assertTrue(
+      self.monitor_setup_url_key in connection_parameter_dict,
+      '%s not in %s' % (self.monitor_setup_url_key, connection_parameter_dict))
+    monitor_setup_url_value = connection_parameter_dict[
+      self.monitor_setup_url_key]
+    monitor_url_match = re.match(r'.*url=(.*)', monitor_setup_url_value)
+    self.assertNotEqual(
+      None, monitor_url_match, '%s not parsable' % (monitor_setup_url_value,))
+    self.assertEqual(1, len(monitor_url_match.groups()))
+    monitor_url = monitor_url_match.groups()[0]
+    monitor_url_split = monitor_url.split('&')
+    self.assertEqual(
+      3, len(monitor_url_split), '%s not splitabble' % (monitor_url,))
+    self.monitor_url = monitor_url_split[0]
+    monitor_username = monitor_url_split[1].split('=')
+    self.assertEqual(
+      2, len(monitor_username), '%s not splittable' % (monitor_username))
+    monitor_password = monitor_url_split[2].split('=')
+    self.assertEqual(
+      2, len(monitor_password), '%s not splittable' % (monitor_password))
+    self.monitor_username = monitor_username[1]
+    self.monitor_password = monitor_password[1]
+
+    opml_text = requests.get(self.monitor_url, verify=False).text
+    opml = ET.fromstring(opml_text)
+
+    body = opml[1]
+    self.assertEqual('body', body.tag)
+
+    outline_list = body[0].findall('outline')
+
+    self.assertEqual(
+      self.monitor_configuration_list,
+      [q.attrib for q in outline_list]
+    )
+
+    expected_status_code_list = []
+    got_status_code_list = []
+    for monitor_configuration in self.monitor_configuration_list:
+      status_code = requests.get(
+          monitor_configuration['url'],
+          verify=False,
+          auth=(self.monitor_username, self.monitor_password)
+        ).status_code
+      expected_status_code_list.append(
+        {
+          'url': monitor_configuration['url'],
+          'status_code': 200
+        }
+      )
+      got_status_code_list.append(
+        {
+          'url': monitor_configuration['url'],
+          'status_code': status_code
+        }
+      )
+    self.assertEqual(
+      expected_status_code_list,
+      got_status_code_list
+    )
+
+
+class EdgeSlaveMixin(MonitorTestMixin):
+  __partition_reference__ = 'edge'
+  instance_max_retry = 20
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'edgetest'
+
+  def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
+    software_url = self.getSoftwareURL()
+    self.slap.request(
+      software_release=software_url,
+      software_type='edgetest',
+      partition_reference=partition_reference,
+      partition_parameter_kw=partition_parameter_kw,
+      shared=True
+    )
+
+  def setUp(self):
+    self.bot_partition_path = os.path.join(
+      self.slap.instance_directory,
+      self.__partition_reference__ + '1')
+    self.surykatka_json = os.path.join(
+        self.bot_partition_path, 'srv', 'surykatka.json')
+    self.surykatka_status_json = os.path.join(
+        self.bot_partition_path, 'bin', 'surykatka-status-json')
+    self.monitor_configuration_list = [
+      {
+        'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'testing partition 0',
+        'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
+        'text': 'testing partition 0',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
+      },
+      {
+        'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
+        'version': 'RSS',
+        'title': 'edgebot-1',
+        'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
+        'text': 'edgebot-1',
+        'type': 'rss',
+        'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
+      }
+    ]
+
+  def assertSurykatkaIni(self):
+    surykatka_ini = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
+
+    expected = self.surykatka_ini % dict(
+        partition_path=self.bot_partition_path)
+    self.assertEqual(
+      expected.strip(),
+      surykatka_ini)
+
+  def assertPromiseContent(self, name, content):
+    promise = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'plugin', name
+      )).read().strip()
+
+    self.assertTrue(content in promise)
+
+  def assertSurykatkaBotPromise(self):
+    self.assertPromiseContent(
+      'surykatka-bot-promise.py',
+      "'report': 'bot_status'")
+    self.assertPromiseContent(
+      'surykatka-bot-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def assertSurykatkaCron(self):
+    surykatka_cron = open(
+      os.path.join(
+        self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
+        ).read().strip()
+    self.assertEqual(
+      '*/2 * * * * %s' % (self.surykatka_status_json,),
+      surykatka_cron
+    )
+
+  def initiateSurykatkaRun(self):
+    try:
+      self.slap.waitForInstance(max_retry=2)
+    except Exception:
+      pass
+
+  def assertSurykatkaStatusJSON(self):
+    if os.path.exists(self.surykatka_json):
+      os.unlink(self.surykatka_json)
+    env = os.environ.copy()
+    env.pop('PYTHONPATH', None)
+    subprocess.check_call(self.surykatka_status_json, shell=True, env=env)
+    self.assertTrue(os.path.exists(self.surykatka_json))
+    with open(self.surykatka_json) as fh:
+      status_json = json.load(fh)
+    self.assertIn('bot_status', status_json)
+
+  def test(self):
+    # Note: Those tests do not run surykatka and do not do real checks, as
+    #       this depends too much on the environment and is really hard to
+    #       mock
+    #       So it is possible that some bugs might slip under the radar
+    #       Nevertheless the surykatka and check_surykatka_json are heavily
+    #       unit tested, and configuration created by the profiles is asserted
+    #       here, so it shall be enough as reasonable status
+    self.requestEdgetestSlaves()
+    self.initiateSurykatkaRun()
+    self.assertSurykatkaStatusJSON()
+    self.assertSurykatkaIni()
+    self.assertSurykatkaBotPromise()
+    self.assertSurykatkaPromises()
+    self.assertSurykatkaCron()
+
+
+class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'status-code': '300'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-300-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-300',
+      {'url': 'https://www.erp5.org/', 'check-status-code': '300'},
+    )
+
+
+class TestEdgeNameserverCheckFrontendIp(
+  EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+NAMESERVER =
+  127.0.1.1
+  127.0.1.2
+
+URL =
+  https://www.erp5.com/"""
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'nameserver': '127.0.1.1 127.0.1.2',
+      'check-frontend-ip': '127.0.0.1 127.0.0.2',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': '127.0.0.1 127.0.0.2'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+
+
+class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-status-code': '500',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'status-code': '501'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-501-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '500'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '15'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-501',
+      {'url': 'https://www.erp5.org/', 'check-status-code': '501'},
+    )
+
+
+class TestEdgeCheckCertificateExpirationDays(
+  EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_ini = """[SURYKATKA]
+INTERVAL = 120
+SQLITE = %(partition_path)s/srv/surykatka.db
+URL =
+  https://www.erp5.com/
+  https://www.erp5.org/"""
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-certificate-expiration-days': '10',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'certificate-expiration-days': '20'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'certificate-expiration-days': '10'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-promise.py',
+      "'json-file': '%s'" % (self.surykatka_json,)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-20',
+      {'url': 'https://www.erp5.org/',
+       'check-certificate-expiration-days': '20'},
+    )

software/nayuos/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [template-instance]
 filename = instance.cfg
-md5sum = ae26da39d6a156a6164782086d6aa00d
+md5sum = aa91245be4b946d0537f6e007f0d72ea

software/nayuos/instance.cfg

@@ -106,7 +106,7 @@ context =
 recipe = slapos.recipe.template:jinja2
 template = {{ software_dir }}/scripts/cros_full_build.in
 rendered = ${directory:run}/cros_full_build
-md5sum = 4086de2b8910a52560b7bdc53ed3af22
+md5sum = f3b3b5408b69f752f2221d3965caf590
 mode = 0700
 context =
   key bash_path                         bin:bash
@@ -119,6 +119,7 @@ context =
   key ebuilds_dir                       nayuos-ebuilds:location
   raw scripts_dir                       {{ software_dir }}/scripts
   raw logo_dir                          {{ software_dir }}/logo
+  raw wallpaper_dir                     {{ software_dir }}/wallpaper
   raw patch_dir                         {{ software_dir }}/patch
 # packages to be copied from our overlay into ChromiuOS' overlay
 # those packages will be marked as dependency of root filesystem

software/nayuos/scripts/cros_full_build.in

@@ -317,6 +317,7 @@ for board in ${BOARDS} ; do
   cros_sdk -- cros_workon_make --board=${board} chromiumos-assets --install
   cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_100_percent/
   cp {{ logo_dir }}/* {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/images_200_percent/
+  cp -r {{ wallpaper_dir }} {{ cros_location }}/{{ branch }}/src/platform/chromiumos-assets/
 
   NAYU_IMAGE_LOCATION=/tmp/${board}.nayuos.img
 

software/neoppod/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = 0a3a54fcc7be0bbd63cbd64f006ceebc
+md5sum = 80599fcc6e5d07270d7900aebfd62139
 
 [root-common]
 filename = root-common.cfg.in
-md5sum = ccc6e33412259415ec6c3452d37b77cc
+md5sum = c03fbfc9df9edc1ef60be970e0627c5e
 
 [instance-neo-admin]
 filename = instance-neo-admin.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = d18e049d580720e733502b774b0d6790
+md5sum = 574acb0cae9af8ec2af52825fb2436d8
 
 [template-neo-my-cnf]
 filename = my.cnf.in
@@ -38,7 +38,7 @@ md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f
 
 [template-neo]
 filename = instance.cfg.in
-md5sum = 83dc9faca482b2ddbd3d5fa968af7c33
+md5sum = 9e63e16eda75e73ad4ffb50afde0505d
 
 [cluster]
 filename = cluster.cfg.in

software/neoppod/instance-common.cfg.in

@@ -15,7 +15,9 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 filename = ${:_buildout_section_name_}.cfg
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-extensions = jinja2.ext.do
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
 extra-context =
 context =
     key ipv4_set slap-configuration:ipv4

software/neoppod/instance-neo-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate a NEO cluster. See https://lab.nexedi.com/nexedi/neoppod/blob/master/neo.conf for more information.",
-  "additionalProperties": false,
-  "require": [
-    "cluster"
-  ],
-  "properties": {
-    "cluster": {
-      "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
-      "type": "string"
-    },
-    "partitions": {
-      "description": "Number of partitions. You cannot change this value once you created a cluster.",
-      "default": 12,
-      "type": "integer"
-    },
-    "replicas": {
-      "description": "Number of replicates.",
-      "default": 0,
-      "type": "integer"
-    },
-    "upstream-cluster": {
-      "description": "Identifier of the cluster to backup.",
-      "type": "string"
-    },
-    "upstream-masters": {
-      "description": "Master nodes in the cluster to backup.",
-      "type": "string"
-    },
-    "monitor": {
-      "description": "Parameters for monitoring.",
+  "definitions": {
+    "neo-cluster": {
+      "additionalProperties": false,
       "properties": {
-        "backup": {
-          "description": "List of cluster names that are expected to backup this cluster.",
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "uniqueItems": true
+        "cluster": {
+          "description": "Cluster unique identifier. Your last line of defense against mixing up NEO clusters and corrupting your data. Choose a unique value for each of your cluster. Space not allowed.",
+          "type": "string"
         },
-        "periodicity": {
-          "description": "Periodicity in minutes at which the cluster health is checked.",
-          "default": 10,
-          "type": "number"
-        }
-      },
-      "type": "object"
-    },
-    "sla-dict": {
-      "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
-      "additionalProperties": {
-        "type": "array",
-        "items": {
+        "partitions": {
+          "description": "Number of partitions. You cannot change this value once you created a cluster.",
+          "default": 12,
+          "type": "integer"
+        },
+        "replicas": {
+          "description": "Number of replicates.",
+          "default": 0,
+          "type": "integer"
+        },
+        "upstream-cluster": {
+          "description": "Identifier of the cluster to backup.",
           "type": "string"
         },
-        "uniqueItems": true
-      },
-      "type": "object"
-    },
-    "ssl": {
-      "description": "Enable SSL. All nodes look for 3 files in ~/etc: ca.crt, neo.crt, neo.key. Waiting that SlapOS provides a way to manage certificates, the user must deploy them manually, or use the temporary _ca/_cert/_key parameters.",
-      "default": true,
-      "type": "boolean"
-    },
-    "_ca": {
-      "type": "string"
-    },
-    "_cert": {
-      "type": "string"
-    },
-    "_key": {
-      "type": "string"
-    },
-    "node-list": {
-      "description": "List of dictionaries containing parameters for each node.",
-      "items": {
-        "description": "Dictionary containing parameters required to configure individual nodes.",
-        "default": {},
-        "additionalProperties": false,
-        "properties": {
-          "admin": {
-            "description": "Port of admin node. 0 to disable.",
-            "default": 2050,
-            "type": "integer"
-          },
-          "master": {
-            "description": "Port of master node. 0 to disable.",
-            "default": 2051,
-            "type": "integer"
-          },
-          "storage-count": {
-            "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
-            "default": 1,
-            "type": "integer"
-          },
-          "data-deduplication": {
-            "description": "Set the --dedup option for storage nodes.",
-            "default": false,
-            "type": "boolean"
-          },
-          "disable-drop-partitions": {
-            "description": "Set the --disable-drop-partitions option for storage nodes.",
-            "default": false,
-            "type": "boolean"
+        "upstream-masters": {
+          "description": "Master nodes in the cluster to backup.",
+          "type": "string"
+        },
+        "monitor": {
+          "description": "Parameters for monitoring.",
+          "properties": {
+            "backup": {
+              "description": "List of cluster names that are expected to backup this cluster.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "uniqueItems": true
+            },
+            "periodicity": {
+              "description": "Periodicity in minutes at which the cluster health is checked.",
+              "default": 10,
+              "type": "number"
+            }
           },
-          "storage-type": {
-            "description": "Storage type. Defaults to MySQL if available, else SQLite.",
-            "enum": [
-              "MySQL",
-              "SQLite"
-            ],
-            "type": "string"
+          "type": "object"
+        },
+        "sla-dict": {
+          "description": "[NEO SR only] Where to request instances. Each key is a query string for criterions (e.g. \"computer_guid=foo\"), and each value is a list of partition references ('node-0', 'node-1', ...). The prefix 'node-' is mandatory and the number must start from 0. The total number of nodes here must be equal to the length of node-list.",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "uniqueItems": true
           },
-          "mysql": {
-            "description": "Dictionary containing parameters for MySQL.",
-            "default": {},
-            "properties": {
-              "relaxed-writes": {
-                "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
-                "default": false,
-                "type": "boolean"
-              }
+          "type": "object"
+        },
+        "ssl": {
+          "description": "Enable SSL. All nodes look for 3 files in ~/etc: ca.crt, neo.crt, neo.key. Waiting that SlapOS provides a way to manage certificates, the user must deploy them manually, or use the temporary _ca/_cert/_key parameters.",
+          "default": true,
+          "type": "boolean"
+        },
+        "_ca": {
+          "type": "string"
+        },
+        "_cert": {
+          "type": "string"
+        },
+        "_key": {
+          "type": "string"
+        },
+        "node-count": {
+          "description": "Number of nodes to deploy.",
+          "default": 1,
+          "type": "integer"
+        },
+        "node": {
+          "description": "Default node parameters.",
+          "default": {},
+          "additionalProperties": false,
+          "properties": {
+            "admin": {
+              "description": "Port of admin node. 0 to disable.",
+              "default": 2050,
+              "type": "integer"
             },
-            "additionalProperties": {
-              "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+            "master": {
+              "description": "Port of master node. 0 to disable.",
+              "default": 2051,
+              "type": "integer"
+            },
+            "storage-count": {
+              "description": "Number of storage nodes to deploy. One master and one admin node is deployed with each storage.",
+              "default": 1,
+              "type": "integer"
+            },
+            "data-deduplication": {
+              "description": "Set the --dedup option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "disable-drop-partitions": {
+              "description": "Set the --disable-drop-partitions option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+            },
+            "storage-type": {
+              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
+              "enum": [
+                "MySQL",
+                "SQLite"
+              ],
               "type": "string"
             },
-            "type": "object"
+            "mysql": {
+              "description": "Dictionary containing parameters for MySQL.",
+              "default": {},
+              "properties": {
+                "relaxed-writes": {
+                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
+                  "default": false,
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": {
+                "description": "To configure important parameters like innodb_buffer_pool_size, tokudb_cache_size, etc.",
+                "type": "string"
+              },
+              "type": "object"
+            },
+            "engine": {
+              "description": "Configures storage engine, currently only InnoDB, RocksDB and TokuDB are supported. Defaults to NEO's default.",
+              "type": "string"
+            }
           },
-          "engine": {
-            "description": "Configures storage engine, currently only InnoDB and TokuDB are supported. Defaults to NEO's default.",
-            "type": "string"
-          }
+          "type": "object"
         },
-        "type": "object"
+        "node!": {
+          "description": "Node parameters are overridden by the value of the first item whose partition reference matches against the pattern.",
+          "items": {
+            "items": [
+              {
+                "description": "Override pattern (Python regular expression).",
+                "type": "string"
+              },
+              {
+                "allOf": [
+                  {
+                    "$ref": "#/definitions/neo-cluster/properties/node"
+                  },
+                  {
+                    "description": "Override value (parameters for maching nodes)."
+                  }
+                ]
+              }
+            ],
+            "type": "array"
+          },
+          "type": "array"
+        }
       },
-      "type": "array"
+      "type": "object"
     }
   },
-  "type": "object"
+  "allOf": [
+    {
+      "$ref": "#/definitions/neo-cluster"
+    },
+    {
+      "required": [
+        "cluster"
+      ]
+    }
+  ]
 }

software/neoppod/instance-neo.cfg.in

@@ -90,7 +90,9 @@ cluster = {{ dumps(slapparameter_dict['cluster']) }}
 masters = {{ dumps(slapparameter_dict['masters']) }}
 database-adapter = {{ storage_type }}
 wait-database = -1
+{%- if mysql %}
 engine = ${my-cnf-parameters:engine}
+{%- endif %}
 dedup = {{ dumps(bool(slapparameter_dict.get('data-deduplication'))) }}
 disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
 

software/neoppod/instance.cfg.in

@@ -8,6 +8,7 @@ rendered = ${buildout:parts-directory}/${:_buildout_section_name_}.cfg
 <= jinja2-template-base
 template = {{ cluster }}
 extra-context =
+    import re re
     import urlparse urlparse
 import-list =
     rawfile root_common {{ root_common }}

software/neoppod/root-common.cfg.in

@@ -16,6 +16,20 @@ sla-computer_guid = ${slap-connection:computer-id}
 {% endif -%}
 {% endmacro -%}
 
+{% macro apply_overrides(dict_, reference) -%}
+{% for key in list(dict_) -%}
+{%   if key.endswith('!') -%}
+{%     for pattern, value in dict_.pop(key, ()) -%}
+{%       set m = re.match(pattern, reference) -%}
+{%       if m and m.group() == reference %}{# PY3: fullmatch -#}
+{%         do dict_.__setitem__(key[:-1], value) -%}
+{%         break -%}
+{%       endif -%}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+{% endmacro -%}
+
 {% macro common_section() -%}
 [request-common-base]
 recipe = slapos.cookbook:request.serialised
@@ -52,7 +66,22 @@ config-ssl = {{ dumps((
 config-upstream-cluster = {{ dumps(parameter_dict.get('upstream-cluster', '')) }}
 config-upstream-masters = {{ dumps(parameter_dict.get('upstream-masters', '')) }}
 software-type = {{ software_type }}
-{% set node_list = parameter_dict.get('node-list', ({},)) -%}
+
+{% set node_list = parameter_dict.get('node-list') -%}
+{% if node_list == None -%}
+{%   set node_list = [] -%}
+{%   for i in range(parameter_dict.get('node-count', 1)) -%}
+{%     set x = parameter_dict.copy() -%}
+{%     do apply_overrides(x, prefix ~ i) -%}
+{%     do node_list.append(x.get('node', {})) -%}
+{%   endfor -%}
+{% else %}{# BBB -#}
+{%   do assert('node-count' not in parameter_dict) -%}
+{%   do assert('node' not in parameter_dict) -%}
+{%   do assert('node!' not in parameter_dict) -%}
+{% endif -%}
+{% do assert(node_list) -%}
+
 {% set storage_count = [] -%}
 {% for node in node_list -%}
 {%   do storage_count.append(node.get('storage-count', 1)) -%}

software/nextcloud/buildout.hash.cfg

@@ -26,4 +26,4 @@ md5sum = fbc5eacda192cb02c8d9861cb628fe6f
 
 [template-nextcloud-instance]
 filename = nextcloud-instance.cfg.in
-md5sum = 426df55e81725cd7faa408d3cd93558e
+md5sum = 28f9a64176e86360a3398742d5ad8470

software/nextcloud/nextcloud-instance.cfg.in

@@ -96,6 +96,23 @@ depends =
   ${nextcloud-backup-cron:recipe}
   ${nextcloud-news-updater:recipe}
   ${nextcloud-config-update:recipe}
+  ${nextcloud-install-promise:name}
+
+[nextcloud-install-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = check-nextcloud-install.py
+config-command = ${nc-install-wrapper:output}
+
+[nc-install-wrapper]
+recipe = collective.recipe.template
+input = inline:#!/bin/bash
+  if [ -f "${instance-parameter:nextcloud}/config/CAN_INSTALL" ]; then
+    echo "Nextcloud is not installed.";
+    exit 1;
+  fi
+output = ${directory:bin}/check-nc-install
+mode = 744
 
 [nextcloud-apache-httpd]
 recipe = slapos.recipe.template:jinja2

software/re6stnet/software.cfg

@@ -88,7 +88,7 @@ context =
 < = download-base
 
 [versions]
-re6stnet = 0.533
+re6stnet = 0.551
 slapos.recipe.template = 4.3
 
 # Required by:

software/seleniumserver/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = c4ac5de141ae6a64848309af03e51d88
 
 [template-selenium]
 filename = instance-selenium.cfg.in
-md5sum = 4f557a7b3aa9b4df1ca1fa6a754ca657
+md5sum = 1f0b67d2a542e94380c35afc9cd1946b

software/seleniumserver/instance-selenium.cfg.in

@@ -184,18 +184,23 @@ extra-args=-t dsa
 <=ssh-keygen-base
 extra-args=-t ecdsa -b 521
 
-[ssh-key-fingerprint-command]
-recipe = plone.recipe.command
-# recent openssh client display ECDSA key's fingerprint as SHA256
-command = ${openssh-output:keygen} -lf $${ssh-host-ecdsa-key:output}
-
-[ssh-key-fingerprint]
+[ssh-key-fingerprint-shelloutput]
 recipe = collective.recipe.shelloutput
-# XXX because collective.recipe.shelloutput ignore errors, we run the same
-# command in a plone.recipe.command so that if fails if something goes wrong.
+# recent openssh client display ECDSA key's fingerprint as SHA256
 commands =
-  fingerprint = $${ssh-key-fingerprint-command:command}
+  fingerprint = ${openssh-output:keygen} -lf $${ssh-host-ecdsa-key:output}
 
+[ssh-key-fingerprint]
+recipe = plone.recipe.command
+stop-on-error = true
+# XXX because collective.recipe.shelloutput ignore errors and capture output
+# "Error ...", we use a plone.recipe.command to check that this command did
+# not fail.
+# This command will always fail on first buildout run, because
+# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step,
+# but the key file is created later at install step.
+command = echo "$${:fingerprint}" | ( grep ^Error || exit 0 && exit 1 )
+fingerprint = $${ssh-key-fingerprint-shelloutput:fingerprint}
 
 [sshd-config]
 recipe = slapos.recipe.template:jinja2

software/seleniumserver/software.cfg

@@ -46,5 +46,4 @@ output = ${buildout:directory}/template-selenium.cfg
 
 [versions]
 plone.recipe.command = 1.1
-collective.recipe.shelloutput = 0.1
 slapos.recipe.template = 4.3

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 3d12778d8dc910180ff3a7fcae8b37f9
+md5sum = 4664f7dae66d3f582e34cec2ca627501

software/slapos-sr-testing/instance.cfg

@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp
 
 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

software/slapos-sr-testing/software.cfg

@@ -102,6 +102,11 @@ setup = ${slapos-repository:location}/software/nextcloud/test/
 egg = slapos.test.nextcloud
 setup = ${slapos-repository:location}/software/turnserver/test/
 
+[slapos.test.theia-setup]
+<= setup-develop-egg
+egg = slapos.test.theia
+setup = ${slapos-repository:location}/software/theia/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 8b78e32b877d591400746ec7fd68ed4c
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 85ea0b78fd18428c242438ebe95f980b
+md5sum = 1216494c03752f0a3c1755e190eed3dc
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -26,7 +26,7 @@ md5sum = fc22e2d2f03ce58631f157a5b4943e15
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = b450c474464a326f3d0b98728460ac97
+md5sum = 918fb2984cb2ed7afba9200167f98a0f
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
@@ -50,7 +50,7 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = da113b3e3e7bac9cc215fede7c4911a5
+md5sum = e6a3ca1604ae5458248135cd6de0f3e6
 
 [template-parameters]
 filename = parameters.xml.in
@@ -82,4 +82,4 @@ md5sum = 75aab99c995ca841f93fc77fc9116c37
 
 [template-buildout-shared-part-list]
 filename = template/buildout-shared-part-list.in
-md5sum = 3203c9ad0b30d3ee39a809a067efff8d
\ No newline at end of file
+md5sum = 3203c9ad0b30d3ee39a809a067efff8d

software/slaprunner/instance-runner-import.cfg.in

@@ -14,12 +14,8 @@ parts +=
   slaprunner-promise
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
-  runner-sshd-graceful
   runner-sshd-promise
-  runner-sshkeys-authority
-  runner-sshkeys-authority-service
-  runner-sshkeys-sshd
-  runner-sshkeys-sshd-service
+  runner-sshd-service
   runtestsuite
   shellinabox
   shellinabox-service

software/slaprunner/instance-runner.cfg

@@ -15,12 +15,8 @@ common-runner-parts =
   apache-httpd-promise
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
-  runner-sshd-graceful
   runner-sshd-promise
-  runner-sshkeys-authority
-  runner-sshkeys-authority-service
-  runner-sshkeys-sshd
-  runner-sshkeys-sshd-service
+  runner-sshd-service
   runtestsuite
   symlinks
   shellinabox
@@ -115,7 +111,7 @@ name = custom_frontend_ready_promise.py
 config-command = $${custom-frontend-url-ready-promise-bin:rendered}
 
 [publish-connection-information]
-custom-frontend-url = $${custom-frontend-url-ready-promise:url}
+custom-frontend-url = $${custom-frontend-url-ready-promise-bin:url}
 {% endif %}
 
 # Create all needed directories
@@ -177,9 +173,7 @@ shared_root = $${runnerdirectory:shared-root}
 buildout-shared-part-list-dump = ${template-buildout-shared-part-list:output}
 pidfile-software = $${directory:run}/slapgrid-cp.pid
 pidfile-instance = $${directory:run}/slapgrid-sr.pid
-ssh_client = ${openssh:location}/bin/ssh
-public_key = $${runner-sshd-raw-server:rsa-keyfile}.pub
-private_key = $${runner-sshd-raw-server:rsa-keyfile}
+public_key = $${runner-sshd-ssh-host-rsa-key:output}
 instance-monitor-url = https://[$${:ipv6}]:$${slap-parameter:monitor-httpd-port}
 etc_dir = $${directory:etc}
 log_dir =  $${directory:log}
@@ -256,106 +250,72 @@ ip = $${slap-network-information:global-ipv6}
 recipe = slapos.recipe.template:jinja2
 rendered = $${directory:etc}/runner-sshd.conf
 path_pid = $${directory:run}/runner-sshd.pid
-host_key = $${directory:ssh}/runner_server_key.rsa
 template = inline:
   PidFile $${:path_pid}
   Port $${runner-sshd-port:port}
   ListenAddress $${slap-network-information:global-ipv6}
   Protocol 2
   UsePrivilegeSeparation no
-  HostKey $${:host_key}
+  HostKey $${runner-sshd-ssh-host-rsa-key:output}
+  HostKey $${runner-sshd-ssh-host-ecdsa-key:output}
   PasswordAuthentication no
   PubkeyAuthentication yes
   AuthorizedKeysFile $${buildout:directory}/.ssh/authorized_keys
   ForceCommand cd $${directory:home}; if [ -z "$SSH_ORIGINAL_COMMAND" ]; then HOME=$${directory:home} $${shell-environment:shell} -l; else HOME=$${directory:home} SHELL=$${shell-environment:shell} PATH=$${shell-environment:path} eval "$SSH_ORIGINAL_COMMAND"; fi
   Subsystem sftp ${openssh:location}/libexec/sftp-server
 
-[runner-sshd-raw-server]
+[runner-sshd-service]
 recipe = slapos.cookbook:wrapper
-host = $${slap-network-information:global-ipv6}
-rsa-keyfile = $${runner-sshd-config:host_key}
-home = $${directory:ssh}
 command-line = ${openssh:location}/sbin/sshd -D -e -f $${runner-sshd-config:rendered}
-wrapper-path = $${directory:bin}/runner_raw_sshd
-
-[runner-sshd-authorized-key]
-<= runner-sshd-raw-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${slap-parameter:user-authorized-key}
-
-[runner-sshd-server]
-recipe = collective.recipe.template
-log = $${directory:log}/runner-sshd.log
-input = inline:#!/bin/sh
-    exec $${runner-sshd-raw-server:wrapper-path} >> $${:log} 2>&1
-
-output = $${directory:bin}/runner_raw_sshd_log
-mode = 700
-
-[runner-sshd-graceful]
-recipe = slapos.cookbook:wrapper
-command-line = $${directory:bin}/killpidfromfile $${runner-sshd-config:path_pid} SIGHUP
-wrapper-path = $${directory:scripts}/runner-sshd-graceful
-
-[runner-sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/runner-requests/
-keys = $${directory:sshkeys}/runner-keys/
-
-[runner-sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${runner-sshkeys-directory:requests}
-keys-directory = $${runner-sshkeys-directory:keys}
-wrapper = $${directory:bin}/runner_sshkeys_authority
-keygen-binary = ${openssh:location}/bin/ssh-keygen
-
-[runner-sshkeys-authority-service]
-recipe = slapos.cookbook:wrapper
-command-line = $${runner-sshkeys-authority:wrapper}
-wrapper-path = $${directory:services}/runner-sshkeys-authority
 hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
-
-[runner-sshkeys-sshd]
-<= runner-sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = sshd
-type = rsa
-executable = $${runner-sshd-server:output}
-public-key = $${runner-sshd-raw-server:rsa-keyfile}.pub
-private-key = $${runner-sshd-raw-server:rsa-keyfile}
-wrapper = $${directory:bin}/runner-sshd
-
-[runner-sshkeys-sshd-service]
-recipe = slapos.cookbook:wrapper
-command-line = $${runner-sshkeys-sshd:wrapper}
 wrapper-path = $${directory:services}/runner-sshd
-hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [runner-sshd-add-authorized-key]
 recipe = slapos.cookbook:dropbear.add_authorized_key
 home = $${buildout:directory}
 key = $${slap-parameter:user-authorized-key}
 
-[runner-sshkeys-publickey-fingerprint-cmd]
+[runner-sshd-ssh-keygen-base]
 recipe = plone.recipe.command
-command = bash -o pipefail -c "$${runner-sshkeys-authority:keygen-binary} -lf $${runner-sshkeys-sshd:public-key} | cut  -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
+output = $${directory:etc}/$${:_buildout_section_name_}
+command = ${openssh-output:keygen} -f $${:output} -N '' $${:extra-args}
 
-[runner-sshkeys-publickey-fingerprint-shelloutput]
+[runner-sshd-ssh-host-rsa-key]
+<=runner-sshd-ssh-keygen-base
+extra-args=-t rsa
+[runner-sshd-ssh-host-ecdsa-key]
+<=runner-sshd-ssh-keygen-base
+extra-args=-t ecdsa -b 521
+
+[runner-sshd-publickey-fingerprint-shelloutput]
 recipe = collective.recipe.shelloutput
 # XXX because collective.recipe.shelloutput ignore errors, we run the same
 # command in a plone.recipe.command so that if fails if something goes wrong.
 commands =
-  fingerprint = $${runner-sshkeys-publickey-fingerprint-cmd:command}
+  fingerprint = bash -o pipefail -c "${openssh-output:keygen} -lf $${runner-sshd-ssh-host-ecdsa-key:output} | cut  -f 2 -d\ | sed 's/+/%2B/g' | sed 's/\//%2F/g' | sed 's/SHA256://'"
 
-[runner-sshkeys-publickey-fingerprint]
+[runner-sshd-publickey-fingerprint]
 # fingerprint for ssh url, see
 # https://tools.ietf.org/id/draft-salowey-secsh-uri-00.html#connparam
 # https://winscp.net/eng/docs/session_url#hostkey
 
+_fingerprint = $${runner-sshd-publickey-fingerprint-shelloutput:fingerprint}
+
 # format is host-key-alg-fingerprint, but we know that
 # $${runner-sshkeys-sshd:public-key} is rsa so for host-key-alg
 # we just use use rsa.
-fingerprint = ssh-rsa-$${runner-sshkeys-publickey-fingerprint-shelloutput:fingerprint}
+fingerprint = ssh-rsa-$${:_fingerprint}
+
+# XXX because collective.recipe.shelloutput ignore errors and capture output
+# "Error ...", we use a plone.recipe.command to check that this command did
+# not fail.
+# This command will always fail on first buildout run, because
+# collective.recipe.shelloutput is evaluated at buildout recipes __init__ step,
+# but the key file is created later at install step.
+recipe = plone.recipe.command
+stop-on-error = true
+command = echo "$${:_fingerprint}" | ( grep ^Error || exit 0 && exit 1 )
+
 
 #---------------------------
 #--
@@ -640,7 +600,7 @@ backend-url = $${slaprunner:access-url}
 init-user = $${runner-htpasswd:user}
 init-password = $${runner-htpasswd:password}
 ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
-ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshkeys-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
+ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshd-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
 git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
 git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
 monitor-base-url = $${monitor-publish-parameters:monitor-base-url}

software/slaprunner/software.cfg

@@ -171,7 +171,6 @@ gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
 slapos.recipe.template = 4.3
-collective.recipe.shelloutput = 0.1
 collective.recipe.environment = 0.2.0
 smmap = 0.9.0
 lockfile = 0.12.2

software/slaprunner/template/slapos.cfg.in

@@ -58,10 +58,8 @@ host = {{ slaprunner['ipv4'] }}
 port = {{ slaprunner['proxy_port'] }}
 database_uri = {{ slaprunner['proxy_database'] }}
 
-[sshkeys_authority]
-ssh_client = {{ slaprunner['ssh_client'] }}
-public_key = {{ slaprunner['public_key'] }}
-private_key = {{ slaprunner['private_key'] }}
-
 [gitclient]
 git = {{ slaprunner['git-binary'] }}
+
+[sshkeys_authority]
+public_key = {{ slaprunner['public_key'] }}

software/slaprunner/test/test.py

@@ -246,7 +246,6 @@ class ServicesTestCase(SlaprunnerTestCase):
     ]
     expected_process_names = [
       'slaprunner-supervisord-{hash}-on-watch',
-      'runner-sshkeys-authority-{hash}-on-watch',
       'runner-sshd-{hash}-on-watch',
       'slaprunner-httpd-{hash}-on-watch',
       'gunicorn-{hash}-on-watch',
@@ -297,3 +296,19 @@ class TestInstanceResilient(SlaprunnerTestCase):
         'takeover-runner-1-url',
         'url',
         'webdav-url']))
+
+
+class TestCustomFrontend(SlaprunnerTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'custom-frontend-backend-url': 'https://www.erp5.com',
+      'custom-frontend-backend-type': 'redirect',
+    }
+
+  def test(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    # slapproxy returns the backend URL when requesting a slave frontend
+    self.assertEqual(
+      parameter_dict['custom-frontend-url'],
+      'https://www.erp5.com')

software/theia/README.md

+# Theia software release
+
+Theia is a cloud (and desktop) IDE https://www.theia-ide.org
+
+This version comes pre-configured with a few plugins, but does not come with python plugin, to let
+you choose between theia and vscode one.
+
+## Installing vscode extensions
+
+To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
+
+<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
+Check theia's log files in case of issues.
+
+see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
+
+## jedi
+
+[jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has
+some support for `zc.buildout`. It looks up for a `buildout.cfg` file and if found will load all scripts
+from the bin directory from this buildout to add eggs to sys.path. In webrunner we have almost 100 scripts
+in bin directory, with maybe 30 eggs in each scripts, so this makes jedi so slow it's unusable. Also, if
+an error occurs parsing these scripts, jedi won't be usable. This issue is tracked in
+https://github.com/davidhalter/jedi/issues/1325
+
+A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
\ No newline at end of file

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 6a0a65ff9f81f907d53c42c65b54442a
+md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = fc2f20d496911241a0d9001c445d6158
+md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 242c8359be6893725ad2ac8e3c864eb0
+md5sum = ce87c9818d64a16fce49c84ed5dba8fc

software/theia/instance.cfg.in

@@ -2,14 +2,13 @@
 parts =
   monitor-base
   promises
-  .bashrc
   frontend-reload
   publish-connection-parameter
 
 extends = ${monitor-template:rendered}
 
 eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory} 
+develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
@@ -79,37 +78,15 @@ recipe = slapos.cookbook:userinfo
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  env -i HOME=$${directory:home} LC_ALL=C.UTF-8 USER=$${user:pw-name} LOGNAME=$${user:pw-name} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
+  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
 
 ip =  $${instance-parameter:ipv4-random}
 hostname =  $${:ip}
 port = 3000
 base-url = http://$${:hostname}:$${:port}/
-
-[.bashrc]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:home}/$${:_buildout_section_name_}
-template = inline:
-  export PS1="$ " # because we are in a gowork workspace
-  # XXX .bash_profile is not executed, so we introduce a bashrc.theia file
-  # to allow customizations.
-  if [ -f "$HOME/.bashrc.theia" ] ; then
-    source $HOME/.bashrc.theia
-  fi
-  # If not running interactively, don't do anything
-  case $- in
-      *i*) ;;
-        *) return;;
-  esac
-  # enable color support
-  if [ -x /usr/bin/dircolors ]; then
-      test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
-      alias ls='ls --color=auto'
-  fi
-  # common ls aliases
-  alias ll='ls -l'
-  alias la='ls -A'
-  alias l='ls -CF'
+hash-existing-files =
+  ${yarn.lock:output}
+  ${theia-wrapper:rendered}
 
 [promises]
 recipe =
@@ -156,4 +133,4 @@ pidfiles = $${:var}/run
 services = $${:etc}/service
 framebuffer = $${:srv}/framebuffer
 fonts = $${:srv}/fonts
-home = $${:srv}/home
\ No newline at end of file
+home = $${:srv}/home

software/theia/python-language-server-requirements.txt

@@ -20,4 +20,4 @@ snowballstemmer==1.2.1
 toml==0.10.0
 typed-ast==1.2.0
 typeshed==0.0.1
-yapf==0.25.0
+yapf==0.28.0

software/theia/software.cfg

@@ -24,7 +24,7 @@ parts =
 
 
 [nodejs]
-<= nodejs-8.9.4
+<= nodejs-10.6.0
 
 [python3]
 <= python3.6.6
@@ -34,10 +34,10 @@ parts =
 # this could become a component, but it needs to be invoked from nodejs explicitly,
 # otherwise it uses system's nodejs
 # XXX why don't we build a wrapper ?
-version = 1.11.0
+version = 1.17.3
 recipe = slapos.recipe.build:download-unpacked
 url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
-md5sum = d4f05075f534dd9a0a8c18c650b55f0d
+md5sum = 4a02e1687a150113ad6b0215f9afdb3e
 
 
 [python-language-server]
@@ -60,8 +60,9 @@ mode = 0644
 [theia]
 recipe = plone.recipe.command
 command = ${bash:location}/bin/bash -c "
-  export PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
+  export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin/:${python2.7:location}/bin/:$PATH &&
   mkdir -p ${:location} && \
+  mkdir -p \$TMPDIR && \
   cd ${:location} && \
   cp ${package.json:rendered} . &&
   cp ${yarn.lock:output} . &&
@@ -69,6 +70,7 @@ command = ${bash:location}/bin/bash -c "
   ${yarn:location}/bin/yarn theia build"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
+uses = ${yarn.lock:recipe}
 
 [yarn.lock]
 recipe = slapos.recipe.template
@@ -78,59 +80,91 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/598d4dc9d4f9e0514869273c069f734a41f20207/theia-full-docker/next.package.json
+# this comes from https://github.com/theia-ide/theia-apps/blob/def6d29a5fb3739aebc17c6a5b67f1b5c16caacd/theia-full-docker/next.package.json
 # but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
 template =
   inline:
-  {
-    "private": true,
-    "dependencies": {
-      "typescript": "latest",
-      "@theia/callhierarchy": "next",
-      "@theia/core": "next",
-      "@theia/cpp": "next",
-      "@theia/docker": "next",
-      "@theia/editor": "next",
-      "@theia/editorconfig": "next",
-      "@theia/extension-manager": "next",
-      "@theia/file-search": "next",
-      "@theia/filesystem": "next",
-      "@theia/git": "next",
-      "@theia/go": "next",
-      "@theia/java": "next",
-      "@theia/json": "next",
-      "@theia/keymaps": "next",
-      "@theia/languages": "next",
-      "@theia/markers": "next",
-      "@theia/merge-conflicts": "next",
-      "@theia/messages": "next",
-      "@theia/metrics": "next",
-      "@theia/mini-browser": "next",
-      "@theia/monaco": "next",
-      "@theia/navigator": "next",
-      "@theia/outline-view": "next",
-      "@theia/output": "next",
-      "@theia/plantuml": "next",
-      "@theia/preferences": "next",
-      "@theia/preview": "next",
-      "@theia/process": "next",
-      "@theia/python": "next",
-      "@theia/ruby": "next",
-      "@theia/rust": "next",
-      "@theia/search-in-workspace": "next",
-      "@theia/task": "next",
-      "@theia/terminal": "next",
-      "@theia/textmate-grammars": "next",
-      "@theia/tslint": "next",
-      "@theia/typescript": "next",
-      "@theia/userstorage": "next",
-      "@theia/variable-resolver": "next",
-      "@theia/workspace": "next",
-      "theia-yang-extension": "next"
-    },
-    "devDependencies": {
-      "@theia/cli": "next"
-    }
+    {
+      "private": true,
+      "theia": {
+          "frontend": {
+              "config": {
+                  "applicationName": "Theia SlapOS",
+                  "preferences": {
+                      "files.enableTrash": false,
+                      "files.exclude": {
+                          "**.pyc": true,
+                          "**.egg-info": true,
+                          "__pycache__": true,
+                          ".git": true,
+                          ".env": true
+                      },
+                      "files.watcherExclude": {
+                        "**/.eggs/**": true,
+                        "**/.env/**": true,
+                        "**/.git/**": true,
+                        "**/node_modules/**": true
+                      },
+                      "editor.multiCursorModifier": "ctrlCmd",
+                      "plantuml.webservice": "//plantuml.host.vifib.net/svg/",
+                      "plantuml.monochrome": false,
+                      "editor.tabSize": 2
+                  }
+              }
+          }
+      },
+      "dependencies": {
+          "@theia/callhierarchy": "next",
+          "@theia/core": "next",
+          "@theia/cpp": "next",
+          "@theia/debug": "next",
+          "@theia/debug-nodejs": "next",
+          "@theia/docker": "next",
+          "@theia/editor": "next",
+          "@theia/editorconfig": "next",
+          "@theia/editor-preview": "next",
+          "@theia/file-search": "next",
+          "@theia/filesystem": "next",
+          "@theia/git": "next",
+          "@theia/go": "next",
+          "@theia/getting-started": "next",
+          "@theia/java": "next",
+          "@theia/json": "next",
+          "@theia/keymaps": "next",
+          "@theia/languages": "next",
+          "@theia/markers": "next",
+          "@theia/merge-conflicts": "next",
+          "@theia/messages": "next",
+          "@theia/metrics": "next",
+          "@theia/mini-browser": "next",
+          "@theia/monaco": "next",
+          "@theia/navigator": "next",
+          "@theia/outline-view": "next",
+          "@theia/output": "next",
+          "@theia/plantuml": "next",
+          "@theia/plugin": "next",
+          "@theia/plugin-ext": "next",
+          "@theia/plugin-ext-vscode": "next",
+          "@theia/preferences": "next",
+          "@theia/preview": "next",
+          "@theia/process": "next",
+          "@theia/ruby": "next",
+          "@theia/rust": "next",
+          "@theia/search-in-workspace": "next",
+          "@theia/task": "next",
+          "@theia/terminal": "next",
+          "@theia/textmate-grammars": "next",
+          "@theia/tslint": "next",
+          "@theia/typescript": "next",
+          "@theia/userstorage": "next",
+          "@theia/variable-resolver": "next",
+          "@theia/workspace": "next",
+          "theia-yang-extension": "next",
+          "typescript": "latest"
+      },
+      "devDependencies": {
+          "@theia/cli": "next"
+      }
   }
 rendered = ${buildout:directory}/${:_buildout_section_name_}
 mode = 0644
@@ -169,6 +203,8 @@ template =
   export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
   . ${gowork:env.sh}
   export SHELL=bash
+  # reset PS1 from gowork
+  export PS1='$ '
   cd ${theia:location}
   exec ${yarn:location}/bin/yarn theia start $@
 

software/theia/test/README.md

+Tests for Theia software release

software/theia/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.theia'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Theia",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/theia/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import textwrap
+import logging
+import tempfile
+import time
+from six.moves.urllib.parse import urlparse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestTheia(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.theia_url = self.computer_partition.getConnectionParameterDict(
+    )['url']
+
+  def test_http_get(self):
+    resp = requests.get(self.theia_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+    # without login/password, this is unauthorized
+    parsed_url = urlparse(self.theia_url)
+    resp = requests.get(
+        parsed_url._replace(
+            netloc='[{}]:{}'.format(
+                parsed_url.hostname,
+                parsed_url.port)).geturl(),
+        verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)

stack/erp5/buildout.cfg

@@ -61,6 +61,7 @@ extends =
   ../../component/postfix/buildout.cfg
   ../../component/zbarlight/buildout.cfg
   ../../component/perl/buildout.cfg
+  ../../component/pylint/buildout.cfg
   ../../stack/caucase/buildout.cfg
   ../../software/jupyter/software.cfg
   ../../software/neoppod/software-common.cfg
@@ -502,6 +503,7 @@ eggs = ${neoppod:eggs}
   feedparser
   validictory
   erp5.util
+  z3c.etestbrowser
   huBarcode
   qrcode
   spyne
@@ -516,7 +518,10 @@ eggs = ${neoppod:eggs}
   decorator
   networkx
 # Needed for checking ZODB Components source code
-  pylint
+  ${pylint:egg}
+  jedi
+  yapf
+  typing
   pytracemalloc
   xlrd
 
@@ -588,6 +593,7 @@ entry-points =
   runwsgi=Products.ERP5.bin.zopewsgi:runwsgi
 scripts =
   apachedex
+  performance_tester_erp5
   repozo
   runwsgi
   runzope
@@ -650,6 +656,8 @@ ocropy = 1.0+SlapOSPatched001
 pysvn = 1.7.10+SlapOSPatched002
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
+## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
+pylint = 1.4.4
 
 # use newer version than specified in ZTK
 PasteDeploy = 1.5.2
@@ -730,8 +738,6 @@ pycountry = 17.1.8
 pycrypto = 2.6.1
 pycurl = 7.43.0
 pyflakes = 1.5.0
-# pylint 1.5.1 breaks testDynamicClassGeneration
-pylint = 1.4.4
 python-memcached = 1.58
 pytracemalloc = 1.2
 qrcode = 5.3
@@ -828,3 +834,30 @@ responses = 0.10.6
 # Required by:
 # responses = 0.10.6
 cookies = 2.2.1
+
+jedi = 0.15.1
+parso = 0.5.1
+
+yapf = 0.28.0
+
+typing = 3.7.4.1
+
+# Required by:
+# erp5.util==0.4.65
+z3c.etestbrowser = 3.0.1
+zope.testbrowser = 5.5.1
+
+# Required by:
+# zope.testbrowser==5.5.1
+WSGIProxy2 = 0.4.6
+WebTest = 2.0.33
+beautifulsoup4 = 4.8.2
+
+# Required by:
+# WSGIProxy2==0.4.6
+WebOb = 1.8.5
+soupsieve = 1.9.5
+
+# Required by:
+# soupsieve==1.9.5
+backports.functools-lru-cache = 1.6.1

stack/erp5/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
 
 [template-zope-conf]
 filename = zope.conf.in
-md5sum = 114e0ac43281b943931754ed317ebc36
+md5sum = 762897486b1e7e28b614224a9a577125
 
 [site-zcml]
 filename = site.zcml
@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
 
 [template]
 filename = instance.cfg.in
-md5sum = f6c6820f9b3653d0d5c29708606fc591
+md5sum = 520b6bf3461dddc9c8b862e50b14465d
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -86,7 +86,7 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = b1685783f4c93da918ccc83702559e6f
+md5sum = 8b4a15dca7e30ba5a792f1a9622216b0
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -267,6 +267,11 @@ timerserver-interval = {{ dumps(timerserver_interval) }}
 [zope-conf-base]
 < = jinja2-template-base
 template = {{ parameter_dict['zope-conf-template'] }}
+extensions =
+    jinja2.ext.do
+    jinja2.ext.loopcontrols
+import-list =
+    rawfile root_common {{ root_common }}
 
 {% macro zope(
   index,
@@ -312,10 +317,10 @@ longrequest-logger-file =
 [{{ conf_name }}]
 < = zope-conf-base
 rendered = ${directory:etc}/{{ name }}.conf
-extensions = jinja2.ext.do
 context =
   section parameter_dict {{ conf_parameter_name }}
   import os os
+  import re re
 
 [{{ section(name) }}]
 < = runzope-base

stack/erp5/instance.cfg.in

@@ -46,6 +46,7 @@ extra-context =
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
+    import re re
     import urlparse urlparse
 import-list =
     file root_common context:root-common
@@ -111,6 +112,7 @@ template = {{ template_zope }}
 filename = instance-zope.cfg
 extra-context =
     key buildout_directory buildout:directory
+    key root_common context:root-common
     section parameter_dict dynamic-template-zope-parameters
     import urlparse urlparse
     import hashlib hashlib

stack/erp5/zope.conf.in

+{% set slapparameter_dict = {} %}{# dummy -#}
+{% import "root_common" as root_common with context -%}
+{% set node_id = parameter_dict['node-id'] -%}
 # Note: Environment is setup in running wrapper script, as zope.conf is read
 # too late for some components.
 %define INSTANCE {{ parameter_dict['instance'] }}
@@ -65,7 +68,7 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 
 {% endif -%}
 <product-config CMFActivity>
-  node-id {{ parameter_dict['node-id'] }}
+  node-id {{ node_id }}
 </product-config>
 
 {% set timerserver_interval = parameter_dict['timerserver-interval'] -%}
@@ -145,13 +148,15 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
 <zodb_db {{ name }}>
 {%- set storage_type = type_dict[zodb_dict.pop('type')] %}
 {%- set storage_dict = zodb_dict.pop('storage-dict') %}
+{%- do root_common.apply_overrides(zodb_dict, node_id) %}
 {%- for key, value in zodb_dict.iteritems() %}
   {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   <{{ storage_type }}>
-{%-   for key, value in storage_dict.iteritems() %}
+{%- do root_common.apply_overrides(storage_dict, node_id) %}
+{%- for key, value in storage_dict.iteritems() %}
     {{ key }} {{ value }}
-{%-   endfor %}
+{%- endfor %}
   </{{ storage_type }}>
 </zodb_db>
 

stack/slapos.cfg

@@ -102,7 +102,7 @@ eggs =
 [versions]
 setuptools = 40.4.3
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos014
+zc.buildout = 2.7.1+slapos001
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -117,6 +117,7 @@ cffi = 1.9.1
 click = 6.7
 cliff = 2.4.0
 cmd2 = 0.7.0
+collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
 cryptography = 2.3.1
 decorator = 4.0.11
@@ -137,14 +138,14 @@ pytz = 2016.10
 requests = 2.13.0
 six = 1.12.0
 slapos.cookbook = 1.0.123
-slapos.core = 1.5.3
+slapos.core = 1.5.7
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
-slapos.libnetworkcache = 0.19
-slapos.rebootstrap = 4.2
+slapos.libnetworkcache = 0.20
+slapos.rebootstrap = 4.3
 slapos.recipe.build = 0.42
-slapos.recipe.cmmi = 0.11
-slapos.toolbox = 0.97
+slapos.recipe.cmmi = 0.12
+slapos.toolbox = 0.104
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1