Update Release Candidate

b69943d5 · Thomas Gambier · 958f433f · 9e4711b8 · b69943d5 · b69943d5
Commit b69943d5 authored Mar 01, 2021 by Thomas Gambier 🚴🏼
39 changed files
--- a/component/bcrypt/buildout.cfg
+++ b/component/bcrypt/buildout.cfg
 [buildout]
 extends =
  ../libffi/buildout.cfg
+  ../python-cffi/buildout.cfg

 parts =
  bcrypt
@@ -16,3 +17,5 @@ egg = bcrypt
 rpath =
  ${libffi:location}/lib/
 environment = bcrypt-env
+setup-eggs =
+  ${python-cffi:egg}
--- a/component/cyrus-sasl/0001-db_gdbm-fix-gdbm_errno-overlay-from-gdbm_close.patch
+++ b/component/cyrus-sasl/0001-db_gdbm-fix-gdbm_errno-overlay-from-gdbm_close.patch
+From af48f6fec9a7b6374d4153c5db894d4a1f349645 Mon Sep 17 00:00:00 2001
+From: Jonas Jelten <jj@sft.mx>
+Date: Sat, 2 Feb 2019 20:53:37 +0100
+Subject: [RFC] db_gdbm: fix gdbm_errno overlay from gdbm_close
+
+`gdbm_close` also sets gdbm_errno since version 1.17.
+This leads to a problem in `libsasl` as the `gdbm_close` incovation overlays
+the `gdbm_errno` value which is then later used for the error handling.
+---
+ sasldb/db_gdbm.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/sasldb/db_gdbm.c b/sasldb/db_gdbm.c
+index ee56a6b..c908808 100644
+--- a/sasldb/db_gdbm.c
+++ b/sasldb/db_gdbm.c
+@@ -107,9 +107,11 @@ int _sasldb_getdata(const sasl_utils_t *utils,
+   gkey.dptr = key;
+   gkey.dsize = key_len;
+   gvalue = gdbm_fetch(db, gkey);
+  int fetch_errno = gdbm_errno;
+
+   gdbm_close(db);
+   if (! gvalue.dptr) {
+-      if (gdbm_errno == GDBM_ITEM_NOT_FOUND) {
+      if (fetch_errno == GDBM_ITEM_NOT_FOUND) {
+           utils->seterror(conn, SASL_NOLOG,
+ 			  "user: %s@%s property: %s not found in %s",
+ 			  authid, realm, propName, path);
+-- 
+2.30.1
+
--- a/component/cyrus-sasl/buildout.cfg
+++ b/component/cyrus-sasl/buildout.cfg
@@ -15,6 +15,9 @@ shared = true
 url = ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz
 md5sum = a7f4e5e559a0e37b3ffc438c9456e425
 location = @@LOCATION@@
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/0001-db_gdbm-fix-gdbm_errno-overlay-from-gdbm_close.patch#eef9bb30101c29f045ed68db45d5e5b3
 pre-configure =
  "${libtool:location}/bin/libtoolize" -c -f &&
  "${automake:location}/bin/aclocal" -I "${libtool:location}/share/aclocal" -I config -I cmulocal &&

--- a/component/dbus/buildout.cfg
+++ b/component/dbus/buildout.cfg
@@ -11,8 +11,8 @@ parts =
 [dbus]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://dbus.freedesktop.org/releases/dbus/dbus-1.11.6.tar.gz
-md5sum = fdb03ad2f2861a171fe7d0caad3b415f 
+url = https://dbus.freedesktop.org/releases/dbus/dbus-1.12.20.tar.gz
+md5sum = dfe8a71f412e0b53be26ed4fbfdc91c4
 location = @@LOCATION@@
 configure-options =
  --disable-static
@@ -21,14 +21,14 @@ make-targets = install && ${:location}/bin/dbus-uuidgen > ${:location}/var/lib/d
 environment =
  PATH=${pkgconfig:location}/bin:%(PATH)s
  PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig
-  CPPFLAGS=-I${libexpat:location}/include
-  LDFLAGS=-L${libexpat:location}/lib
+  EXPAT_CFLAGS=-I${libexpat:location}/include
+  EXPAT_LIBS=-L${libexpat:location}/lib -lexpat

 [dbus-glib]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.104.tar.gz
-md5sum = 5497d2070709cf796f1878c75a72a039
+url = https://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.110.tar.gz
+md5sum = d7cebf1d69445cbd28b4983392145192
 configure-options =
  --disable-static
  --disable-gtk-doc-html

--- a/component/firewalld/buildout.cfg
+++ b/component/firewalld/buildout.cfg
@@ -10,22 +10,34 @@ extends =
  ../python-slip/buildout.cfg
  ../dbus/buildout.cfg
  ../flex/buildout.cfg
+  ../nftables/buildout.cfg

 [firewalld]
 recipe = slapos.recipe.cmmi
-url = https://github.com/t-woerner/firewalld/archive/v0.3.14.2.tar.gz
-md5sum = b8f81b536ede502721f69300d374447b
+url = https://github.com/firewalld/firewalld/releases/download/v0.8.6/firewalld-0.8.6.tar.gz
+md5sum = a120286a2c2bc12ece24d719ef2879b5
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
+location = @@LOCATION@@
+sbin-dir = sbin-orig
 pre-configure =
  sed -i -E 's,(^SUBDIRS *=.*) doc(\b.*),\1\2,' Makefile.am
  aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal
  ./autogen.sh
+configure-options =
+  --sbindir=${:location}/${:sbin-dir}
+  --disable-docs
+  --with-iptables=${iptables:location}/sbin/iptables-legacy
+  --with-iptables-restore=${iptables:location}/sbin/iptables-legacy-restore
+  --with-ip6tables=${iptables:location}/sbin/ip6tables-legacy
+  --with-ip6tables-restore=${iptables:location}/sbin/ip6tables-legacy-restore
+  --with-ebtables=${iptables:location}/sbin/ebtables-legacy
+  --with-ebtables-restore=${iptables:location}/sbin/ebtables-legacy-restore
+  --with-ipset=/bin/false

 environment =
  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
  PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${perl:location}/bin:%(PATH)s
  CPPFLAGS=-I${gettext:location}/include
-  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
  M4=${m4:location}/bin/m4
  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
  FIREWALLD_DEVEL_ENV=${buildout:parts-directory}/${:_buildout_section_name_}/src
@@ -33,9 +45,9 @@ environment =

 [dbus-python]
 recipe = slapos.recipe.cmmi
-version = 1.2.4
+version = 1.2.16
 url = http://dbus.freedesktop.org/releases/dbus-python/dbus-python-${:version}.tar.gz
-md5sum = 7372a588c83a7232b4e08159bfd48fe5 
+md5sum = 51a45c973d82bedff033a4b57d69d5d8
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
 environment =
  PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
@@ -83,8 +95,6 @@ environment =
  FFI_LIBS=-L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include
  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
-#  GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
-#  GLIB_LIBS=-L${glib:location}/lib -lgobject-2.0 -lgthread-2.0 -pthread -lrt -lglib-2.0
  GIO_LIBS=-L${glib:location}/lib -lgio-2.0
  GI_CFLAGS=-I${gobject-introspection:location}/include/gobject-introspection-1.0
  GI_LIBS=-L${gobject-introspection:location}/lib -lgirepository-1.0
@@ -101,17 +111,29 @@ md5sum = 893752ba2e93a1e96334dfee19f884ad
 recipe = plone.recipe.command
 python = ${buildout:bin-directory}/${firewalld-eggs:interpreter}
 command =
-  sed -i 's#/usr/bin/python -Es#${:python}#' ${firewalld:location}/bin/firewall-cmd
-  sed -i 's#/usr/bin/python -Es#${:python}#' ${firewalld:location}/sbin/firewalld
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-applet
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-cmd
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-config
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/bin/firewall-offline-cmd
+  sed -i 's#python2.7#${:python}#' ${firewalld:location}/${firewalld:sbin-dir}/firewalld
+
+  sed -i 's#python2.7 /usr/bin/firewall-config#${:python} ${firewalld:location}/bin/firewall-config#' ${firewalld:location}/etc/firewalld/lockdown-whitelist.xml
+
  sed -i 's#DefaultZone=public#DefaultZone=trusted#' ${firewalld:location}/etc/firewalld/firewalld.conf
+
+  sed -i 's#/etc/firewalld#${firewalld:location}/etc/firewalld#'     ${firewalld:python-egg}/firewall/config/__init__.py
  sed -i 's#/usr/lib/firewalld#${firewalld:location}/lib/firewalld#' ${firewalld:python-egg}/firewall/config/__init__.py
-  sed -i 's#/usr/share/#${firewalld:location}/share#' ${firewalld:python-egg}/firewall/config/__init__.py
+  sed -i 's#/usr/share/#${firewalld:location}/share#'                ${firewalld:python-egg}/firewall/config/__init__.py
+
  sed -i "s#import sys#import sys, os\n\nos.environ['GI_TYPELIB_PATH'] = '${gobject-introspection:location}/lib/girepository-1.0/'#" ${:python}
  sed -i 's#<syslog/>#<!-- no syslog -->#' ${dbus:location}/etc/dbus-1/system.conf
  sed -i 's#<user>messagebus</user>#<user>slapsoft</user>#' ${dbus:location}/share/dbus-1/system.conf
  cp -f ${firewalld:location}/lib/firewalld/zones/trusted.xml ${firewalld:location}/etc/firewalld/zones/
-  cp -f ${firewalld:location}/etc/dbus-1/system.d/FirewallD.conf ${dbus:location}/share/dbus-1/system.d/
-  mkdir -p ${buildout:directory}/sbin
+  cp -f ${firewalld:location}/share/dbus-1/system.d/FirewallD.conf ${dbus:location}/share/dbus-1/system.d/
+  mkdir -p ${firewalld:location}/sbin
+  echo -n '#!/bin/sh\nLD_LIBRARY_PATH=${nftables:location}/lib ${firewalld:location}/${firewalld:sbin-dir}/firewalld "$@"' > ${firewalld:location}/sbin/firewalld
+  chmod a+x ${firewalld:location}/sbin/firewalld
+
 update-command = ${:command}
 stop-on-error = true

@@ -127,3 +149,4 @@ extra-paths =
  ${dbus-python:python-egg}
  ${firewalld:python-egg}
  ${pygobject3:python-egg}
+  ${nftables:python-egg}
--- a/component/gmp/buildout.cfg
+++ b/component/gmp/buildout.cfg
@@ -9,10 +9,12 @@ parts =
 [gmp]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 6.1.1
+version = 6.2.1
 url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
-md5sum = e70e183609244a332d80529e7e155a35
+md5sum = 0b82665c4a92fd2ade7440c13fcaa42b
 configure-options =
+  --disable-assembly
+  --host=none-pc-linux-gnu
  --enable-cxx
  --disable-static
 environment =

--- a/component/golang/buildout.cfg
+++ b/component/golang/buildout.cfg
@@ -52,8 +52,8 @@ environment-extra =

 [golang1.14]
 <= golang-common
-url = https://golang.org/dl/go1.14.12.src.tar.gz
-md5sum = ae124560e0185a82883b57cc9662aa73
+url = https://golang.org/dl/go1.14.15.src.tar.gz
+md5sum = 1065b705a0d330136084f4e1628baf82

 # go1.14 needs go1.4 to bootstrap
 environment-extra =
@@ -61,13 +61,22 @@ environment-extra =

 [golang1.15]
 <= golang-common
-url = https://golang.org/dl/go1.15.5.src.tar.gz
-md5sum = 12a60d319ff58c03c323e8dcc125ae81
+url = https://golang.org/dl/go1.15.8.src.tar.gz
+md5sum = 7e4b8580bfe25ffc51501b1dc3d2d8ce

 # go1.15 needs go1.4 to bootstrap
 environment-extra =
  GOROOT_BOOTSTRAP=${golang14:location}

+[golang1.16]
+<= golang-common
+url = https://golang.org/dl/go1.16.src.tar.gz
+md5sum = b767f7e81d886b4208874ea953db2ce3
+
+# go1.16 needs go1.4 to bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}
+

 # ---- infrastructure to build Go workspaces / projects ----

@@ -113,7 +122,7 @@ bin	= ${:directory}/bin
 depends = ${gowork.goinstall:recipe}

 # go version used for the workspace (possible to override in applications)
-golang  = ${golang1.14:location}
+golang  = ${golang1.15:location}

 # no special build flags by default
 buildflags =
@@ -133,7 +142,7 @@ recipe	= slapos.recipe.template
 url     = ${:_profile_base_location_}/goenv.sh.in
 output	= ${gowork:directory}/env.sh
 depends = ${gowork.mkdir:recipe}
-md5sum	= f89553711ea95ad1f6727b76747a62bf
+md5sum	= 6efdf5d63381c240c7d206d7939a63f7

 [gowork.mkdir]
 # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)

--- a/component/golang/goenv.sh.in
+++ b/component/golang/goenv.sh.in
@@ -10,11 +10,18 @@ export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_
 # ---- 8< ----

 export GOPATH=$X:$GOPATH
-export PATH=$X/bin:$PATH
-export PS1="(`basename $X`) $PS1"
+export GOBIN=$X/bin
 export GOCACHE=$X/cache
+export GOMODCACHE=$X/pkg/mod
 export GOENV=$X/goenv

+export PATH=$GOBIN:$PATH
+export PS1="(`basename $X`) $PS1"
+
+# by default make go run in module-aware mode only if go.mod is present, and in GOPATH-mode otherwise.
+# (we still have some GOPATH uses; TODO -> default to "on" after GOPATH-mode usage is gone)
+export GO111MODULE=auto
+
 # strip trailing : from $GOPATH, $PKG_CONFIG_PATH
 GOPATH=$${GOPATH%:}
 PKG_CONFIG_PATH=$${PKG_CONFIG_PATH%:}
--- a/component/groonga/buildout.cfg
+++ b/component/groonga/buildout.cfg
@@ -15,12 +15,12 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/groonga/groonga-10.0.5.tar.gz
-md5sum = 12f5c7b522afd72d0be2caa1e5f254da
+url = https://packages.groonga.org/source/groonga/groonga-11.0.0.tar.gz
+md5sum = 2ea277c19f21b9d45410547173ff41c2
 groonga-plugin-dir =  @@LOCATION@@/lib/groonga/plugins/
 # temporary patch to respect more tokens in natural language mode.
 patches =
-  ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
+  ${:_profile_base_location_}/groonga.patch#5d831331ddfdcd87e91b68949c339a1d
 patch-options = -p1
 configure-options =
  --disable-static

--- a/component/groonga/groonga.patch
+++ b/component/groonga/groonga.patch
-diff --git a/lib/ii.c b/lib/ii.c
-index d10b84d..e20bdce 100644
 --- a/lib/ii.c
 +++ b/lib/ii.c
-@@ -5695,7 +5695,9 @@ grn_ii_similar_search(grn_ctx *ctx, grn_ii *ii,
-     ? (optarg->similarity_threshold > GRN_HASH_SIZE(h)
+@@ -10387,7 +10387,9 @@
+     ? (data->optarg->similarity_threshold > GRN_HASH_SIZE(h)
        ? GRN_HASH_SIZE(h)
-        : optarg->similarity_threshold)
+        : data->optarg->similarity_threshold)
 -    : (GRN_HASH_SIZE(h) >> 3) + 1;
 +    : (GRN_HASH_SIZE(h) < 8
 +       ? GRN_HASH_SIZE(h)

--- a/component/h5py/buildout.cfg
+++ b/component/h5py/buildout.cfg
@@ -17,6 +17,8 @@ egg = h5py
 setup-eggs = 
  ${cython:egg}
  ${numpy:egg}
+  pkgconfig
+  six
 include-dirs =
  ${hdf5:location}/include
 library-dirs =

--- a/component/helloweb/buildout.cfg
+++ b/component/helloweb/buildout.cfg
@@ -25,7 +25,7 @@ parts =
 install =
    lab.nexedi.com/nexedi/helloweb/go/...

-golang  = ${golang1.15:location}
+golang  = ${golang1.16:location}

 # -*- go -*-
 [helloweb-go]

--- a/component/mariadb/buildout.cfg
+++ b/component/mariadb/buildout.cfg
@@ -30,8 +30,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
-version = 10.4.17
-md5sum = e8193b9cd008b6d7f177f5a5c44c7a9f
+version = 10.4.18
+md5sum = 22e429d532fcfcb0e4beba64e1c296f0
 location = @@LOCATION@@
 pre-configure =
  set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -96,8 +96,8 @@ post-install =
 # as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://packages.groonga.org/source/mroonga/mroonga-10.05.tar.gz
-md5sum = a690811be5cc0593acc12ecf4ac9c40f
+url = https://packages.groonga.org/source/mroonga/mroonga-11.00.tar.gz
+md5sum = 0aefcbb3c0dc6e29c53c6ee6287114b8
 pre-configure = 
  rm -rf fake_mariadb_source
  mkdir -p fake_mariadb_source
@@ -127,7 +127,7 @@ environment =
  PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig


-### XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### XXX keep using mariadb 10.3 because 10.4 causes crash
 ### (we just override here for easier revert)
 [mariadb-10.3]
 <= mariadb-10.4
@@ -143,4 +143,4 @@ post-install =

 [mariadb]
 location = ${mariadb-10.3:location}
-### /XXX keep using mariadb 10.3.22 because 10.4 cause crash
+### /XXX keep using mariadb 10.3 because 10.4 causes crash
--- a/component/nftables/buildout.cfg
+++ b/component/nftables/buildout.cfg
+[buildout]
+parts =
+  nftables
+
+extends =
+  ../defaults.cfg
+  ../gmp/buildout.cfg
+
+
+[libnml]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/libmnl/files/libmnl-1.0.4.tar.bz2
+md5sum = be9b4b5328c6da1bda565ac5dffadb2d
+
+[libnftnl]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/libnftnl/files/libnftnl-1.1.9.tar.bz2
+md5sum = e03cefd53f4b076d959abe36de5c38f8
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+
+[jansson]
+recipe = slapos.recipe.cmmi
+url = https://digip.org/jansson/releases/jansson-2.13.1.tar.gz
+md5sum = 570af45b8203e95876d71fecd56cee20
+
+[iptables]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/iptables/files/iptables-1.8.7.tar.bz2
+md5sum = 602ba7e937c72fbb7b1c2b71c3b0004b
+location = @@LOCATION@@
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig:${libnftnl:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${:location}/lib
+
+[nftables]
+recipe = slapos.recipe.cmmi
+url = http://www.netfilter.org/projects/nftables/files/nftables-0.9.8.tar.bz2
+md5sum = 77bf0bd43e65e92212fc73139a2e47fc
+python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python${python:version}/site-packages
+environment =
+  PKG_CONFIG_PATH=${libnml:location}/lib/pkgconfig:${libnftnl:location}/lib/pkgconfig
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  LDFLAGS = -L${gmp:location}/lib -L${jansson:location}/lib
+  CFLAGS = -I${gmp:location}/include -I${jansson:location}/include
+configure-options =
+  --enable-python
+  --with-json
+  --without-cli
+
--- a/component/nxdtest/buildout.cfg
+++ b/component/nxdtest/buildout.cfg
@@ -20,6 +20,6 @@ egg     = nxdtest
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = 40e2c4ab
+revision = b5a74214
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git
--- a/component/python-pynacl/buildout.cfg
+++ b/component/python-pynacl/buildout.cfg
@@ -2,6 +2,7 @@
 extends =
  ../libffi/buildout.cfg
  ../pkgconfig/buildout.cfg
+  ../python-cffi/buildout.cfg

 parts =
  python-pynacl
@@ -14,6 +15,8 @@ library-dirs =
  ${libffi:location}/lib/
 rpath =
  ${libffi:location}/lib/
+setup-eggs =
+  ${python-cffi:egg}

 [python-pynacl-env]
 PATH = ${pkgconfig:location}/bin:%(PATH)s

--- a/component/slapos/obs.cfg
+++ b/component/slapos/obs.cfg
@@ -184,6 +184,8 @@ configure-command =
 environment +=
  DESTDIR=${buildout:destdir}

-[versions]
-# More recently version of cliff uses pbr which will break package build on OBS.
-cliff = 1.4.5
+[iptables]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+environment +=
+  DESTDIR=${buildout:destdir}
--- a/component/statsmodels/buildout.cfg
+++ b/component/statsmodels/buildout.cfg
 [buildout]
 extends =
  ../pandas/buildout.cfg
+  ../scipy/buildout.cfg

 parts =
  statsmodels
@@ -14,3 +15,4 @@ egg = statsmodels
 environment = statsmodels-env
 setup-eggs =
  ${pandas:egg}
+  ${scipy:egg}
--- a/slapos/test/test_json_schema.py
+++ b/slapos/test/test_json_schema.py
@@ -25,14 +25,16 @@
 #
 ##############################################################################

-import unittest
-import os
+import collections
 import glob
 import json
-import collections
-import slapos.test
+import os
+import unittest
+
 import jsonschema
+import six

+import slapos.test

 def createInstanceParameterSchemaValidatorTest(path):
  # Test that json is a valid json schema, supports several
@@ -65,7 +67,7 @@ def createSoftwareCfgValidatorTest(path, software_cfg_schema):
      # also make sure request and response schemas can be resolved
      schema.setdefault('$id', 'file://' + path)
      resolver = jsonschema.RefResolver.from_schema(schema)
-      for software_type_definition in schema['software-type'].itervalues():
+      for software_type_definition in six.itervalues(schema['software-type']):
        resolver.resolve(software_type_definition['request'])
        resolver.resolve(software_type_definition['response'])


--- a/software/caddy-frontend/README.caddy_frontend.rst
+++ b/software/caddy-frontend/README.caddy_frontend.rst
@@ -6,6 +6,8 @@ Frontend system using Caddy, based on apache-frontend software release, allowing

 Caddy Frontend works using the master instance / slave instance design. It means that a single main instance of Caddy will be used to act as frontend for many slaves.

+This documentation covers only specific scenarios. Most of the parameters are described in `software.cfg.json <software.cfg.json>`_.
+
 Software type
 =============

@@ -129,7 +131,7 @@ Example sessions is::

  cat certificate.pem ca.pem key.pem > bundle.pem

-  curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem master-key-upload-url+authtoken
+  curl -g --upload-file bundle.pem --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" master-key-upload-url+authtoken

 This replaces old request parameters:

@@ -157,7 +159,7 @@ Example sessions is::

  cat certificate.pem ca.pem key.pem > bundle.pem

-  curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem key-upload-url+authtoken
+  curl -g --upload-file bundle.pem --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" key-upload-url+authtoken

 This replaces old request parameters:

@@ -215,29 +217,15 @@ Example of value: "/erp5/web_site_module/hosting/"

 url
 ~~~
-Necessary to activate cache. ``url`` of backend to use.
-
-``url`` is an optional parameter.
+URL of the backend to use, optional but will result with non functioning slave.

 Example: http://mybackend.com/myresource

-domain
-~~~~~~
-
-Necessary to activate cache.
-
-The frontend will be accessible from this domain.
-
-``domain`` is an optional parameter.
-
-Example: www.mycustomdomain.com
-
 enable_cache
 ~~~~~~~~~~~~

-Necessary to activate cache.
+Enables HTTP cache, optional.

-``enable_cache`` is an optional parameter.

 health-check-*
 ~~~~~~~~~~~~~~

--- a/software/caddy-frontend/buildout.hash.cfg
+++ b/software/caddy-frontend/buildout.hash.cfg
@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = de69a8c408ce4f228fc22eacb7e96657
+md5sum = 04015a7a552285984d091293ef573fb9

 [profile-common]
 filename = instance-common.cfg.in
@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a

 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 9cc78e7ce1960691e37f103855ff0dc9
+md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3

 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = eb98ffd96b2768cc6a5cf664b23aabd3
+md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772

 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in

--- a/software/caddy-frontend/instance-apache-replicate.cfg.in
+++ b/software/caddy-frontend/instance-apache-replicate.cfg.in
@@ -21,7 +21,6 @@
        'ciphers',
        'request-timeout',
        'authenticate-to-backend',
-        'strict-transport-security',
    ]
 %}
 {% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}

--- a/software/caddy-frontend/instance-caddy-input-schema.json
+++ b/software/caddy-frontend/instance-caddy-input-schema.json
@@ -19,7 +19,7 @@
      "type": "string"
    },
    "domain": {
-      "description": "Base Domain for create subdomains (ie.: example.com).",
+      "description": "Domain used to generate automatic hostnames for slaves. For example 'example.com' will result with slave hostname 'slaveref.example.com'.",
      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
      "title": "Domain",
      "type": "string"
@@ -107,12 +107,6 @@
      ],
      "title": "Authenticate to backend",
      "type": "string"
-    },
-    "strict-transport-security": {
-      "title": "Strict Transport Security",
-      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
-      "default": "0",
-      "type": "integer"
    }
  },
  "title": "Input Parameters",

--- a/software/caddy-frontend/instance-slave-caddy-input-schema.json
+++ b/software/caddy-frontend/instance-slave-caddy-input-schema.json
@@ -288,7 +288,7 @@
    },
    "strict-transport-security": {
      "title": "Strict Transport Security",
-      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
+      "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
      "default": "0",
      "type": "integer"
    },

--- a/software/caddy-frontend/instance.cfg.in
+++ b/software/caddy-frontend/instance.cfg.in
@@ -106,4 +106,3 @@ configuration.backend-haproxy-statistic-port = 21444
 configuration.authenticate-to-backend = False
 configuration.rotate-num = 4000
 configuration.slave-introspection-https-port = 22443
-configuration.strict-transport-security = 0
--- a/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
+++ b/software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
@@ -68,7 +68,7 @@ context =
 {%-   for key in ['disabled-cookie-list'] %}
 {%-     do slave_instance.__setitem__(key, slave_instance.get(key, '').split()) %}
 {%-   endfor %}
-{%-   for key, default in [('virtualhostroot-http-port', '80'), ('virtualhostroot-https-port', '443')] %}
+{%-   for key, default in [('virtualhostroot-http-port', '80'), ('virtualhostroot-https-port', '443'), ('strict-transport-security', '0')] %}
 {%-     do slave_instance.__setitem__(key, int(slave_instance.get(key, default))) %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('default-path', slave_instance.get('default-path', '').strip('/') | urlencode) %}
@@ -128,7 +128,7 @@ context =
 {%-   do part_list.extend([slave_section_title]) %}
 {%-   set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
 {#-   Pass backend timeout values #}
-{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend', 'strict-transport-security'] %}
+{%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
 {%-     if slave_instance.get(key, '') == '' %}
 {%-       do slave_instance.__setitem__(key, configuration[key]) %}
 {%-     endif %}

--- a/software/caddy-frontend/test/test.py
+++ b/software/caddy-frontend/test/test.py
@@ -6814,7 +6814,6 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
      'ciphers': 'ciphers',
      'request-timeout': 100,
      'authenticate-to-backend': True,
-      'strict-transport-security': 200,
      # specific parameters
      '-frontend-config-1-ram-cache-size': '512K',
      '-frontend-config-2-ram-cache-size': '256K',
@@ -6912,8 +6911,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
        u'ram-cache-size': u'512K',
        u're6st-verification-url': u're6st-verification-url',
        u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
      },
      'caddy-frontend-2': {
        'X-software_release_url': self.frontend_2_sr,
@@ -6940,8 +6938,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
        u'ram-cache-size': u'256K',
        u're6st-verification-url': u're6st-verification-url',
        u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
      },
      'caddy-frontend-3': {
        'X-software_release_url': self.frontend_3_sr,
@@ -6967,8 +6964,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
        u'port': u'11443',
        u're6st-verification-url': u're6st-verification-url',
        u'request-timeout': u'100',
-        u'slave-kedifa-information': u'{}',
-        u'strict-transport-security': u'200'
+        u'slave-kedifa-information': u'{}'
      },
      'kedifa': {
        'X-software_release_url': self.kedifa_sr,
@@ -7014,8 +7010,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
        'request-timeout': '100',
        'root_instance_title': 'testing partition 0',
        'slap_software_type': 'RootSoftwareInstance',
-        'slave_instance_list': [],
-        'strict-transport-security': '200'
+        'slave_instance_list': []
      }
    }
    self.assertEqual(

--- a/software/galene/buildout.hash.cfg
+++ b/software/galene/buildout.hash.cfg
@@ -15,4 +15,4 @@

 [instance-cfg]
 filename = instance.cfg.in
-md5sum = 8341d1d8c1839a1c623dd0fd771ef104
+md5sum = e54af429aba9b395c13471de254c4621
--- a/software/galene/instance.cfg.in
+++ b/software/galene/instance.cfg.in
@@ -20,7 +20,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
-configuration.ice-servers.json = [{"urls":["stun:turn.api.nexedi.net:3478"]}]
+configuration.ice-servers.json = [{"urls":["stun:turn-paris-rapidspace.api.nexedi.net:443"]}]

 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -50,28 +50,29 @@ storage-path = $${directory:data}/.passwd
 username = admin

 [ice-servers.json]
-recipe = collective.recipe.template
-input = inline:
+recipe = slapos.recipe.template:jinja2
+template = inline:
  $${slap-configuration:configuration.ice-servers.json}
-output = $${directory:data}/ice-servers.json
+rendered = $${directory:data}/ice-servers.json

 [groups-json]
-recipe = collective.recipe.template
-input = inline:{
+recipe = slapos.recipe.template:jinja2
+template = inline:{
    "public":true,
    "op": [{"username":"$${admin-password:username}","password":"$${admin-password:passwd}"}],
    "other": [],
    "presenter": [{"username": "", "password": "nexedi"}],
    "max-users":100
  }
-output = $${directory:groups}/public.json
+rendered = $${directory:groups}/public.json

 [galene-wrapper]
-recipe = collective.recipe.template
+recipe = slapos.recipe.template:jinja2
 port = 8443
 ip = $${slap-configuration:ipv6-random}
-input = inline:
-  #!/bin/bash
+template =
+  inline:#!/bin/sh
+  
  ulimit -n $(ulimit -Hn)
  exec ${galene:location}/bin/galene \
    -static ${galene-repository:location}/static \
@@ -80,8 +81,7 @@ input = inline:
    -data $${directory:data} \
    -http [$${:ip}]:$${:port} \
    -turn ""
-output = $${directory:services}/galene
-mode = 0755
+rendered = $${directory:services}/galene
 depends =
  $${ice-servers.json:recipe}
  $${groups-json:recipe}

--- a/software/html5as-base/software.cfg
+++ b/software/html5as-base/software.cfg
@@ -16,6 +16,8 @@ parts =
 # Call creation of instance.cfg file that will be called for deployment of
 # instance
    template-cfg
+# Add extra egg
+    extra-eggs

 # Download instance.cfg.in (buildout profile used to deployment of instance),
 # replace all {{ foo_bar }} parameters by real values
@@ -55,7 +57,7 @@ mode = 0644
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 _update_hash_filename_ = templates/launcher.in
-md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+md5sum = 6cb0d64905ae7fc67277c1bf76b86875
 mode = 0644

 [template_mime_types]
@@ -65,6 +67,12 @@ _update_hash_filename_ = templates/mime_types.in
 md5sum = 4ef94a7b458d885cd79ba0b930a5727e
 mode = 0644

+[extra-eggs]
+recipe  = zc.recipe.egg
+eggs    =
+    plone.recipe.command
+
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
+plone.recipe.command = 1.1
--- a/software/html5as/buildout.hash.cfg
+++ b/software/html5as/buildout.hash.cfg
@@ -29,7 +29,7 @@ md5sum = 61dc4c82bf48563228ce4dea6c5c6319

 [template_launcher]
 _update_hash_filename_ = templates/launcher.in
-md5sum = 8d4d3152f5125f73d0c3f3ea1c3699dd
+md5sum = 6cb0d64905ae7fc67277c1bf76b86875

 [template_mime_types]
 _update_hash_filename_ = templates/mime_types.in

--- a/software/jupyter/software.cfg.json
+++ b/software/jupyter/software.cfg.json
@@ -8,7 +8,7 @@
      "software-type": "default",
      "description": "Default",
      "request": "instance-jupyter-input-schema.json",
-      "response": "instance-jupyter-ouput-schema.json",
+      "response": "instance-jupyter-output-schema.json",
      "index": 1
    }
  }

--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -2,6 +2,7 @@

 extends =
  ../../component/bcrypt/buildout.cfg
+  ../../component/python-cliff/buildout.cfg
  ../../component/curl/buildout.cfg
  ../../component/openssl/buildout.cfg
  ../../component/git/buildout.cfg
@@ -185,6 +186,7 @@ setup = ${slapos.core-repository:location}
 [python-interpreter]
 eggs += ${:extra-eggs}
 extra-eggs =
+  ${python-cliff:egg}
  ${lxml-python:egg}
  ${slapos.core-setup:egg}
  ${pillow-python:egg}

--- a/software/slapos-testing/software.cfg
+++ b/software/slapos-testing/software.cfg
@@ -14,6 +14,7 @@ extends =
  ../../component/lmsensors/buildout.cfg
  ../../component/rsync/buildout.cfg
  ../../stack/slapos.cfg
+  ../../stack/caucase/buildout.cfg
  ../../stack/nxdtest.cfg
  ./buildout.hash.cfg

@@ -235,12 +236,8 @@ httmock = 1.2.6
 manuel = 1.9.0
 mock = 2.0.0
 testfixtures = 6.11.0
-pem = 18.1.0
 pycurl = 7.43.0.2
 pyflakes = 2.0.0
 zope.testing = 4.6.2
 urllib3 = 1.24.1
 pathlib = 1.0.1
-# Required by:
-# caucase
-PyJWT = 1.6.4
--- a/software/theia/buildout.hash.cfg
+++ b/software/theia/buildout.hash.cfg
@@ -15,7 +15,7 @@

 [instance]
 filename = instance.cfg.in
-md5sum = 397a2e0ac0d7f8813e0b594af6785300
+md5sum = 4f119df2b2d71e5ec1fd04051fd8fff8

 [yarn.lock]
 filename = yarn.lock

--- a/software/theia/instance.cfg.in
+++ b/software/theia/instance.cfg.in
--- a/software/theia/software.cfg.json
+++ b/software/theia/software.cfg.json
@@ -8,7 +8,7 @@
      "software-type": "default",
      "description": "Default",
      "request": "instance-input-schema.json",
-      "response": "instance-ouput-schema.json",
+      "response": "instance-output-schema.json",
      "index": 1
    }
  }

--- a/stack/erp5/buildout.cfg
+++ b/stack/erp5/buildout.cfg
@@ -378,6 +378,31 @@ initialization =
  repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
  sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]

+[zope-product-with-eggtestinfo]
+recipe = zc.recipe.egg:custom
+setup-eggs =
+  eggtestinfo
+egg = ${:_buildout_section_name_}
+
+[Products.CMFUid]
+<= zope-product-with-eggtestinfo
+[Products.CMFActionIcons]
+<= zope-product-with-eggtestinfo
+[Products.CMFCalendar]
+<= zope-product-with-eggtestinfo
+[Products.CMFCore]
+<= zope-product-with-eggtestinfo
+[Products.CMFDefault]
+<= zope-product-with-eggtestinfo
+[Products.CMFTopic]
+<= zope-product-with-eggtestinfo
+[Products.DCWorkflow]
+<= zope-product-with-eggtestinfo
+Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
+Products.DCWorkflow-patch-options = -p1
+[Products.GenericSetup]
+<= zope-product-with-eggtestinfo
+
 [eggs]
 <= neoppod
 eggs = ${neoppod:eggs}
@@ -472,14 +497,14 @@ eggs = ${neoppod:eggs}
  Products.PluginRegistry

 # CMF 2.2
-  Products.CMFActionIcons
-  Products.CMFCalendar
-  Products.CMFCore
-  Products.CMFDefault
-  Products.CMFTopic
-  Products.CMFUid
-  Products.DCWorkflow
-  Products.GenericSetup
+  ${Products.CMFActionIcons:egg}
+  ${Products.CMFCalendar:egg}
+  ${Products.CMFCore:egg}
+  ${Products.CMFDefault:egg}
+  ${Products.CMFTopic:egg}
+  ${Products.CMFUid:egg}
+  ${Products.DCWorkflow:egg}
+  ${Products.GenericSetup:egg}
  five.localsitemanager

 # Other products
@@ -545,8 +570,6 @@ PyPDF2-patches = ${:_profile_base_location_}/../../component/egg-patch/PyPDF2/00
 PyPDF2-patch-options = -p1
 Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#1d9a56e9af4371f5b6951ebf217a15d7
 Acquisition-patch-options = -p1
-Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
-Products.DCWorkflow-patch-options = -p1
 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
 python-magic-patch-options = -p1

@@ -773,3 +796,5 @@ beautifulsoup4 = 4.8.2
 # WSGIProxy2==0.4.6
 WebOb = 1.8.5
 soupsieve = 1.9.5
+
+eggtestinfo = 0.3
--- a/stack/slapos.cfg
+++ b/stack/slapos.cfg
@@ -121,6 +121,7 @@ depends = ${slapos-toolbox-dependencies:eggs}
 # slapos.toolbox dependencies.
 recipe = zc.recipe.egg
 eggs =
+  ${python-cliff:egg}
  ${lxml-python:egg}
  ${pycurl:egg}
  ${python-cryptography:egg}
@@ -135,6 +136,7 @@ setup-eggs = setuptools_scm
 [slapos-command]
 recipe = zc.recipe.egg
 eggs =
+  ${python-cliff:egg}
  ${lxml-python:egg}
  slapos.core
  slapos.libnetworkcache
@@ -198,13 +200,13 @@ setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
 slapos.cookbook = 1.0.171
-slapos.core = 1.6.4
+slapos.core = 1.6.5
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
-slapos.recipe.cmmi = 0.16
+slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.5
 slapos.toolbox = 0.117
 stevedore = 1.21.0
@@ -354,6 +356,8 @@ chardet = 3.0.4
 # requests==2.24.0
 urllib3 = 1.25.9

+pkgconfig = 1.5.1
+
 [networkcache]
 download-cache-url = http://shacache.nxdcdn.com
 download-dir-url = http://shadir.nxdcdn.com