Merge remote-tracking branch 'origin/master' into t

* origin/master: (91 commits)
  NEO: fix stress test against recent zodbtools update
  version up: MariaDB 10.3.26 & 10.4.16
  software/kvm: Fix textarea definition
  Revert "software/kvm: Allow many CD-ROMs to boot from"
  stack/erp5: Make bt5 default value a string at the last step.
  software/helloworld: move to python 3
  stack/slapos.cfg: upgrade rubygemsrecipe version
  software/theia: version up theia 1.7.0
  software/plantuml: move to python3
  component/caddy: Switch to Go 1.14
  Add fmtlib component
  stack/slapos: slapos.cookbook version up (1.0.167)
  Release slapos.cookbook (1.0.167)
  stack/slapos.cfg: version up erp5.util 0.4.69
  lamp: remove redundant items in buildout:parts
  fixup! Clean up usage of CMake
  erp5testnode: request a frontend for logs
  Make easier to use MariaDB 10.4
  version up: GCC 8.4
  version up: CMake 3.18.4
  ...

component/caddy/buildout.cfg

@@ -11,7 +11,7 @@ parts =
 command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
 
 [gowork]
-golang  = ${golang1.12:location}
+golang  = ${golang1.14:location}
 install =
   github.com/caddyserver/caddy/...
 

component/cmake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [cmake]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
-md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a
+url = https://cmake.org/files/v3.18/cmake-3.18.4.tar.gz
+md5sum = 0380beaee1c39a22455db02651abe7be
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/cuneiform/buildout.cfg

-[buildout]
-extends =
-  ../cmake/buildout.cfg
-  ../imagemagick/buildout.cfg
-parts = cuneiform
-
-[cuneiform]
-recipe = slapos.recipe.cmmi
-url = http://launchpad.net/cuneiform-linux/1.0/1.0/+download/cuneiform-linux-1.0.0.tar.bz2
-md5sum = 785232ffffad7d82446fbac08a1c3ef9
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DCMAKE_INSTALL_RPATH=${:location}/lib64:${:location}/lib \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    -DNO_SYSTEM_ENVIRONMENT_PATH=ON \
-    -DCMAKE_INCLUDE_PATH=${imagemagick:location}/include \
-    -DCMAKE_LIBRARY_PATH=${imagemagick:location}/lib \
-    -DCMAKE_INSTALL_RPATH=${:location}/lib64:${:location}/lib:${imagemagick:location}/lib \
-    -DCMAKE_BUILD_TYPE=release \
-    ..
-make-binary =
-  cd build && make

component/defaults.cfg

@@ -34,7 +34,7 @@ depends =
   ${patch:recipe}
 recipe = slapos.recipe.build
 # Latest version provided by SlapOS.
-part = gcc-8.2
+part = gcc-8.4
 # Minimum version for all components that might be required for
 # slapos.rebootstrap (see https://bugs.python.org/issue34112 about Python 3.7+).
 min_version = 5.4

component/fmtlib/buildout.cfg

+[buildout]
+extends =
+  ../cmake/buildout.cfg
+parts =
+  fmtlib
+
+[fmtlib]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/fmtlib/fmt/archive/7.0.3.tar.gz
+md5sum = 57392b7ea09592a2b237a02950f35bb0
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+  -DBUILD_SHARED_LIBS=ON
+  -DFMT_TEST=OFF

component/gcc/buildout.cfg

@@ -10,7 +10,7 @@ extends =
   ../binutils/buildout.cfg
 
 parts =
-  gcc-8.2
+  gcc-8.4
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
@@ -65,16 +65,10 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/libsanitizer_Use_pre-computed_size_of_struct_ustat_for_Linux.patch#1e5f33e89f9fe1ca3e406eabcc621762
 
-[gcc-8.2]
+[gcc-8.4]
 <= gcc-common
-version = 8.2.0
-md5sum = 4ab282f414676496483b3e1793d07862
-patch-binary = ${patch:location}/bin/patch
-patch-options = -p1
-# glibc-2.31-libsanitizer comes from https://github.com/spack/spack/pull/15403
-patches =
-  ${:_profile_base_location_}/glibc-2.31-libsanitizer-1.patch
-  ${:_profile_base_location_}/glibc-2.31-libsanitizer-2.patch
+version = 8.4.0
+md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
 
 [gcc-minimal]
 <= gcc-5.5

component/gcc/glibc-2.31-libsanitizer-1.patch

-From ce9568e9e9cf6094be30e748821421e703754ffc Mon Sep 17 00:00:00 2001
-From: Jakub Jelinek <jakub@redhat.com>
-Date: Fri, 8 Nov 2019 19:53:18 +0100
-Subject: [PATCH] backport: re PR sanitizer/92154 (new glibc breaks arm
- bootstrap due to libsanitizer)
-
-	Backported from mainline
-	2019-10-22  Tamar Christina  <tamar.christina@arm.com>
-
-	PR sanitizer/92154
-	* sanitizer_common/sanitizer_platform_limits_posix.cc:
-	Cherry-pick compiler-rt revision r375220.
-
-From-SVN: r277981
----
- libsanitizer/ChangeLog                                   | 9 +++++++++
- .../sanitizer_common/sanitizer_platform_limits_posix.cc  | 6 +++++-
- 2 files changed, 14 insertions(+), 1 deletion(-)
-
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-index 6cd4a5bac8b0..06a605ff4670 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-@@ -1156,8 +1156,12 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
--#if !defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)
-+#if (!defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)) && \
-+    !defined(__arm__)
- /* On aarch64 glibc 2.20 and earlier provided incorrect mode field.  */
-+/* On Arm glibc 2.31 and later provide a different mode field, this field is
-+   never used by libsanitizer so we can simply ignore this assert for all glibc
-+   versions.  */
- CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
- #endif
- 

component/gcc/glibc-2.31-libsanitizer-2.patch

-From 75003cdd23c310ec385344e8040d490e8dd6d2be Mon Sep 17 00:00:00 2001
-From: Jakub Jelinek <jakub@redhat.com>
-Date: Fri, 20 Dec 2019 17:58:35 +0100
-Subject: [PATCH] backport: re PR sanitizer/92154 (new glibc breaks arm
- bootstrap due to libsanitizer)
-
-	Backported from mainline
-	2019-11-26  Jakub Jelinek  <jakub@redhat.com>
-
-	PR sanitizer/92154
-	* sanitizer_common/sanitizer_platform_limits_posix.h: Cherry-pick
-	llvm-project revision 947f9692440836dcb8d88b74b69dd379d85974ce.
-	* sanitizer_common/sanitizer_platform_limits_posix.cc: Likewise.
-
-From-SVN: r279653
----
- libsanitizer/ChangeLog                            | 10 ++++++++++
- .../sanitizer_platform_limits_posix.cc            |  9 +++------
- .../sanitizer_platform_limits_posix.h             | 15 +--------------
- 3 files changed, 14 insertions(+), 20 deletions(-)
-
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-index 06a605ff4670..d823a12190c0 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc
-@@ -1156,12 +1156,9 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
- CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
--#if (!defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21)) && \
--    !defined(__arm__)
--/* On aarch64 glibc 2.20 and earlier provided incorrect mode field.  */
--/* On Arm glibc 2.31 and later provide a different mode field, this field is
--   never used by libsanitizer so we can simply ignore this assert for all glibc
--   versions.  */
-+#if !SANITIZER_LINUX || __GLIBC_PREREQ (2, 31)
-+/* glibc 2.30 and earlier provided 16-bit mode field instead of 32-bit
-+   on many architectures.  */
- CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
- #endif
- 
-diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-index 73af92af1e8f..6a673a7c9959 100644
---- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
-@@ -211,26 +211,13 @@ namespace __sanitizer {
-     u64 __unused1;
-     u64 __unused2;
- #elif defined(__sparc__)
--#if defined(__arch64__)
-     unsigned mode;
--    unsigned short __pad1;
--#else
--    unsigned short __pad1;
--    unsigned short mode;
-     unsigned short __pad2;
--#endif
-     unsigned short __seq;
-     unsigned long long __unused1;
-     unsigned long long __unused2;
--#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__)
--    unsigned int mode;
--    unsigned short __seq;
--    unsigned short __pad1;
--    unsigned long __unused1;
--    unsigned long __unused2;
- #else
--    unsigned short mode;
--    unsigned short __pad1;
-+    unsigned int mode;
-     unsigned short __seq;
-     unsigned short __pad2;
- #if defined(__x86_64__) && !defined(_LP64)

component/h2o/buildout.cfg

@@ -14,7 +14,7 @@ recipe = slapos.recipe.cmmi
 shared = True
 url = https://github.com/h2o/h2o/archive/v2.2.6.tar.gz
 md5sum = 075283f92347fcb7b2f87dcc5251b922
-configure-command = ${cmake:location}/bin/cmake .
+configure-command = ${cmake:location}/bin/cmake
 configure-options =
   -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DWITH_MRUBY=on

component/inkscape/buildout.cfg

@@ -41,11 +41,10 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://inkscape.org/gallery/item/13330/inkscape-0.92.4_A6N0YOn.tar.bz2
 md5sum = ac30f6d5747fd9c620c00dad500f414f
-location = @@LOCATION@@
 pkg_config_depends = ${freetype:location}/lib/pkgconfig:${gtkmm:location}/lib/pkgconfig:${gtkmm:pkg_config_depends}:${gsl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
-  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DENABLE_POPPLER=OFF
   -DWITH_GNOME_VFS=OFF
   -DWITH_IMAGE_MAGICK=OFF

component/libzip/buildout.cfg

@@ -13,17 +13,16 @@ shared = true
 url = https://libzip.org/download/libzip-1.5.2.tar.xz
 md5sum = f9dd38d273bcdec5d3d1498fe6684f42
 location = @@LOCATION@@
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    -DCMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include \
-    -DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib \
-    ..
-make-binary =
-  cd build && make
+configure-command = cmake
+configure-options =
+  -Bbuild
+  -DCMAKE_BUILD_TYPE=Release
+  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
+  -DCMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
+make-options = -C build
 environment =
   PATH=${cmake:location}/bin:%(PATH)s
   CMAKE_INCLUDE_PATH=${zlib:location}/include:${bzip2:location}/include
   CMAKE_LIBRARY_PATH=${zlib:location}/lib:${bzip2:location}/lib
-  LDFLAGS=-L${:location}/lib -Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
+  LDFLAGS=-Wl,-rpath=${:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/llvm/buildout.cfg

+[buildout]
+extends =
+    ../cmake/buildout.cfg
+    ../git/buildout.cfg
+
+parts = llvm
+
+[llvm]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/llvm/llvm-project/releases/download/llvmorg-${:version}/llvm-${:version}.src.tar.xz
+version = 11.0.0
+md5sum = 85844102335b2e01b3c64b6734fb56f2
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+    -Bbuild
+    -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
+    -DCMAKE_BUILD_TYPE=Release
+    -DLLVM_INSTALL_UTILS=ON
+    -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
+    -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
+make-options = -C build
+CMAKE_CFLAGS = -I${libxml2:location}/include/libxml2 -I${ncurses:location}/include -I${zlib:location}/include
+environment =
+    PATH=${git:location}/bin:%(PATH)s
+    LDFLAGS=-L${libxml2:location}/lib -L${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${zlib:location}/lib

component/mariadb/buildout.cfg

@@ -26,12 +26,12 @@ extends =
 parts =
   mariadb
 
-[mariadb]
+[mariadb-10.4]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.4.14
-md5sum = 9801120ae8acb33904ab4a3366a7714f
+url = https://archive.mariadb.org//mariadb-${:version}/source/mariadb-${:version}.tar.gz
+version = 10.4.16
+md5sum = e7e5071cc879de5358f60789f53df99e
 location = @@LOCATION@@
 pre-configure =
   set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
@@ -129,14 +129,10 @@ environment =
 
 ### XXX keep using mariadb 10.3.22 because 10.4 cause crash
 ### (we just override here for easier revert)
-[mariadb]
-version = 10.3.22
-url = https://downloads.mariadb.org/f/mariadb-10.3.22/source/mariadb-10.3.22.tar.gz/from/http%3A//ftp.hosteurope.de/mirror/archive.mariadb.org/?serve
-md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
-pre-configure =
-  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
-  grep -q "$@"
-  sed -i "/$1/d" "$2"
+[mariadb-10.3]
+<= mariadb-10.4
+version = 10.3.26
+md5sum = 7a3eda171db192c15ec31ac2520551ad
 post-install =
   ldd=`ldd ${:location}/lib/plugin/ha_rocksdb.so`
   for x in ${lz4:location} ${snappy:location} ${zstd:location}
@@ -145,14 +141,6 @@ post-install =
   mkdir -p ${:location}/include/wsrep &&
   cp -p wsrep/wsrep_api.h ${:location}/include/wsrep
 
-[mroonga-mariadb]
-pre-configure =
-  rm -rf fake_mariadb_source
-  mkdir -p fake_mariadb_source
-  cd fake_mariadb_source
-  ln -s ${mariadb:location}/wsrep-lib
-  cp -a ${mariadb:location}/include/mysql/server include
-  cp -a include/private sql
-  chmod -R a+w include sql # so that buildout can delete this compile-dir after install
-  mkdir -p ${:plugin-dir}
+[mariadb]
+location = ${mariadb-10.3:location}
 ### /XXX keep using mariadb 10.3.22 because 10.4 cause crash

component/proftpd/buildout.cfg

@@ -33,6 +33,7 @@ configure-options =
   --with-modules=mod_sftp:mod_ban
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
 environment =
+  CFLAGS=-DPR_RUN_DIR=\"/proc/self/cwd/var\"
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
   install_user=${proftpd-environment:USER}

component/rust/buildout.cfg

 [buildout]
-parts =
-  rustc
-
 extends =
-    ../cmake/buildout.cfg
-    ../curl/buildout.cfg
-    ../git/buildout.cfg
+    ../llvm/buildout.cfg
     ../openssl/buildout.cfg
     ../pkgconfig/buildout.cfg
 
+parts = rustc
 
 [rustc]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://static.rust-lang.org/dist/rustc-1.45.2-src.tar.gz
-md5sum = 1c67d7c3f211e49e12e7c20abab08e70
+url = https://static.rust-lang.org/dist/rustc-1.47.0-src.tar.gz
+md5sum = a460bed79b92f6a7833ba6e6390ee6ae
 # --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
-configure-options = --enable-extended --sysconfdir=@@LOCATION@@/etc/
+configure-options =
+    --enable-extended
+    --llvm-config=${llvm:location}/bin/llvm-config
+    --sysconfdir=@@LOCATION@@/etc
 environment =
-    PATH=${cmake:location}/bin/:${curl:location}/bin/:${git:location}/bin/:${pkgconfig:location}/bin/:%(PATH)s
-    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:
+    PATH=${curl:location}/bin/:${git:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+    PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
+    RUSTFLAGS=-C link-arg=-Wl,-rpath=${openssl:location}/lib -C link-arg=-L${libxml2:location}/lib -C link-arg=-L${ncurses:location}/lib -C link-arg=-L${zlib:location}/lib -C link-arg=-Wl,-rpath=${libxml2:location}/lib -C link-arg=-Wl,-rpath=${ncurses:location}/lib -C link-arg=-Wl,-rpath=${zlib:location}/lib

component/snappy/buildout.cfg

@@ -9,10 +9,8 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/google/snappy/archive/1.1.8.tar.gz
 md5sum = 70e48cba7fecf289153d009791c9977f
-location = @@LOCATION@@
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
-  -DCMAKE_INSTALL_PREFIX=${:location}
+  -DCMAKE_BUILD_TYPE=Release
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DBUILD_SHARED_LIBS=ON
-environment =
-  CMAKE_PROGRAM_PATH=${cmake:location}/bin

component/srslte/buildout.cfg

@@ -15,28 +15,22 @@ parts =
 recipe = slapos.recipe.cmmi
 url = https://github.com/srsLTE/srsLTE/archive/release_18_09.zip
 md5sum = d9007ff9bd03ba67c6ed8acfd9bf2a7a 
-pre-configure = 
-  mkdir -p build
-configure-command = cd build && ${cmake:location}/bin/cmake ../  
-
-make-binary =
-make-targets= cd build &&
-    make DESTDIR=@@LOCATION@@ -j1 install
-
-
+configure-command = cmake
 configure-options =
+  -Bbuild
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
   -DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
   -DCMAKE_INCLUDE_PATH=${mbedtls:location}/include
   -DCMAKE_LIBRARY_PATH=${mbedtls:location}/lib
+make-options = -C build
 
 CMAKE_CFLAGS = -I${bzip2:location}/include -I${fftw3:location}/include -I${mbedtls:location}/include -I${lksctp-tools:location}/include -I${libconfig:location}/include -I${boost-lib:location}/include 
 CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${mbedtls:location}/lib:${lksctp-tools:location}/lib:${libconfig:location}/lib:${boost-lib:location}/lib:${fftw3:location}/lib
 
 environment =
   CMAKE_INSTALL_PREFIX=${buildout:directory}/bin
-  CMAKE_PROGRAM_PATH=${cmake:location}/bin
   CMAKE_INCLUDE_PATH=${bzip2:location}/include:${mbedtls:location}/include:${lksctp-tools:location}/include:${libconfig:location}/include:${boost-lib:location}/include:${fftw3:location}/include
   CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
   CPPFLAGS=${:CMAKE_CFLAGS}

component/tomcat/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 <= tomcat9
 
 [tomcat7]
-recipe = hexagonit.recipe.download
+recipe = slapos.recipe.build:download-unpacked
 ignore-existing = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-7/v${:version}/bin/apache-tomcat-${:version}.tar.gz
@@ -18,7 +18,7 @@ version = 7.0.100
 md5sum = 79be4ba5a6e770730a4be3d5cb3c7862
 
 [tomcat9]
-recipe = hexagonit.recipe.download
+recipe = slapos.recipe.build:download-unpacked
 ignore-existing = true
 strip-top-level-dir = true
 url = https://archive.apache.org/dist/tomcat/tomcat-9/v${:version}/bin/apache-tomcat-${:version}.tar.gz

component/yaml-cpp/buildout.cfg

@@ -10,11 +10,9 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.3.tar.gz
 md5sum = b45bf1089a382e81f6b661062c10d0c2
-location = @@LOCATION@@
-configure-command =
-  mkdir build && cd build && \
-  ${cmake:location}/bin/cmake \
-    -DYAML_BUILD_SHARED_LIBS=ON \
-    -DCMAKE_INSTALL_PREFIX=${:location} \
-    ..
+configure-command = ${cmake:location}/bin/cmake
+configure-options =
+    -Bbuild
+    -DYAML_BUILD_SHARED_LIBS=ON
+    -DCMAKE_INSTALL_PREFIX=@@LOCATION@@
 make-options = -C build

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.152'
+version = '1.0.167'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/erp5testnode/__init__.py

@@ -58,7 +58,6 @@ class Recipe(GenericBaseRecipe):
           "\npath_list = %s" % ",".join(software_path_list) 
     CONFIG['computer_id'] = self.buildout['slap-connection']['computer-id']
     CONFIG['server_url'] = self.buildout['slap-connection']['server-url']
-    CONFIG['frontend_url'] = self.buildout['testnode-frontend']['connection-secure_access']
     configuration_file = self.createFile(
       self.options['configuration-file'],
       self.substituteTemplate(

slapos/recipe/erp5testnode/template/erp5testnode.cfg.in

@@ -20,6 +20,7 @@ httpd_software_access_port = %(httpd_software_access_port)s
 computer_id = %(computer_id)s
 server_url = %(server_url)s
 frontend_url = %(frontend_url)s
+log_frontend_url = %(log_frontend_url)s
 keep_log_days = %(keep_log_days)s
 
 # Binaries

slapos/recipe/switch_softwaretype.py

@@ -25,8 +25,45 @@
 #
 ##############################################################################
 
-import os, subprocess, sys
-import six
+
+from zc.buildout.buildout import Buildout
+
+
+class SubBuildout(Buildout):
+  """Run buildout in buildout, partially copied from infrae.buildout
+  """
+  def __init__(self, main_buildout, config, options, **kwargs):
+    # Use same logger
+    self._logger = main_buildout._logger
+    self._log_level = main_buildout._log_level
+
+    # Use same options
+    for opt in (
+        'offline',
+        'verbosity',
+        'newest',
+        'directory',
+        'eggs-directory',
+        'develop-eggs-directory',
+    ):
+      if opt in main_buildout['buildout']:
+        options.append((
+            'buildout',
+            opt,
+            main_buildout['buildout'][opt],
+        ))
+    # Use same slap connection
+    for k, v in main_buildout["slap-connection"].items():
+      options.append(('slap-connection', k, v))
+
+    Buildout.__init__(self, config, options, **kwargs)
+
+  def _setup_logging(self):
+    """We don't want to setup any logging, since it's already done
+    by the main buildout.
+    """
+    pass
+
 
 class Recipe:
 
@@ -39,26 +76,23 @@ class Recipe:
     self.base = self.buildout[section][key]
 
   def install(self):
-    # XXX-Antoine: We gotta find a better way to do this. I tried to check
-    # out how slapgrid-cp was running buildout. But it is worse than that.
-    args = sys.argv[:]
-    for x in six.iteritems(self.buildout["slap-connection"]):
-      args.append("slap-connection:%s=%s" % x)
-    for x in "directory", "eggs-directory", "develop-eggs-directory":
-      args.append("buildout:%s=%s" % (x, self.buildout["buildout"][x]))
-    args.append("buildout:installed=.installed-%s.cfg" % self.name)
-    # Options.get (from zc.buildout) should deserialize.
+    options = [("buildout", "installed", ".installed-%s.cfg" % self.name)]
+    profile = self.base
     try:
-      override = self.options["override"][self.software_type]
+      # XXX this assume using slapos.buildout, which serializes arbitrary python objects for options
+      extended_profile = self.options["override"][self.software_type]
     except (KeyError, TypeError):
-      buildout = self.base
+      pass
     else:
-      # unfortunately, buildout:extends does not work when given at command line
-      buildout = os.path.join(self.buildout["buildout"]["parts-directory"],
-                              self.name + ".cfg")
-      with open(override) as src, open(buildout, "w", 0) as dst:
-        dst.write("[buildout]\nextends = %s\n\n" % self.base + src.read())
-    subprocess.check_call(args + ["-oc", buildout])
-    return []
+      options.append(["buildout", "extends", profile])
+      profile = extended_profile
+
+    sub_buildout = SubBuildout(
+        self.buildout,
+        profile,
+        options,
+    )
+
+    sub_buildout.install([])
 
   update = install

software/caddy-frontend/README.caddy_frontend.rst

@@ -448,6 +448,15 @@ This allows backends to:
 Technical notes
 ===============
 
+Profile development guidelines
+------------------------------
+
+Keep the naming in instance profiles:
+
+ * ``software_parameter_dict`` for values coming from software
+ * ``instance_parameter_dict`` for **local** values generated by the instance, except ``configuration``
+ * ``slapparameter_dict`` for values coming from SlapOS Master
+
 Instantiated cluster structure
 ------------------------------
 

software/caddy-frontend/buildout.hash.cfg

@@ -14,36 +14,32 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = bfb647325103640c19c38e7b7f6e2833
+md5sum = 28bf0c4c75c028bed79fc38786831b3e
 
-[template-common]
+[profile-common]
 filename = instance-common.cfg.in
 md5sum = 5784bea3bd608913769ff9a8afcccb68
 
-[template-apache-frontend]
+[profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 02aac183352a6fd6ddd336d2e3757405
+md5sum = e7d7e1448b6420657e953026573311ca
 
-[template-caddy-replicate]
+[profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 54c0648c8593699dae0c565bc7dd8629
+md5sum = b70f9ce80dd927ead51b4526997b75ed
 
-[template-slave-list]
+[profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 2690fb2b5b6cefb7de53f35c214bdd52
+md5sum = ab143bfa2e20725aa35940c9033fa0ee
 
-[template-replicate-publish-slave-information]
+[profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
-md5sum = 7e3ee70c447f8203273d78f66ab519c3
+md5sum = de268251dafa5ad83ebf5b20636365d9
 
-[template-caddy-frontend-configuration]
+[profile-caddy-frontend-configuration]
 _update_hash_filename_ = templates/Caddyfile.in
 md5sum = 2503056e35463e045db3329bb8b6fae8
 
-[caddy-backend-url-validator]
-filename = templates/caddy-backend-url-validator.in
-md5sum = 0979a03476e86bf038516c9565dadc17
-
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html
 md5sum = 88af61e7abbf30dc99a1a2526161128d
@@ -54,7 +50,7 @@ md5sum = 266f175dbdfc588af7a86b0b1884fe73
 
 [template-backend-haproxy-configuration]
 _update_hash_filename_ = templates/backend-haproxy.cfg.in
-md5sum = 80081a7ded0029bb24b4fe2d06b6ae95
+md5sum = bf40f8d0a049a8dd924ccc731956c87e
 
 [template-log-access]
 _update_hash_filename_ = templates/template-log-access.conf.in
@@ -112,13 +108,13 @@ md5sum = 8e1c6c06c09beb921965b3ce98c67c9e
 filename = caddyprofiledummy.py
 md5sum = 38792c2dceae38ab411592ec36fff6a8
 
-[template-kedifa]
+[profile-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = d76fe7bf062410eda7049446ed06a736
+md5sum = 3daebc4b37088fa01183a853920d4143
 
 [template-backend-haproxy-rsyslogd-conf]
 _update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
-md5sum = be899b04e1aa652ed510f20d4ea523dd
+md5sum = 3ec9e088817f6a0e3b3b71919590e6b3
 
 [template-slave-introspection-httpd-nginx]
 _update_hash_filename_ = templates/slave-introspection-httpd-nginx.conf.in

software/caddy-frontend/common.cfg

-[buildout]
-extends =
-  buildout.hash.cfg
-  ../../stack/slapos.cfg
-  ../../component/dash/buildout.cfg
-  ../../component/caddy/buildout.cfg
-  ../../component/gzip/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../component/trafficserver/buildout.cfg
-  ../../component/6tunnel/buildout.cfg
-  ../../component/xz-utils/buildout.cfg
-  ../../component/rsyslogd/buildout.cfg
-  ../../component/haproxy/buildout.cfg
-  ../../component/nginx/buildout.cfg
-  ../../component/numpy/buildout.cfg
-
-  ../../stack/caucase/buildout.cfg
-# Monitoring stack (keep on bottom)
-  ../../stack/monitor/buildout.cfg
-
-parts +=
-  caucase-eggs
-  template
-  template-caddy-frontend
-  template-caddy-replicate
-  caddy
-
-  logrotate
-  rdiff-backup
-  caddyprofiledeps
-
-  kedifa-develop
-  kedifa
-
-[kedifa-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/kedifa.git
-git-executable = ${git:location}/bin/git
-revision = d6bbd7db215e12871c1536f22a8fbf994227270c
-
-[kedifa-develop]
-recipe = zc.recipe.egg:develop
-setup = ${kedifa-repository:location}
-
-[kedifa]
-recipe = zc.recipe.egg
-eggs =
-  ${python-cryptography:egg}
-  kedifa
-
-[caddyprofiledeps-setup]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/setup.py
-
-[caddyprofiledeps-dummy]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/caddyprofiledummy.py
-
-[caddyprofiledeps-prepare]
-recipe = plone.recipe.command
-stop-on-error = True
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-update-command = ${:command}
-command =
-  rm -fr ${:location} &&
-  mkdir -p ${:location} &&
-  cp ${caddyprofiledeps-setup:target} ${:location}/ &&
-  cp ${caddyprofiledeps-dummy:target} ${:location}/
-
-[caddyprofiledeps-develop]
-recipe = zc.recipe.egg:develop
-setup = ${caddyprofiledeps-prepare:location}
-
-[caddyprofiledeps]
-depends = ${caddyprofiledeps-develop:recipe}
-recipe = zc.recipe.egg
-eggs =
-  caddyprofiledeps
-  websockify
-  collective.recipe.shelloutput
-
-[template-common]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-common.cfg.in
-rendered = ${buildout:directory}/instance-common.cfg
-mode = 0644
-context =
-  key develop_eggs_directory buildout:develop-eggs-directory
-  key eggs_directory buildout:eggs-directory
-
-[template-frontend-parameter-section]
-common_profile = ${template-common:rendered}
-logrotate_base_instance = ${template-logrotate-base:rendered}
-
-bin_directory = ${buildout:bin-directory}
-
-sixtunnel = ${6tunnel:location}
-nginx = ${nginx-output:nginx}
-nginx_mime = ${nginx-output:mime}
-caddy = ${caddy:output}
-caddy_location = ${caddy:location}
-haproxy_executable = ${haproxy:location}/sbin/haproxy
-rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
-curl = ${curl:location}
-dash = ${dash:location}
-gzip = ${gzip:location}
-logrotate = ${logrotate:location}
-openssl = ${openssl:location}/bin/openssl
-openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
-trafficserver = ${trafficserver:location}
-sha256sum = ${coreutils:location}/bin/sha256sum
-kedifa = ${:bin_directory}/kedifa
-kedifa-updater = ${:bin_directory}/kedifa-updater
-kedifa-csr = ${:bin_directory}/kedifa-csr
-xz_location = ${xz-utils:location}
-htpasswd = ${:bin_directory}/htpasswd
-
-monitor_template = ${monitor-template:output}
-template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
-template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
-template_caddy_frontend_configuration = ${template-caddy-frontend-configuration:target}
-template_graceful_script = ${template-graceful-script:target}
-template_validate_script = ${template-validate-script:target}
-template_rotate_script = ${template-rotate-script:target}
-template_configuration_state_script = ${template-configuration-state-script:target}
-template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
-template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
-template_empty = ${template-empty:target}
-template_log_access = ${template-log-access:target}
-template_not_found_html = ${template-not-found-html:target}
-template_slave_list = ${template-slave-list:target}
-template_trafficserver_records_config = ${template-trafficserver-records-config:target}
-template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
-template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
-template_wrapper = ${template-wrapper:output}
-template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
-
-[template]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance.cfg.in
-rendered = ${buildout:directory}/template.cfg
-mode = 0644
-context =
-  key common_profile template-common:rendered
-
-  key monitor2_template monitor2-template:rendered
-  key template_caddy_frontend template-caddy-frontend:target
-  key template_caddy_replicate template-caddy-replicate:target
-  key template_kedifa template-kedifa:target
-  key template_replicate_publish_slave_information template-replicate-publish-slave-information:target
-  key caddy_backend_url_validator caddy-backend-url-validator:output
-
-  section template_frontend_parameter_dict template-frontend-parameter-section
-  key caucase_jinja2_library caucase-jinja2-library:target
-
-[template-caddy-frontend]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
-mode = 0644
-
-[caddy-backend-url-validator]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/${:filename}
-output = ${buildout:directory}/caddy-backend-url-validator
-mode = 0750
-
-[template-caddy-replicate]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-mode = 0644
-
-[template-kedifa]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-kedifa.cfg.in
-mode = 0644
-
-[download-template]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-mode = 640
-
-[template-slave-list]
-<=download-template
-
-[template-replicate-publish-slave-information]
-<=download-template
-
-[template-caddy-frontend-configuration]
-<=download-template
-
-[template-not-found-html]
-<=download-template
-
-[template-default-slave-virtualhost]
-<=download-template
-
-[template-backend-haproxy-configuration]
-<=download-template
-
-[template-log-access]
-<=download-template
-
-[template-empty]
-<=download-template
-
-[template-slave-introspection-httpd-nginx]
-<=download-template
-
-[template-wrapper]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/templates/wrapper.in
-output = ${buildout:directory}/template-wrapper.cfg
-mode = 0644
-
-[template-trafficserver-records-config]
-<=download-template
-
-[template-trafficserver-storage-config]
-<=download-template
-
-[template-trafficserver-logging-yaml]
-<=download-template
-
-[template-rotate-script]
-<=download-template
-
-[template-caddy-lazy-script-call]
-<=download-template
-
-[template-graceful-script]
-<=download-template
-
-[template-validate-script]
-<=download-template
-
-[template-configuration-state-script]
-<=download-template
-
-[template-backend-haproxy-rsyslogd-conf]
-<=download-template

software/caddy-frontend/development.cfg

-# Development profile of caddy-frontend.
-# Exactly the same as software.cfg, but fetch the slapos.cookbook
-# from git repository instead of fetching stable version,
-# allowing to play with bleeding edge environment.
-
-# You'll need to run buildout twice for this profile.
-
-[buildout]
-extends =
-# Extend in this order, otherwise "parts" will be taken from git profile
-  common.cfg
-
-parts +=
-  slapos.toolbox-dev
-
-[slapos.toolbox-dev]
-recipe = zc.recipe.egg:develop
-egg = slapos.toolbox
-setup = ${slapos.toolbox-repository:location}

software/caddy-frontend/instance-apache-frontend.cfg.in

-{%- if slap_software_type == software_type -%}
+{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
 {% import "caucase" as caucase with context %}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 [buildout]
 extends =
-  {{ parameter_dict['common_profile'] }}
-  {{ parameter_dict['monitor_template'] }}
-  {{ parameter_dict['logrotate_base_instance'] }}
+  {{ software_parameter_dict['profile_common'] }}
+  {{ software_parameter_dict['profile_monitor'] }}
+  {{ software_parameter_dict['profile_logrotate_base'] }}
 
 parts =
   directory
@@ -98,20 +98,14 @@ slave-introspection-var = ${:var}/slave-introspection
 
 [switch-caddy-softwaretype]
 recipe = slapos.cookbook:softwaretype
-single-default = ${dynamic-custom-personal-template-slave-list:rendered}
-single-custom-personal = ${dynamic-custom-personal-template-slave-list:rendered}
+single-default = ${dynamic-custom-personal-profile-slave-list:rendered}
+single-custom-personal = ${dynamic-custom-personal-profile-slave-list:rendered}
 
 [frontend-configuration]
-template-log-access = {{ parameter_dict['template_log_access'] }}
 log-access-configuration = ${directory:etc}/log-access.conf
 ip-access-certificate = ${self-signed-ip-access:certificate}
-caddy-directory = {{ parameter_dict['caddy_location'] }}
-caddy-ipv6 = {{ instance_parameter['ipv6-random'] }}
+caddy-ipv6 = {{ instance_parameter_dict['ipv6-random'] }}
 caddy-https-port = ${configuration:port}
-nginx = {{ parameter_dict['nginx'] }}
-nginx_mime = {{ parameter_dict['nginx_mime'] }}
-htpasswd = {{ parameter_dict['htpasswd'] }}
-slave-introspection-template = {{ parameter_dict['template_slave_introspection_httpd_nginx'] }}
 slave-introspection-configuration = ${directory:etc}/slave-introspection-httpd-nginx.conf
 slave-introspection-https-port = ${configuration:slave-introspection-https-port}
 slave-introspection-secure_access = ${slave-introspection-frontend:connection-secure_access}
@@ -122,21 +116,22 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
 recipe = plone.recipe.command
 update-command = ${:command}
 ipv6 = ${slap-network-information:global-ipv6}
-ipv4 = {{instance_parameter['ipv4-random']}}
+ipv4 = {{instance_parameter_dict['ipv4-random']}}
 certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 command =
   [ -f ${:certificate} ] && exit 0
   rm -f ${:certificate}
   /bin/bash -c ' \
-  {{ parameter_dict['openssl'] }} req \
+  {{ software_parameter_dict['openssl'] }} req \
      -new -newkey rsa:2048 -sha256 \
      -nodes -x509 -days 36500 \
      -keyout ${:certificate} \
      -subj "/CN=Self Signed IP Access" \
      -reqexts SAN \
      -extensions SAN \
-     -config <(cat {{ parameter_dict['openssl_cnf'] }} \
+     -config <(cat {{ software_parameter_dict['openssl_cnf'] }} \
          <(printf "\n[SAN]\nsubjectAltName=IP:${:ipv6},IP:${:ipv4}")) \
      -out ${:certificate}'
 
@@ -145,18 +140,19 @@ command =
 recipe = plone.recipe.command
 update-command = ${:command}
 ipv6 = ${slap-network-information:global-ipv6}
-ipv4 = {{instance_parameter['ipv4-random']}}
+ipv4 = {{instance_parameter_dict['ipv4-random']}}
 certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 command =
   [ -f ${:certificate} ] && exit 0
   rm -f ${:certificate}
   /bin/bash -c ' \
-  {{ parameter_dict['openssl'] }} req \
+  {{ software_parameter_dict['openssl'] }} req \
      -new -newkey rsa:2048 -sha256 \
      -nodes -x509 -days 36500 \
      -keyout ${:certificate} \
-     -subj "/CN=Fallback certificate/OU={{ instance_parameter['configuration.frontend-name'] }}" \
+     -subj "/CN=Fallback certificate/OU={{ instance_parameter_dict['configuration.frontend-name'] }}" \
      -out ${:certificate}'
 
 [jinja2-template-base]
@@ -164,24 +160,23 @@ recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:filename}
 extensions = jinja2.ext.do
 extra-context =
-slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
-slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
+slapparameter_dict = {{ dumps(slapparameter_dict) }}
+slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
 context =
     import json_module json
-    raw common_profile {{ parameter_dict['common_profile'] }}
-    raw logrotate_base_instance {{ parameter_dict['logrotate_base_instance'] }}
-    raw monitor_template {{ parameter_dict['monitor_template'] }}
+    raw profile_common {{ software_parameter_dict['profile_common'] }}
+    raw profile_logrotate_base {{ software_parameter_dict['profile_logrotate_base'] }}
+    raw profile_monitor {{ software_parameter_dict['profile_monitor'] }}
     key slap_software_type :slap_software_type
     key slapparameter_dict :slapparameter_dict
     section directory directory
     ${:extra-context}
 
 [software-release-path]
-template-empty = {{ parameter_dict['template_empty'] }}
-template-default-slave-virtualhost = {{ parameter_dict['template_default_slave_virtualhost'] }}
-template-backend-haproxy-configuration = {{ parameter_dict['template_backend_haproxy_configuration'] }}
-template-backend-haproxy-rsyslogd-conf = {{ parameter_dict['template_backend_haproxy_rsyslogd_conf'] }}
-caddy-location = {{ parameter_dict['caddy_location'] }}
+template-empty = {{ software_parameter_dict['template_empty'] }}
+template-default-slave-virtualhost = {{ software_parameter_dict['template_default_slave_virtualhost'] }}
+template-backend-haproxy-configuration = {{ software_parameter_dict['template_backend_haproxy_configuration'] }}
+template-backend-haproxy-rsyslogd-conf = {{ software_parameter_dict['template_backend_haproxy_rsyslogd_conf'] }}
 
 [kedifa-login-config]
 d = ${directory:ca-dir}
@@ -195,11 +190,11 @@ crl = ${:d}/kedifa-login-crl.pem
 [kedifa-login-csr]
 recipe = plone.recipe.command
 organization = {{ slapparameter_dict['cluster-identification'] }}
-organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
+organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
 command =
 {% if slapparameter_dict['kedifa-caucase-url'] %}
   if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ]  ; then
-    {{ parameter_dict['openssl'] }} req -new -sha256 \
+    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
       -newkey rsa:2048 -nodes -keyout ${:key} \
       -subj "/O=${:organization}/OU=${:organizational_unit}" \
       -out ${:template-csr}
@@ -209,11 +204,12 @@ command =
 update-command = ${:command}
 template-csr = ${kedifa-login-config:template-csr}
 key = ${kedifa-login-config:key}
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 
 {{ caucase.updater(
      prefix='caucase-updater',
-     buildout_bin_directory=parameter_dict['bin_directory'],
+     buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/kedifa-login-certificate-caucase-updater',
      url=slapparameter_dict['kedifa-caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
@@ -231,7 +227,6 @@ certificate = ${kedifa-login-config:certificate}
 cas-ca-certificate = ${kedifa-login-config:cas-ca-certificate}
 csr = ${caucase-updater-csr:csr}
 crl = ${kedifa-login-config:crl}
-kedifa-updater = {{ parameter_dict['kedifa-updater'] }}
 kedifa-updater-mapping-file = ${directory:etc}/kedifa_updater_mapping.txt
 kedifa-updater-state-file = ${directory:srv}/kedifa_updater_state.json
 slave_kedifa_information = {{ dumps(slapparameter_dict['slave-kedifa-information']) }}
@@ -248,11 +243,11 @@ crl = ${:d}/crl.pem
 [backend-client-login-csr]
 recipe = plone.recipe.command
 organization = {{ slapparameter_dict['cluster-identification'] }}
-organizational_unit = {{ instance_parameter['configuration.frontend-name'] }}
+organizational_unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
 command =
 {% if slapparameter_dict['backend-client-caucase-url'] %}
   if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ]  ; then
-    {{ parameter_dict['openssl'] }} req -new -sha256 \
+    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
       -newkey rsa:2048 -nodes -keyout ${:key} \
       -subj "/O=${:organization}/OU=${:organizational_unit}" \
       -out ${:template-csr}
@@ -262,11 +257,12 @@ command =
 update-command = ${:command}
 template-csr = ${backend-client-login-config:template-csr}
 key = ${backend-client-login-config:key}
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 
 {{ caucase.updater(
      prefix='backend-client-caucase-updater',
-     buildout_bin_directory=parameter_dict['bin_directory'],
+     buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/backend-client-login-certificate-caucase-updater',
      url=slapparameter_dict['backend-client-caucase-url'],
      data_dir='${directory:srv}/backend-client-caucase-updater',
@@ -277,79 +273,54 @@ stop-on-error = True
      template_csr='${backend-client-login-csr:template-csr}'
 )}}
 
-[dynamic-custom-personal-template-slave-list]
+[dynamic-custom-personal-profile-slave-list]
 < = jinja2-template-base
 depends = ${caddyprofiledeps:recipe}
-template = {{ parameter_dict['template_slave_list'] }}
+template = {{ software_parameter_dict['profile_slave_list'] }}
 filename = custom-personal-instance-slave-list.cfg
-slave_instance_list = {{ dumps(instance_parameter['slave-instance-list']) }}
-extra_slave_instance_list = {{ dumps(instance_parameter.get('configuration.extra_slave_instance_list')) }}
 master_key_download_url = {{ dumps(slapparameter_dict['master-key-download-url']) }}
-local_ipv4 = {{ dumps(instance_parameter['ipv4-random']) }}
-local_ipv6 = {{ dumps(instance_parameter['ipv6-random']) }}
 software_type = single-custom-personal
-bin_directory = {{ parameter_dict['bin_directory'] }}
-caddy_executable = {{ parameter_dict['caddy'] }}
-sixtunnel_executable = {{ parameter_dict['sixtunnel'] }}/bin/6tunnel
 organization = {{ slapparameter_dict['cluster-identification'] }}
-organizational-unit = {{ instance_parameter['configuration.frontend-name'] }}
+organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
 backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
 extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
     key backend_client_caucase_url :backend-client-caucase-url
     import urlparse_module urlparse
     import furl_module furl
-    key caddy_executable :caddy_executable
-    key http_port configuration:plain_http_port
-    key https_port configuration:port
-    key public_ipv4 configuration:public-ipv4
-    key slave_instance_list :slave_instance_list
-    key extra_slave_instance_list :extra_slave_instance_list
     key master_key_download_url :master_key_download_url
     key autocert caddy-directory:autocert
-    key master_certificate caddy-configuration:master-certificate
     key caddy_log_directory caddy-directory:slave-log
     key expose_csr_id_organization :organization
     key expose_csr_id_organizational_unit :organizational-unit
-    key local_ipv4 :local_ipv4
-    key local_ipv6 :local_ipv6
     key global_ipv6 slap-network-information:global-ipv6
     key empty_template software-release-path:template-empty
     key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
     key software_type :software_type
     key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
-    key frontend_graceful_reload caddy-configuration:frontend-graceful-command
-    section frontend_configuration frontend-configuration
-    section caddy_configuration caddy-configuration
     key monitor_base_url monitor-instance-parameter:monitor-base-url 
-    key bin_directory :bin_directory
-    key enable_http2_by_default configuration:enable-http2-by-default
-    key global_disable_http2 configuration:global-disable-http2
-    key ciphers configuration:ciphers
-    key access_log caddy-configuration:access-log
-    key error_log caddy-configuration:error-log
-    key sixtunnel_executable :sixtunnel_executable
-    key not_found_file caddy-configuration:not-found-file
     key custom_ssl_directory caddy-directory:custom-ssl-directory
-    section kedifa_configuration kedifa-configuration
 # BBB: SlapOS Master non-zero knowledge BEGIN
     key apache_certificate apache-certificate:rendered
 # BBB: SlapOS Master non-zero knowledge END
 ## backend haproxy
     key template_backend_haproxy_configuration software-release-path:template-backend-haproxy-configuration
-    section backend_haproxy_configuration backend-haproxy-configuration
-## full configuration
+## Configuration passed by section
     section configuration configuration
+    section backend_haproxy_configuration backend-haproxy-configuration
+    section instance_parameter_dict instance-parameter-section
+    section frontend_configuration frontend-configuration
+    section caddy_configuration caddy-configuration
+    section kedifa_configuration kedifa-configuration
+    section software_parameter_dict software-parameter-section
 
 # Deploy Caddy Frontend with Jinja power
 [dynamic-caddy-frontend-template]
 < = jinja2-template-base
-template = {{ parameter_dict['template_caddy_frontend_configuration'] }}
+template = {{ software_parameter_dict['template_caddy_frontend_configuration'] }}
 rendered = ${caddy-configuration:frontend-configuration}
-local_ipv4 =  {{ dumps(instance_parameter['ipv4-random']) }}
+local_ipv4 =  {{ dumps(instance_parameter_dict['ipv4-random']) }}
 extra-context =
-    key httpd_home software-release-path:caddy-location
-    key httpd_mod_ssl_cache_directory caddy-directory:mod-ssl
     key instance_home buildout:directory
     key master_certificate caddy-configuration:master-certificate
     key access_log caddy-configuration:access-log
@@ -373,16 +344,16 @@ template = inline:
   #!/bin/sh
   export CADDYPATH=${directory:frontend_cluster}
   ulimit -n $(ulimit -Hn)
-  exec {{ parameter_dict['caddy'] }} \
+  exec {{ software_parameter_dict['caddy'] }} \
   -conf ${dynamic-caddy-frontend-template:rendered} \
   -log ${caddy-configuration:error-log} \
   -log-roll-mb 0 \
-{% if instance_parameter['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
+{% if instance_parameter_dict['configuration.global-disable-http2'].lower() in TRUE_VALUES %}
   -http2=false \
 {% else %}
   -http2=true \
 {% endif %}
-  -grace {{ instance_parameter['configuration.mpm-graceful-shutdown-timeout'] }}s \
+  -grace {{ instance_parameter_dict['configuration.mpm-graceful-shutdown-timeout'] }}s \
   -disable-http-challenge \
   -disable-tls-alpn-challenge \
    "$@"
@@ -400,14 +371,12 @@ hash-files = ${caddy-wrapper:rendered}
 recipe = plone.recipe.command
 update-command = ${:command}
 filename = notfound.html
-command = ln -sf  {{ parameter_dict['template_not_found_html'] }} ${caddy-directory:document-root}/${:filename}
+command = ln -sf  {{ software_parameter_dict['template_not_found_html'] }} ${caddy-directory:document-root}/${:filename}
 
 [caddy-directory]
 recipe = slapos.cookbook:mkdirectory
 document-root = ${directory:srv}/htdocs
 slave-configuration = ${directory:etc}/caddy-slave-conf.d/
-cache = ${directory:var}/cache
-mod-ssl = ${:cache}/httpd_mod_ssl
 slave-log = ${directory:log}/httpd
 autocert = ${directory:srv}/autocert
 master-autocert-dir = ${:autocert}/master-autocert
@@ -469,7 +438,7 @@ delaycompress =
 recipe = slapos.cookbook:mkdirectory
 configuration = ${directory:etc}/trafficserver
 local-state = ${directory:var}/trafficserver
-bin_path = {{ parameter_dict['trafficserver'] }}/bin
+bin_path = {{ software_parameter_dict['trafficserver'] }}/bin
 log = ${directory:log}/trafficserver
 cache-path = ${directory:srv}/ats_cache
 logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
@@ -477,7 +446,7 @@ logrotate-backup = ${logrotate-directory:logrotate-backup}/trafficserver
 [trafficserver-variable]
 wrapper-path = ${directory:service}/trafficserver
 reload-path = ${directory:etc-run}/trafficserver-reload
-local-ip = {{ instance_parameter['ipv4-random'] }}
+local-ip = {{ instance_parameter_dict['ipv4-random'] }}
 input-port = 23432
 hostname = ${configuration:frontend-name}
 plugin-config =
@@ -485,24 +454,24 @@ ip-allow-config = src_ip=0.0.0.0-255.255.255.255 action=ip_allow
 cache-path = ${trafficserver-directory:cache-path}
 disk-cache-size = ${configuration:disk-cache-size}
 ram-cache-size = ${configuration:ram-cache-size}
-templates-dir = {{ parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
+templates-dir = {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
 request-timeout = ${configuration:request-timeout}
 
 [trafficserver-configuration-directory]
 recipe = plone.recipe.command
-command = cp -rn {{ parameter_dict['trafficserver'] }}/etc/trafficserver/* ${:target}
+command = cp -rn {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/* ${:target}
 target = ${trafficserver-directory:configuration}
 
 [trafficserver-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_manager
+command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_manager
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [trafficserver-reload]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
+command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_ctl config reload
 wrapper-path = ${trafficserver-variable:reload-path}
 environment = TS_ROOT=${buildout:directory}
 
@@ -519,19 +488,19 @@ context =
 
 [trafficserver-records-config]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_trafficserver_records_config'] }}
+template = {{ software_parameter_dict['template_trafficserver_records_config'] }}
 filename = records.config
 extra-context =
     import os_module os
 
 [trafficserver-storage-config]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_trafficserver_storage_config'] }}
+template = {{ software_parameter_dict['template_trafficserver_storage_config'] }}
 filename = storage.config
 
 [trafficserver-logging-yaml]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_trafficserver_logging_yaml'] }}
+template = {{ software_parameter_dict['template_trafficserver_logging_yaml'] }}
 filename = logging.yaml
 
 [trafficserver-remap-config]
@@ -542,7 +511,7 @@ template = inline:
   map / http://{{ ipv4 }}:{{ http_port }}
 {%- endraw %}
 extra-context =
-  raw ipv4 {{ instance_parameter['ipv4-random'] }}
+  raw ipv4 {{ instance_parameter_dict['ipv4-random'] }}
   key https_port backend-haproxy-configuration:https-port
   key http_port backend-haproxy-configuration:http-port
 
@@ -550,14 +519,14 @@ filename = remap.config
 
 [trafficserver-plugin-config]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 filename = plugin.config
 context =
     key content trafficserver-variable:plugin-config
 
 [trafficserver-ip-allow-config]
 < = trafficserver-jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 filename = ip_allow.config
 context =
     key content trafficserver-variable:ip-allow-config
@@ -571,7 +540,7 @@ config-port = ${trafficserver-variable:input-port}
 
 [trafficserver-ctl]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_ctl
+command-line = {{ software_parameter_dict['trafficserver'] }}/bin/traffic_ctl
 wrapper-path = ${directory:bin}/traffic_ctl
 environment = TS_ROOT=${buildout:directory}
 
@@ -583,10 +552,10 @@ config-wrapper-path = ${trafficserver-ctl:wrapper-path}
 
 [trafficserver-rotate-script]
 < = jinja2-template-base
-template = {{ parameter_dict['template_rotate_script'] }}
+template = {{ software_parameter_dict['template_rotate_script'] }}
 rendered = ${directory:bin}/trafficserver-rotate
 mode = 0700
-xz_binary = {{ parameter_dict['xz_location'] ~ '/bin/xz' }}
+xz_binary = {{ software_parameter_dict['xz_location'] ~ '/bin/xz' }}
 pattern = *.old
 # days to keep log files
 keep_days = 365
@@ -610,12 +579,12 @@ command = ${trafficserver-rotate-script:rendered}
 ### Caddy Graceful and promises
 [frontend-caddy-configuration-state]
 < = jinja2-template-base
-template = {{ parameter_dict['template_configuration_state_script'] }}
+template = {{ software_parameter_dict['template_configuration_state_script'] }}
 rendered = ${directory:bin}/${:_buildout_section_name_}
 mode = 0700
 
 path_list = ${caddy-configuration:frontend-configuration} ${frontend-configuration:log-access-configuration} ${caddy-directory:slave-configuration}/*.conf ${caddy-directory:master-autocert-dir}/*.key ${caddy-directory:master-autocert-dir}/*.crt ${caddy-directory:master-autocert-dir}/*.pem ${caddy-directory:autocert}/*.pem ${caddy-directory:custom-ssl-directory}/*.proxy_ca_crt ${directory:bbb-ssl-dir}/*.crt
-sha256sum = {{ parameter_dict['sha256sum'] }}
+sha256sum = {{ software_parameter_dict['sha256sum'] }}
 
 extra-context =
     key path_list :path_list
@@ -632,7 +601,7 @@ signature_file = ${directory:run}/validate_configuration_state_signature
 
 [frontend-caddy-graceful]
 < = jinja2-template-base
-template = {{ parameter_dict['template_graceful_script'] }}
+template = {{ software_parameter_dict['template_graceful_script'] }}
 rendered = ${directory:etc-run}/frontend-caddy-safe-graceful
 mode = 0700
 
@@ -642,7 +611,7 @@ extra-context =
 
 [frontend-caddy-validate]
 < = jinja2-template-base
-template = {{ parameter_dict['template_validate_script'] }}
+template = {{ software_parameter_dict['template_validate_script'] }}
 rendered = ${directory:bin}/frontend-caddy-validate
 mode = 0700
 last_state_file = ${directory:run}/caddy_configuration_last_state
@@ -654,7 +623,7 @@ extra-context =
 
 [frontend-caddy-lazy-graceful]
 < = jinja2-template-base
-template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
+template = {{ software_parameter_dict['template_caddy_lazy_script_call'] }}
 rendered = ${directory:bin}/frontend-caddy-lazy-graceful
 mode = 0700
 pid-file = ${directory:run}/lazy-graceful.pid
@@ -667,7 +636,7 @@ extra-context =
 # Promises checking configuration:
 [promise-helper-last-configuration-state]
 < = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 rendered = ${directory:bin}/frontend-read-last-configuration-state
 mode = 0700
 content =
@@ -686,42 +655,42 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
 <= monitor-promise-base
 module = check_port_listening
 name = caddy_frontend_ipv4_https.py
-config-hostname = {{ instance_parameter['ipv4-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v4-http]
 <= monitor-promise-base
 module = check_port_listening
 name = caddy_frontend_ipv4_http.py
-config-hostname = {{ instance_parameter['ipv4-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-caddy-frontend-v6-https]
 <= monitor-promise-base
 module = check_port_listening
 name = caddy_frontend_ipv6_https.py
-config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:port}
 
 [promise-caddy-frontend-v6-http]
 <= monitor-promise-base
 module = check_port_listening
 name = caddy_frontend_ipv6_http.py
-config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${configuration:plain_http_port}
 
 [promise-backend-haproxy-http]
 <= monitor-promise-base
 module = check_port_listening
 name = backend_haproxy_http.py
-config-hostname = {{ instance_parameter['ipv4-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:http-port}
 
 [promise-backend-haproxy-https]
 <= monitor-promise-base
 module = check_port_listening
 name = backend_haproxy_https.py
-config-hostname = {{ instance_parameter['ipv4-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
 config-port = ${backend-haproxy-configuration:https-port}
 
 [backend-haproxy-configuration]
@@ -748,13 +717,13 @@ statistic-frontend-secure_access = ${backend-haproxy-statistic-frontend:connecti
 
 [backend-haproxy]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file}
+command-line = {{ software_parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file}
 wrapper-path = ${directory:service}/backend-haproxy
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [backend-haproxy-rsyslogd-lazy-graceful]
 < = jinja2-template-base
-template = {{ parameter_dict['template_caddy_lazy_script_call'] }}
+template = {{ software_parameter_dict['template_caddy_lazy_script_call'] }}
 rendered = ${directory:bin}/backend-haproxy-rsyslogd-lazy-graceful
 mode = 0700
 pid-file = ${directory:run}/backend-haproxy-rsyslogd-lazy-graceful.pid
@@ -779,12 +748,12 @@ delaycompress =
 
 [backend-haproxy-configuration-state]
 <= jinja2-template-base
-template = {{ parameter_dict['template_configuration_state_script'] }}
+template = {{ software_parameter_dict['template_configuration_state_script'] }}
 rendered = ${directory:bin}/${:_buildout_section_name_}
 mode = 0700
 
 path_list = ${backend-haproxy-configuration:file} ${backend-client-login-config:certificate}
-sha256sum = {{ parameter_dict['sha256sum'] }}
+sha256sum = {{ software_parameter_dict['sha256sum'] }}
 
 extra-context =
     key path_list :path_list
@@ -801,7 +770,7 @@ signature_file = ${directory:run}/backend_haproxy_validate_configuration_state_s
 
 [backend-haproxy-graceful]
 < = jinja2-template-base
-template = {{ parameter_dict['template_graceful_script'] }}
+template = {{ software_parameter_dict['template_graceful_script'] }}
 rendered = ${directory:etc-run}/backend-haproxy-safe-graceful
 mode = 0700
 
@@ -811,11 +780,11 @@ extra-context =
 
 [backend-haproxy-validate]
 <= jinja2-template-base
-template = {{ parameter_dict['template_validate_script'] }}
+template = {{ software_parameter_dict['template_validate_script'] }}
 rendered = ${directory:bin}/backend-haproxy-validate
 mode = 0700
 last_state_file = ${directory:run}/backend_haproxy_configuration_last_state
-validate_command = {{ parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file} -c
+validate_command = {{ software_parameter_dict['haproxy_executable'] }} -f ${backend-haproxy-configuration:file} -c
 extra-context =
     key validate_command :validate_command
     key configuration_state_command backend-haproxy-configuration-state-validate:rendered
@@ -829,7 +798,7 @@ config-verification-script = ${promise-backend-haproxy-configuration-helper:rend
 
 [promise-backend-haproxy-configuration-helper]
 < = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 rendered = ${directory:bin}/backend-haproxy-read-last-configuration-state
 mode = 0700
 content =
@@ -855,7 +824,7 @@ extra-context =
 
 [backend-haproxy-rsyslogd]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['rsyslogd_executable'] }} -i ${backend-haproxy-rsyslogd-config:pid-file} -n -f ${backend-haproxy-rsyslogd-configuration:rendered}
+command-line = {{ software_parameter_dict['rsyslogd_executable'] }} -i ${backend-haproxy-rsyslogd-config:pid-file} -n -f ${backend-haproxy-rsyslogd-configuration:rendered}
 wrapper-path = ${directory:service}/backend-haproxy-rsyslogd
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
@@ -867,8 +836,8 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 # Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
 #       directly, and in our case it can come from the network, thus resulting
 #       with need to strip !py!'u'
-monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
-password = {{ instance_parameter['configuration.monitor-password'] | string }}
+monitor-httpd-port = {{ instance_parameter_dict['configuration.monitor-httpd-port'] | int }}
+password = {{ instance_parameter_dict['configuration.monitor-password'] | string }}
 
 [monitor-conf-parameters]
 private-path-list += 
@@ -877,35 +846,35 @@ private-path-list +=
 
 [monitor-traffic-summary-last-stats-wrapper]
 < = jinja2-template-base
-template = {{ parameter_dict['template_wrapper'] }}
+template = {{ software_parameter_dict['template_wrapper'] }}
 rendered = ${directory:bin}/traffic-summary-last-stats_every_1_hour
 mode = 0700
-command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ parameter_dict['trafficserver'] }}/bin/traffic_logstats -f ${trafficserver-directory:log}/squid.blog)</pre>"
+command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ software_parameter_dict['trafficserver'] }}/bin/traffic_logstats -f ${trafficserver-directory:log}/squid.blog)</pre>"
 extra-context =
   key content monitor-traffic-summary-last-stats-wrapper:command
 
 # Produce ATS Cache stats
 [monitor-ats-cache-stats-wrapper]
 < = jinja2-template-base
-template = {{ parameter_dict['template_wrapper'] }}
+template = {{ software_parameter_dict['template_wrapper'] }}
 rendered = ${directory:bin}/ats-cache-stats_every_1_hour
 mode = 0700
-command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ parameter_dict['trafficserver'] }}/bin/traffic_shell ${monitor-ats-cache-stats-config:rendered})</pre>"
+command = export TS_ROOT=${buildout:directory} && echo "<pre>$({{ software_parameter_dict['trafficserver'] }}/bin/traffic_shell ${monitor-ats-cache-stats-config:rendered})</pre>"
 extra-context =
   key content monitor-ats-cache-stats-wrapper:command
 
 [monitor-caddy-server-status-wrapper]
 < = jinja2-template-base
-template = {{ parameter_dict['template_wrapper'] }}
+template = {{ software_parameter_dict['template_wrapper'] }}
 rendered = ${directory:bin}/monitor-caddy-server-status-wrapper
 mode = 0700
-command = {{ parameter_dict['curl'] }}/bin/curl -s http://{{ instance_parameter['ipv4-random'] }}:${configuration:plain_http_port}/server-status -u ${monitor-instance-parameter:username}:${monitor-htpasswd:passwd} 2>&1
+command = {{ software_parameter_dict['curl'] }}/bin/curl -s http://{{ instance_parameter_dict['ipv4-random'] }}:${configuration:plain_http_port}/server-status -u ${monitor-instance-parameter:username}:${monitor-htpasswd:passwd} 2>&1
 extra-context =
   key content monitor-caddy-server-status-wrapper:command
 
 [monitor-ats-cache-stats-config]
 < = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 rendered = ${trafficserver-configuration-directory:target}/cache-config.stats
 mode = 644
 context =
@@ -930,31 +899,31 @@ extra-context =
 [slave-introspection-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = Slave Introspection Frontend {{ instance_parameter['configuration.frontend-name'] }}
+name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.slave-introspection-https-port'] }}/
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
 config-https-only = true
 return = domain secure_access
 
 [backend-haproxy-statistic-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = Backend Haproxy Statistic Frontend {{ instance_parameter['configuration.frontend-name'] }}
+name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter['configuration.backend-haproxy-statistic-port'] }}/
+config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
 config-https-only = true
 return = domain secure_access
 
 [slave-introspection-configuration-state]
 <= jinja2-template-base
-template = {{ parameter_dict['template_configuration_state_script'] }}
+template = {{ software_parameter_dict['template_configuration_state_script'] }}
 rendered = ${directory:bin}/${:_buildout_section_name_}
 mode = 0700
 
 path_list = ${frontend-configuration:slave-introspection-configuration} ${frontend-configuration:ip-access-certificate}
-sha256sum = {{ parameter_dict['sha256sum'] }}
+sha256sum = {{ software_parameter_dict['sha256sum'] }}
 
 extra-context =
     key path_list :path_list
@@ -971,7 +940,7 @@ signature_file = ${directory:run}/slave_introspection_validate_configuration_sta
 
 [slave-introspection-graceful]
 < = jinja2-template-base
-template = {{ parameter_dict['template_graceful_script'] }}
+template = {{ software_parameter_dict['template_graceful_script'] }}
 rendered = ${directory:etc-run}/slave-introspection-safe-graceful
 mode = 0700
 
@@ -981,11 +950,11 @@ extra-context =
 
 [slave-introspection-validate]
 <= jinja2-template-base
-template = {{ parameter_dict['template_validate_script'] }}
+template = {{ software_parameter_dict['template_validate_script'] }}
 rendered = ${directory:bin}/slave-introspection-validate
 mode = 0700
 last_state_file = ${directory:run}/slave_introspection_configuration_last_state
-validate_command = {{ parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
+validate_command = {{ software_parameter_dict['nginx'] }} -c ${frontend-configuration:slave-introspection-configuration} -t
 extra-context =
     key validate_command :validate_command
     key configuration_state_command slave-introspection-configuration-state-validate:rendered
@@ -999,7 +968,7 @@ config-verification-script = ${promise-slave-introspection-configuration-helper:
 
 [promise-slave-introspection-configuration-helper]
 < = jinja2-template-base
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 rendered = ${directory:bin}/slave-introspection-read-last-configuration-state
 mode = 0700
 content =
@@ -1012,7 +981,7 @@ context =
 <= monitor-promise-base
 module = check_port_listening
 name = slave_introspection_https.py
-config-hostname = {{ instance_parameter['ipv6-random'] }}
+config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
 config-port = ${frontend-configuration:slave-introspection-https-port}
 
 [logrotate-entry-slave-introspection]
@@ -1031,9 +1000,25 @@ config-command =
   ${logrotate:wrapper-path} -d
 
 [configuration]
-{%- for key, value in instance_parameter.iteritems() -%}
+{%- for key, value in instance_parameter_dict.iteritems() -%}
 {%-   if key.startswith('configuration.') %}
 {{ key.replace('configuration.', '') }} = {{ dumps(value) }}
 {%-   endif -%}
-{%- endfor -%}
-{%- endif -%} {# if slap_software_type == software_type #}
+{%- endfor %}
+
+[instance-parameter-section]
+{#- There are dangerous keys like recipe, etc #}
+{#- XXX: Some other approach would be useful #}
+{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
+{%- for key, value in instance_parameter_dict.iteritems() -%}
+{%-   if not key.startswith('configuration.') and key not in DROP_KEY_LIST %}
+{{ key }} = {{ dumps(value) }}
+{%-   endif -%}
+{%- endfor %}
+
+[software-parameter-section]
+{%- for key, value in software_parameter_dict.iteritems() %}
+{{ key }} = {{ dumps(value) }}
+{%- endfor %}
+
+{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}

software/caddy-frontend/instance-apache-replicate.cfg.in

-{% if slap_software_type in software_type %}
+{% if instance_parameter_dict['slap-software-type'] in software_type %}
 {% set aibcc_enabled = True %}
 {% import "caucase" as caucase with context %}
 {#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
 {%- set SERVER_POLLUTED_KEY_LIST = ['connection-parameter-hash', 'timestamp', 'slave_title', 'slap_software_type'] -%}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 {%- set GOOD_CIPHER_LIST = ['ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-WITH-CHACHA20-POLY1305', 'ECDHE-RSA-AES256-CBC-SHA', 'ECDHE-RSA-AES128-CBC-SHA', 'ECDHE-ECDSA-AES256-CBC-SHA', 'ECDHE-ECDSA-AES128-CBC-SHA', 'RSA-AES256-CBC-SHA', 'RSA-AES128-CBC-SHA', 'ECDHE-RSA-3DES-EDE-CBC-SHA', 'RSA-3DES-EDE-CBC-SHA'] %}
+{#- Allow to pass only some parameters to frontend nodes #}
+{%- set FRONTEND_NODE_PASSED_KEY_LIST = [
+        'plain_http_port',
+        'port',
+        'apache-certificate',
+        'apache-key',
+        'domain',
+        'enable-http2-by-default',
+        'global-disable-http2',
+        'mpm-graceful-shutdown-timeout',
+        'public-ipv4',
+        're6st-verification-url',
+        'backend-connect-timeout',
+        'backend-connect-retries',
+        'ciphers',
+        'request-timeout',
+        'authenticate-to-backend',
+    ]
+%}
 {% set aikc_enabled = slapparameter_dict.get('automatic-internal-kedifa-caucase-csr', 'true').lower() in TRUE_VALUES %}
 {% set aibcc_enabled = slapparameter_dict.get('automatic-internal-backend-client-caucase-csr', 'true').lower() in TRUE_VALUES %}
 {# Ports 8401, 8402 and 8410+1..N are reserved for monitor ports on various partitions #}
 {% set master_partition_monitor_monitor_httpd_port = 8401 %}
 {% set kedifa_partition_monitor_httpd_port = 8402 %}
 {% set frontend_monitor_httpd_base_port = 8410 %}
-{% set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
-{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_backend_client_port'] %}
+{% set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
+{% set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_backend_client_port'] %}
 {% set caucase_url = 'http://' ~ caucase_netloc %}
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -20,18 +39,18 @@ rendered = ${buildout:directory}/${:filename}
 extra-context =
 context =
     import json_module json
-    raw common_profile {{ common_profile }}
+    raw profile_common {{ software_parameter_dict['profile_common'] }}
     ${:extra-context}
 
 {% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
 {% set part_list = [] %}
 {% set single_type_key = 'single-' %}
-{% if slap_software_type == "replicate" %}
+{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
 {%   set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
-{% elif slap_software_type in ['default', 'RootSoftwareInstance'] %}
+{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
 {%   set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
 {% else %}
-{%   set frontend_type = "%s%s" % (single_type_key, slap_software_type) %}
+{%   set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
 {% endif %}
 {% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
 {% set slave_list_name = 'extra_slave_instance_list' %}
@@ -70,16 +89,19 @@ context =
 {%   endfor %}
 {%   do config_dict.__setitem__('monitor-httpd-port', frontend_monitor_httpd_base_port + i) %}
 {%   do config_dict.__setitem__('backend-client-caucase-url', caucase_url) %}
+{%   set state_key = "-frontend-%s-state" % i %}
+{%   set frontend_state = slapparameter_dict.pop(state_key, None) %}
+{%   if frontend_state != 'destroyed' %}
 {%     do frontend_list.append(frontend_name) %}
 {%     do frontend_section_list.append(request_section_title) %}
+{%   endif %}
 {%   do part_list.append(request_section_title) %}
 # Filling request dict for slave
-{%   set state_key = "-frontend-%s-state" % i %}
 {%   set request_content_dict = {
                                   'config': config_dict,
                                   'name': frontend_name,
                                   'sla': sla_dict,
-                                  'state': slapparameter_dict.pop(state_key, None)
+                                  'state': frontend_state
                                   } %}
 {%   set frontend_software_url_key = "-frontend-%s-software-release-url" % i %}
 {%   do request_content_dict.__setitem__('software-url', slapparameter_dict.get(frontend_software_url_key) or '${slap-connection:software-release-url}') %}
@@ -92,7 +114,7 @@ context =
 {% set rejected_slave_title_dict = {} %}
 {% set warning_slave_dict = {} %}
 {% set used_host_list = [] %}
-{% for slave in sorted(slave_instance_list) %}
+{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
 {%   set slave_error_list = [] %}
 {%   set slave_warning_list = [] %}
 {%   set slave_server_alias_unclashed = [] %}
@@ -142,7 +164,7 @@ context =
 {%   for url_key in ['url', 'https-url'] %}
 {%     if url_key in slave %}
 {%       set url = (slave[url_key] or '').strip() %}
-{%       if subprocess_module.call([caddy_backend_url_validator, url]) == 1 or not validators.url(url) %}
+{%       if not validators.url(url) %}
 {%         do slave_error_list.append('slave %s %r invalid' % (url_key, url)) %}
 {%       elif url != slave[url_key] %}
 {%         do slave_warning_list.append('slave %s %r has been converted to %r' % (url_key, slave[url_key], url)) %}
@@ -151,7 +173,7 @@ context =
 {%   endfor %}
 {%   if 'ssl_proxy_ca_crt' in slave %}
 {%     set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
-{%     set check_popen = popen([parameter_dict['openssl'], 'x509', '-noout']) %}
+{%     set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
 {%     do check_popen.communicate(ssl_proxy_ca_crt) %}
 {%     if check_popen.returncode != 0 %}
 {%       do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
@@ -167,8 +189,8 @@ context =
 {%     do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required')  %}
 {%   endif %}
 {%   if slave.get('ssl_key') and slave.get('ssl_crt') %}
-{%     set key_popen = popen([parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
-{%     set crt_popen = popen([parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
+{%     set key_popen = popen([software_parameter_dict['openssl'], 'rsa', '-noout', '-modulus']) %}
+{%     set crt_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout', '-modulus']) %}
 {%     set key_modulus = key_popen.communicate(slave['ssl_key'])[0] | trim %}
 {%     set crt_modulus = crt_popen.communicate(slave['ssl_crt'])[0] | trim %}
 {%     if not key_modulus or key_modulus != crt_modulus  %}
@@ -217,6 +239,13 @@ config-monitor-password = ${monitor-htpasswd:passwd}
 software-type = {{frontend_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url backend-client-csr_id-url csr_id-url csr_id-certificate backend-haproxy-statistic-url
 
+{#- Send only needed parameters to frontend nodes #}
+{%- set base_node_configuration_dict = {} %}
+{%- for key in FRONTEND_NODE_PASSED_KEY_LIST %}
+{%-   if key in slapparameter_dict %}
+{%-     do base_node_configuration_dict.__setitem__(key,  slapparameter_dict[key]) %}
+{%-   endif %}
+{%- endfor %}
 {% for section, frontend_request in request_dict.iteritems() %}
 {%   set state = frontend_request.get('state', '') %}
 [{{section}}]
@@ -230,15 +259,18 @@ config-slave-kedifa-information = ${request-kedifa:connection-slave-kedifa-infor
 config-kedifa-caucase-url = ${request-kedifa:connection-caucase-url}
 config-backend-client-caucase-url = {{ caucase_url }}
 config-master-key-download-url = ${request-kedifa:connection-master-key-download-url}
-config-cluster-identification = {{ cluster_identification }}
+config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
 {#   Do not send additional parameters for destroyed nodes #}
 {%   if state != 'destroyed' %}
-{%     set slave_configuration_dict = slapparameter_dict %}
-{%     do slave_configuration_dict.update(frontend_request.get('config')) %}
+{%     set node_configuration_dict = {} %}
+{%     do node_configuration_dict.update(frontend_request.get('config')) %}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
-{%     do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
-{%     do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
-{%-     for config_key, config_value in slave_configuration_dict.iteritems() %}
+{%     do node_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list, sort_keys=True)) %}
+{%     do node_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
+{%-     for config_key, config_value in node_configuration_dict.iteritems() %}
+config-{{ config_key }} = {{ dumps(config_value) }}
+{%     endfor -%}
+{%-     for config_key, config_value in base_node_configuration_dict.iteritems() %}
 config-{{ config_key }} = {{ dumps(config_value) }}
 {%     endfor -%}
 {%   endif %}
@@ -260,7 +292,7 @@ sla-{{ parameter }} = {{ value }}
 <= monitor-publish
 recipe = slapos.cookbook:publish
 domain = {{ slapparameter_dict.get('domain') }}
-slave-amount = {{ slave_instance_list | length }}
+slave-amount = {{ instance_parameter_dict['slave-instance-list'] | length }}
 accepted-slave-amount = {{ authorized_slave_list | length }}
 rejected-slave-amount = {{ rejected_slave_dict | length }}
 backend-client-caucase-url = {{ caucase_url }}
@@ -327,7 +359,7 @@ config-{{ key }} = {{ dumps(slapparameter_dict[key]) }}
 {%-   endif %}
 {%- endfor %}
 config-slave-list = {{ dumps(authorized_slave_list) }}
-config-cluster-identification = {{ cluster_identification }}
+config-cluster-identification = {{ instance_parameter_dict['root-instance-title'] }}
 
 {% set software_url_key = "-kedifa-software-release-url" %}
 {% if slapparameter_dict.has_key(software_url_key) %}
@@ -365,7 +397,7 @@ sla-{{ key[sla_kedifa_key_length:] }} = {{ slapparameter_dict.pop(key) }}
 
 [active-slave-instance]
 {% set active_slave_instance_list = [] %}
-{% for slave_instance in slave_instance_list %}
+{% for slave_instance in instance_parameter_dict['slave-instance-list'] %}
 {# Provide a list of slave titles send by master, in order to filter out already destroyed slaves #}
 {# Note: This functionality is not yet covered by tests, please modify with care #}
 {%   do active_slave_instance_list.append(slave_instance['slave_reference']) %}
@@ -375,7 +407,7 @@ active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, so
 
 [dynamic-publish-slave-information]
 < = jinja2-template-base
-template = {{ template_publish_slave_information }}
+template = {{ software_parameter_dict['profile_replicate_publish_slave_information'] }}
 filename = dynamic-publish-slave-information.cfg
 extensions = jinja2.ext.do
 extra-context =
@@ -399,6 +431,8 @@ backup = ${:srv}/backup
 # CAUCASE directories
 caucased = ${:srv}/caucased
 backup-caucased = ${:backup}/caucased
+# NGINX
+rejected-var = ${:var}/rejected-nginx
 
 {% if aikc_enabled %}
 [directory]
@@ -418,11 +452,11 @@ csr_id = ${directory:aikc}/csr_id
 
 [aikc-user-csr]
 recipe = plone.recipe.command
-organization = {{ cluster_identification }}
+organization = {{ instance_parameter_dict['root-instance-title'] }}
 organizational_unit = Automatic Internal Kedifa Caucase CSR
 command =
   if [ ! -f ${:csr} ] && [ ! -f ${:key} ]  ; then
-    {{ parameter_dict['openssl'] }} req -new -sha256 \
+    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
       -newkey rsa:2048 -nodes -keyout ${:key} \
       -subj "/O=${:organization}/OU=${:organizational_unit}" \
       -out ${:csr}
@@ -430,6 +464,7 @@ command =
 update-command = ${:command}
 csr = ${aikc-config:csr}
 key = ${aikc-config:key}
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 
 
@@ -438,8 +473,8 @@ stop-on-error = True
 recipe = slapos.recipe.template:jinja2
 context =
   key caucase_url aikc-config:caucase-url
-template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
-  exec {{ parameter_dict['bin_directory'] }}/caucase \
+template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
+  exec {{ software_parameter_dict['bin_directory'] }}/caucase \
 {# raw block to use context #}
 {% raw %}
   --ca-url {{ caucase_url }} \
@@ -456,7 +491,8 @@ mode = 0700
 {% do part_list.append('aikc-create-user') %}
 [aikc-create-user]
 recipe = plone.recipe.command
-stop-on-error = True
+{#- The called command is smart enough to survive errors and retry #}
+stop-on-error = False
 update-command = ${:command}
 command =
   if ! [ -f ${aikc-config:user-created} ] ; then
@@ -472,7 +508,7 @@ command =
 {% do part_list.append('aikc-user-caucase-updater-promise') %}
 {{ caucase.updater(
      prefix='aikc-user-caucase-updater',
-     buildout_bin_directory=parameter_dict['bin_directory'],
+     buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/aikc-user-caucase-updater',
      url='${aikc-config:caucase-url}',
      data_dir='${directory:srv}/caucase-updater',
@@ -503,7 +539,7 @@ recipe = slapos.recipe.template:jinja2
 context =
   key csr_id_url request-{{ csr }}:connection-csr_id-url
   key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
-template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
+template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
   test -f ${directory:aikc}/{{ csr }}-done && exit 0
   ${buildout:executable} ${aikc-check-certificate:rendered} \
 {# raw block to use context #}
@@ -512,7 +548,7 @@ template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
   """{{ csr_id_certificate }}"""
 {% endraw %}
   if [ $? = 0 ]; then
-    csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
+    csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
 {% raw %}
   {{ csr_id_url }} \
 {% endraw %}
@@ -525,7 +561,8 @@ mode = 0700
 {% do part_list.append('aikc-%s' % (csr,)) %}
 [aikc-{{ csr }}]
 recipe = plone.recipe.command
-stop-on-error = True
+{#- The called command is smart enough to survive errors and retry #}
+stop-on-error = False
 command =
   ${aikc-{{ csr }}-wrapper:rendered}
 update-command = ${:command}
@@ -550,11 +587,11 @@ csr_id = ${directory:aibcc}/csr_id
 
 [aibcc-user-csr]
 recipe = plone.recipe.command
-organization = {{ cluster_identification }}
+organization = {{ instance_parameter_dict['root-instance-title'] }}
 organizational_unit = Automatic Sign Backend Client Caucase CSR
 command =
   if [ ! -f ${:csr} ] && [ ! -f ${:key} ]  ; then
-    {{ parameter_dict['openssl'] }} req -new -sha256 \
+    {{ software_parameter_dict['openssl'] }} req -new -sha256 \
       -newkey rsa:2048 -nodes -keyout ${:key} \
       -subj "/O=${:organization}/OU=${:organizational_unit}" \
       -out ${:csr}
@@ -562,6 +599,7 @@ command =
 update-command = ${:command}
 csr = ${aibcc-config:csr}
 key = ${aibcc-config:key}
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 
 
@@ -570,8 +608,8 @@ stop-on-error = True
 recipe = slapos.recipe.template:jinja2
 context =
   key caucase_url aibcc-config:caucase-url
-template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
-  exec {{ parameter_dict['bin_directory'] }}/caucase \
+template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
+  exec {{ software_parameter_dict['bin_directory'] }}/caucase \
 {# raw block to use context #}
 {% raw %}
   --ca-url {{ caucase_url }} \
@@ -590,6 +628,7 @@ mode = 0700
 recipe = plone.recipe.command
 # the caucase for this part is provided in this profile, so we can't fail
 # as otherwise caucase will never be started...
+{#- XXX: Create promise #}
 stop-on-error = False
 update-command = ${:command}
 command =
@@ -606,7 +645,7 @@ command =
 {% do part_list.append('aibcc-user-caucase-updater-promise') %}
 {{ caucase.updater(
      prefix='aibcc-user-caucase-updater',
-     buildout_bin_directory=parameter_dict['bin_directory'],
+     buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/aibcc-user-caucase-updater',
      url='${aibcc-config:caucase-url}',
      data_dir='${directory:srv}/caucase-updater',
@@ -636,7 +675,7 @@ recipe = slapos.recipe.template:jinja2
 context =
   key csr_id_url request-{{ csr }}:connection-backend-client-csr_id-url
   key csr_id_certificate request-{{ csr }}:connection-csr_id-certificate
-template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
+template = inline:#!{{ software_parameter_dict['dash'] }}/bin/dash
   test -f ${directory:aibcc}/{{ csr }}-done && exit 0
   ${buildout:executable} ${aibcc-check-certificate:rendered} \
 {# raw block to use context #}
@@ -645,7 +684,7 @@ template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
   """{{ csr_id_certificate }}"""
 {% endraw %}
   if [ $? = 0 ]; then
-    csr_id=`{{ parameter_dict['curl'] }}/bin/curl -s -k -g \
+    csr_id=`{{ software_parameter_dict['curl'] }}/bin/curl -s -k -g \
 {% raw %}
   {{ csr_id_url }} \
 {% endraw %}
@@ -658,7 +697,8 @@ mode = 0700
 {% do part_list.append('aibcc-%s' % (csr,)) %}
 [aibcc-{{ csr }}]
 recipe = plone.recipe.command
-stop-on-error = True
+{#- The called command is smart enough to survive errors and retry #}
+stop-on-error = False
 command =
   ${aibcc-{{ csr }}-wrapper:rendered}
 update-command = ${:command}
@@ -670,7 +710,7 @@ recipe = slapos.recipe.template:jinja2
 filename = rejected-slave.json
 directory = ${directory:promise-output}
 rendered = ${:directory}/${:filename}
-template = {{ parameter_dict['template_empty'] }}
+template = {{ software_parameter_dict['template_empty'] }}
 {% if rejected_slave_title_dict %}
 {# sort_keys are important in order to avoid shuffling parameters on each run #}
 content = {{ dumps(json_module.dumps(rejected_slave_title_dict, indent=2, sort_keys=True)) }}
@@ -685,20 +725,15 @@ service = ${:etc}/service
 promise-output = ${:srv}/promise-output
 
 [rejected-slave-publish-configuration]
-ip = {{ instance_parameter['ipv6-random'] }}
+ip = {{ instance_parameter_dict['ipv6-random'] }}
 port = 14455
 
 [rejected-slave-publish]
 directory = ${rejected-slave-json:directory}
 url = https://${rejected-slave-password:user}:${rejected-slave-password:passwd}@[${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port}/${rejected-slave-json:filename}
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['caddy'] }}
-  -conf ${rejected-slave-template:rendered}
-  -log stderr
-  -http2=true
-  -disable-http-challenge
-  -disable-tls-alpn-challenge
-  -root ${:directory}
+command-line = {{ software_parameter_dict['nginx'] }}
+  -c ${rejected-slave-template:rendered}
 
 wrapper-path = ${directory:service}/rejected-slave-publish
 hash-existing-files =
@@ -712,6 +747,7 @@ recipe = plone.recipe.command
 certificate = ${directory:etc}/rejected-slave.pem
 key = ${:certificate}
 
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 update-command = ${:command}
 command =
@@ -728,18 +764,54 @@ storage-path = ${directory:etc}/.rejected-slave.passwd
 bytes = 8
 user = admin
 
+[rejected-slave-htpasswd]
+recipe = plone.recipe.command
+{#- Can be stopped on error, as does not rely on self provided service #}
+stop-on-error = True
+file = ${directory:var}/nginx-rejected.htpasswd
+{#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #}
+password = ${rejected-slave-password:passwd}
+command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} ${rejected-slave-password:user} ${:password}
+
 [rejected-slave-template]
 recipe = slapos.recipe.template:jinja2
+var = ${directory:rejected-var}
+pid = ${directory:var}/nginx-rejected.pid
 template = inline:
-  https://:${rejected-slave-publish-configuration:port}/ {
-    basicauth / ${rejected-slave-password:user} ${rejected-slave-password:passwd}
-    tls ${rejected-slave-certificate:certificate} ${rejected-slave-certificate:key}
-    bind ${rejected-slave-publish-configuration:ip}
-    log stderr
-    errors stderr
+  daemon off;
+  pid ${:pid};
+  error_log stderr;
+  events {
+  }
+  http {
+    include {{ software_parameter_dict['nginx_mime'] }};
+    server {
+      server_name_in_redirect off;
+      port_in_redirect off;
+      error_log stderr;
+      access_log /dev/null;
+      listen [${rejected-slave-publish-configuration:ip}]:${rejected-slave-publish-configuration:port} ssl;
+      ssl_certificate ${rejected-slave-certificate:certificate};
+      ssl_certificate_key ${rejected-slave-certificate:certificate};
+      default_type application/octet-stream;
+      client_body_temp_path ${:var} 1 2;
+      proxy_temp_path ${:var} 1 2;
+      fastcgi_temp_path ${:var} 1 2;
+      uwsgi_temp_path ${:var} 1 2;
+      scgi_temp_path ${:var} 1 2;
+
+      location / {
+        alias ${rejected-slave-json:directory}/;
+        autoindex off;
+        sendfile on;
+        sendfile_max_chunk 1m;
+        auth_basic "Rejected slave template";
+        auth_basic_user_file ${rejected-slave-htpasswd:file};
+      }
+    }
   }
 
-rendered = ${directory:etc}/Caddyfile-rejected-slave
+rendered = ${directory:etc}/nginx-rejected-slave.conf
 
 [promise-rejected-slave-publish-ip-port]
 <= monitor-promise-base
@@ -761,7 +833,7 @@ config-url = ${rejected-slave-publish:url}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 {{  caucase.caucased(
       prefix='caucased-backend-client',
-      buildout_bin_directory=parameter_dict['bin_directory'],
+      buildout_bin_directory=software_parameter_dict['bin_directory'],
       caucased_path='${directory:service}/caucased-backend-client',
       backup_dir='${directory:backup-caucased}',
       data_dir='${directory:caucased}',
@@ -773,8 +845,8 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [buildout]
 extends =
-  {{ common_profile }}
-  {{ template_monitor }}
+  {{ software_parameter_dict['profile_common'] }}
+  {{ software_parameter_dict['profile_monitor2'] }}
 parts =
   monitor-base
   publish-slave-information

software/caddy-frontend/instance-kedifa.cfg.in

-{%- if slap_software_type == software_type -%}
+{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
 {% import "caucase" as caucase with context %}
 # KeDiFa instance profile
 [buildout]
 extends =
-  {{ parameter_dict['common_profile'] }}
-  {{ parameter_dict['monitor_template'] }}
-  {{ parameter_dict['logrotate_base_instance'] }}
+  {{ software_parameter_dict['profile_common'] }}
+  {{ software_parameter_dict['profile_monitor'] }}
+  {{ software_parameter_dict['profile_logrotate_base'] }}
 
 parts =
   monitor-base
@@ -25,18 +25,18 @@ parts =
 # Note: Workaround for monitor stack, which uses monitor-httpd-port parameter
 #       directly, and in our case it can come from the network, thus resulting
 #       with need to strip !py!'u'
-monitor-httpd-port = {{ instance_parameter['configuration.monitor-httpd-port'] | int }}
-password = {{ instance_parameter['configuration.monitor-password'] | string }}
+monitor-httpd-port = {{ instance_parameter_dict['configuration.monitor-httpd-port'] | int }}
+password = {{ instance_parameter_dict['configuration.monitor-password'] | string }}
 
 [caucased]
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-{%  set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
-{%  set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_port'] -%}
+{%  set caucase_host = '[' ~ instance_parameter_dict['ipv6-random'] ~ ']' %}
+{%  set caucase_netloc = caucase_host ~ ':' ~ instance_parameter_dict['configuration.caucase_port'] -%}
 {%  set caucase_url = 'http://' ~ caucase_netloc -%}
 {{  caucase.caucased(
       prefix='caucased',
-      buildout_bin_directory=parameter_dict['bin_directory'],
+      buildout_bin_directory=software_parameter_dict['bin_directory'],
       caucased_path='${directory:service}/caucased',
       backup_dir='${directory:backup-caucased}',
       data_dir='${directory:caucased}',
@@ -75,7 +75,8 @@ reservation = ${:srv}/reservation
 
 # csr_id publication
 csr_id = ${:srv}/csr_id
-caddy-csr_id = ${:etc}/caddy-csr_id
+certificate-csr_id = ${:var}/certificate-csr_id
+expose-csr_id-var = ${:var}/expose-csr_id
 
 [kedifa-csr]
 recipe = plone.recipe.command
@@ -83,22 +84,23 @@ organization = {{ slapparameter_dict['cluster-identification'] }}
 organizational_unit = Kedifa Partition
 command =
   if [ ! -f ${:template-csr} ] && [ ! -f ${:key} ]  ; then
-    /bin/bash -c '{{ parameter_dict['openssl'] }} req -new -sha256 \
+    /bin/bash -c '{{ software_parameter_dict['openssl'] }} req -new -sha256 \
       -newkey rsa:2048 -nodes -keyout ${:key} \
       -subj "/O=${:organization}/OU=${:organizational_unit}" \
       -reqexts SAN \
-      -config <(cat {{ parameter_dict['openssl_cnf'] }} \
+      -config <(cat {{ software_parameter_dict['openssl_cnf'] }} \
       <(printf "\n[SAN]\nsubjectAltName=IP:${kedifa-config:ip}")) \
       -out ${:template-csr}'
   fi
 update-command = ${:command}
 template-csr = ${kedifa-config:template-csr}
 key = ${kedifa-config:key}
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 
 {{ caucase.updater(
      prefix='caucase-updater',
-     buildout_bin_directory=parameter_dict['bin_directory'],
+     buildout_bin_directory=software_parameter_dict['bin_directory'],
      updater_path='${directory:service}/caucase-updater',
      url=caucase_url,
      data_dir='${directory:srv}/caucase-updater',
@@ -119,7 +121,7 @@ csr_work_path = ${directory:tmp}/${:_buildout_section_name_}
 stop-on-error = False
 update-command = ${:command}
 command =
-  {{ parameter_dict['bin_directory'] }}/caucase \
+  {{ software_parameter_dict['bin_directory'] }}/caucase \
     --ca-url {{ caucase_url }} \
     --ca-crt ${kedifa-config:ca-certificate} \
     --crl ${kedifa-config:crl} \
@@ -131,20 +133,21 @@ command =
 
 [certificate-csr_id]
 recipe = plone.recipe.command
-certificate = ${directory:caddy-csr_id}/certificate.pem
-key = ${directory:caddy-csr_id}/key.pem
+certificate = ${directory:certificate-csr_id}/certificate.pem
+key = ${directory:certificate-csr_id}/key.pem
 
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 update-command = ${:command}
 command =
   if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
-    {{ parameter_dict['openssl'] }} req -new -newkey rsa:2048 -sha256 -subj \
-      "/O=${kedifa-csr:organization}/OU=${kedifa-csr:organizational_unit}/CN={{ instance_parameter['ipv6-random'] }}" \
+    {{ software_parameter_dict['openssl'] }} req -new -newkey rsa:2048 -sha256 -subj \
+      "/O=${kedifa-csr:organization}/OU=${kedifa-csr:organizational_unit}/CN={{ instance_parameter_dict['ipv6-random'] }}" \
       -days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
   fi
 
 [expose-csr_id-configuration]
-ip = {{ instance_parameter['ipv6-random'] }}
+ip = {{ instance_parameter_dict['ipv6-random'] }}
 port = 17000
 key = ${certificate-csr_id:key}
 certificate = ${certificate-csr_id:certificate}
@@ -152,14 +155,40 @@ error-log = ${directory:log}/expose-csr_id.log
 
 [expose-csr_id-template]
 recipe = slapos.recipe.template:jinja2
+var = ${directory:expose-csr_id-var}
+pid = ${directory:var}/nginx-expose-csr_id.pid
+rendered = ${directory:etc}/nginx-expose-csr_id.conf
 template = inline:
-  https://:${expose-csr_id-configuration:port}/ {
-    bind ${expose-csr_id-configuration:ip}
-    tls ${expose-csr_id-configuration:certificate} ${expose-csr_id-configuration:key}
-    log ${expose-csr_id-configuration:error-log}
+  daemon off;
+  pid ${:pid};
+  error_log ${expose-csr_id-configuration:error-log};
+  events {
+  }
+  http {
+    include {{ software_parameter_dict['nginx_mime'] }};
+    server {
+      server_name_in_redirect off;
+      port_in_redirect off;
+      error_log ${expose-csr_id-configuration:error-log};
+      access_log /dev/null;
+      listen [${expose-csr_id-configuration:ip}]:${expose-csr_id-configuration:port} ssl;
+      ssl_certificate ${expose-csr_id-configuration:certificate};
+      ssl_certificate_key ${expose-csr_id-configuration:key};
+      default_type application/octet-stream;
+      client_body_temp_path ${:var} 1 2;
+      proxy_temp_path ${:var} 1 2;
+      fastcgi_temp_path ${:var} 1 2;
+      uwsgi_temp_path ${:var} 1 2;
+      scgi_temp_path ${:var} 1 2;
+
+      location / {
+        alias ${directory:csr_id}/;
+        autoindex off;
+        sendfile on;
+        sendfile_max_chunk 1m;
+      }
+    }
   }
-
-rendered = ${directory:caddy-csr_id}/Caddyfile
 
 [promise-expose-csr_id-ip-port]
 <= monitor-promise-base
@@ -171,13 +200,8 @@ config-port = ${expose-csr_id-configuration:port}
 [expose-csr_id]
 depends = ${store-csr_id:command}
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['caddy'] }}
-  -conf ${expose-csr_id-template:rendered}
-  -log ${expose-csr_id-configuration:error-log}
-  -http2=true
-  -disable-http-challenge
-  -disable-tls-alpn-challenge
-  -root ${directory:csr_id}
+command-line = {{ software_parameter_dict['nginx'] }}
+  -c ${expose-csr_id-template:rendered}
 
 wrapper-path = ${directory:service}/expose-csr_id
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
@@ -191,19 +215,19 @@ commands =
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:filename}
 extra-context =
-slapparameter_dict = {{ dumps(instance_parameter['configuration']) }}
-slap_software_type = {{ dumps(instance_parameter['slap-software-type']) }}
+slapparameter_dict = {{ dumps(slapparameter_dict) }}
+slap_software_type = {{ dumps(instance_parameter_dict['slap-software-type']) }}
 context =
     import json_module json
-    raw common_profile {{ parameter_dict['common_profile'] }}
+    raw profile_common {{ software_parameter_dict['profile_common'] }}
     key slap_software_type :slap_software_type
     key slapparameter_dict :slapparameter_dict
     section directory directory
     ${:extra-context}
 
 [kedifa-config]
-ip = {{ instance_parameter['ipv6-random'] }}
-port = {{ instance_parameter['configuration.kedifa_port'] }}
+ip = {{ instance_parameter_dict['ipv6-random'] }}
+port = {{ instance_parameter_dict['configuration.kedifa_port'] }}
 db = ${directory:kedifa}/kedifa.sqlite
 certificate = ${directory:etc-kedifa}/certificate.pem
 key = ${:certificate}
@@ -215,7 +239,7 @@ logfile = ${directory:log}/kedifa.log
 
 [kedifa-reloader]
 <= jinja2-template-base
-template = {{ parameter_dict['template_wrapper'] }}
+template = {{ software_parameter_dict['template_wrapper'] }}
 rendered = ${directory:etc-run}/kedifa-reloader
 command =
   kill -HUP `cat ${kedifa-config:pidfile}`
@@ -236,12 +260,12 @@ config-ca-cert-file = ${kedifa-config:ca-certificate}
 <= logrotate-entry-base
 name = kedifa
 log = ${kedifa-config:logfile}
-rotate-num = {{ instance_parameter['configuration.rotate-num'] | int }}
+rotate-num = {{ instance_parameter_dict['configuration.rotate-num'] | int }}
 delaycompress =
 
 [kedifa]
 recipe = slapos.cookbook:wrapper
-command-line = {{ parameter_dict['kedifa'] }}
+command-line = {{ software_parameter_dict['kedifa'] }}
   --ip ${kedifa-config:ip}
   --port ${kedifa-config:port}
   --db ${kedifa-config:db}
@@ -268,7 +292,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 recipe = plone.recipe.command
 file = ${directory:reservation}/${:_buildout_section_name_}
 command =
-  [ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
+  [ ! -f ${:file} ] && {{ software_parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
 update-command = ${:command}
 
 [{{ slave_reference }}-auth-random]
@@ -283,7 +307,7 @@ commands =
 recipe = plone.recipe.command
 file = ${directory:reservation}/${:_buildout_section_name_}
 command =
-  [ ! -f ${:file} ] && {{ parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
+  [ ! -f ${:file} ] && {{ software_parameter_dict['curl'] }}/bin/curl -s -g -X POST https://[${kedifa-config:ip}]:${kedifa-config:port}/reserve-id --cert ${kedifa-config:certificate} --cacert ${kedifa-config:ca-certificate} > ${:file}.tmp && mv ${:file}.tmp ${:file}
 update-command = ${:command}
 
 [master-auth-random]
@@ -311,4 +335,4 @@ name = ${:_buildout_section_name_}.py
 config-command =
   ${logrotate:wrapper-path} -d
 
-{%- endif -%} {# if slap_software_type in software_type #}
+{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}

software/caddy-frontend/instance.cfg.in

 [buildout]
-extends = {{ common_profile }}
+extends = {{ software_parameter_dict['profile_common'] }}
 
 parts =
-  dynamic-template-caddy-replicate
   switch-softwaretype
 
 [caddyprofiledeps]
@@ -11,76 +10,59 @@ recipe = caddyprofiledeps
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:filename}
+extensions = jinja2.ext.do
 extra-context =
 context =
     import json_module json
-    key slap_software_type instance-parameter:slap-software-type
     key slapparameter_dict instance-parameter:configuration
-    key slave_instance_list instance-parameter:slave-instance-list
-    section instance_parameter instance-parameter
+    section instance_parameter_dict instance-parameter
+    section software_parameter_dict software-parameter-section
     ${:extra-context}
+caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
+import-list =
+  file caucase :caucase-jinja2-library
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${dynamic-template-caddy-replicate:rendered}
-RootSoftwareInstance = ${dynamic-template-caddy-replicate:rendered}
-custom-personal = ${dynamic-template-caddy-replicate:rendered}
-single-default = ${dynamic-template-caddy-frontend:rendered}
-single-custom-personal = ${dynamic-template-caddy-frontend:rendered}
-replicate = ${dynamic-template-caddy-replicate:rendered}
-kedifa = ${dynamic-template-kedifa:rendered}
+default = ${dynamic-profile-caddy-replicate:rendered}
+RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered}
+custom-personal = ${dynamic-profile-caddy-replicate:rendered}
+single-default = ${dynamic-profile-caddy-frontend:rendered}
+single-custom-personal = ${dynamic-profile-caddy-frontend:rendered}
+replicate = ${dynamic-profile-caddy-replicate:rendered}
+kedifa = ${dynamic-profile-kedifa:rendered}
 
-[dynamic-template-caddy-frontend-parameters]
-{% for key,value in template_frontend_parameter_dict.iteritems() %}
+[software-parameter-section]
+{% for key,value in software_parameter_dict.iteritems() %}
 {{ key }} = {{ dumps(value) }}
 {% endfor -%}
 
-[dynamic-template-caddy-frontend]
+[dynamic-profile-caddy-frontend]
 < = jinja2-template-base
-template = {{ template_caddy_frontend }}
+template = {{ software_parameter_dict['profile_caddy_frontend'] }}
 filename = instance-caddy-frontend.cfg
-extensions = jinja2.ext.do
 extra-context =
   import furl_module furl
-  section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type single-custom-personal
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
-[dynamic-template-caddy-replicate]
+[dynamic-profile-caddy-replicate]
 < = jinja2-template-base
 depends = ${caddyprofiledeps:recipe}
-template = {{ template_caddy_replicate }}
+template = {{ software_parameter_dict['profile_caddy_replicate'] }}
 filename = instance-caddy-replicate.cfg
-extensions = jinja2.ext.do
 extra-context =
     import subprocess_module subprocess
     import functools_module functools
     import validators validators
-    key cluster_identification instance-parameter:root-instance-title
-    raw caddy_backend_url_validator {{ caddy_backend_url_validator }}
-    raw template_publish_slave_information {{ template_replicate_publish_slave_information }}
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type RootSoftwareInstance-default-custom-personal-replicate
-    raw template_monitor {{ monitor2_template }}
-    raw common_profile {{ common_profile }}
-    section parameter_dict dynamic-template-caddy-frontend-parameters
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
-[dynamic-template-kedifa]
+[dynamic-profile-kedifa]
 < = jinja2-template-base
-template = {{ template_kedifa }}
+template = {{ software_parameter_dict['profile_kedifa'] }}
 filename = instance-kedifa.cfg
-extensions = jinja2.ext.do
 extra-context =
-  section parameter_dict dynamic-template-caddy-frontend-parameters
   raw software_type kedifa
-caucase-jinja2-library = {{ caucase_jinja2_library }}
-import-list =
-  file caucase :caucase-jinja2-library
 
 [instance-parameter]
 # Fetches parameters defined in SlapOS Master for this instance.

software/caddy-frontend/software.cfg

 [buildout]
-extends = common.cfg
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/caddy/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/trafficserver/buildout.cfg
+  ../../component/6tunnel/buildout.cfg
+  ../../component/xz-utils/buildout.cfg
+  ../../component/rsyslogd/buildout.cfg
+  ../../component/numpy/buildout.cfg
+  ../../component/haproxy/buildout.cfg
+  ../../component/nginx/buildout.cfg
+
+  ../../stack/caucase/buildout.cfg
+# Monitoring stack (keep on bottom)
+  ../../stack/monitor/buildout.cfg
+
+parts +=
+  caucase-eggs
+  template
+  rdiff-backup
+  caddyprofiledeps
+  kedifa-develop
+  kedifa
+
+[kedifa-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/kedifa.git
+git-executable = ${git:location}/bin/git
+revision = d6bbd7db215e12871c1536f22a8fbf994227270c
+
+[kedifa-develop]
+recipe = zc.recipe.egg:develop
+setup = ${kedifa-repository:location}
+
+[kedifa]
+recipe = zc.recipe.egg
+eggs =
+  ${python-cryptography:egg}
+  kedifa
+
+[caddyprofiledeps-setup]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/setup.py
+
+[caddyprofiledeps-dummy]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/caddyprofiledummy.py
+
+[caddyprofiledeps-prepare]
+recipe = plone.recipe.command
+stop-on-error = True
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+update-command = ${:command}
+command =
+  rm -fr ${:location} &&
+  mkdir -p ${:location} &&
+  cp ${caddyprofiledeps-setup:target} ${:location}/ &&
+  cp ${caddyprofiledeps-dummy:target} ${:location}/
+
+[caddyprofiledeps-develop]
+recipe = zc.recipe.egg:develop
+setup = ${caddyprofiledeps-prepare:location}
+
+[caddyprofiledeps]
+depends = ${caddyprofiledeps-develop:recipe}
+recipe = zc.recipe.egg
+eggs =
+  caddyprofiledeps
+  websockify
+  collective.recipe.shelloutput
+
+[profile-common]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-common.cfg.in
+rendered = ${buildout:directory}/instance-common.cfg
+mode = 0644
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+
+[software-parameter-section]
+# libraries
+caucase_jinja2_library = ${caucase-jinja2-library:target}
+
+# profiles
+profile_caddy_frontend = ${profile-caddy-frontend:target}
+profile_caddy_replicate = ${profile-caddy-replicate:target}
+profile_common = ${profile-common:rendered}
+profile_kedifa = ${profile-kedifa:target}
+profile_logrotate_base = ${template-logrotate-base:rendered}
+profile_monitor = ${monitor-template:output}
+profile_monitor2 = ${monitor2-template:rendered}
+profile_replicate_publish_slave_information = ${profile-replicate-publish-slave-information:target}
+profile_slave_list = ${profile-slave-list:target}
+
+# templates
+template_backend_haproxy_configuration = ${template-backend-haproxy-configuration:target}
+template_backend_haproxy_rsyslogd_conf = ${template-backend-haproxy-rsyslogd-conf:target}
+template_caddy_frontend_configuration = ${profile-caddy-frontend-configuration:target}
+template_caddy_lazy_script_call = ${template-caddy-lazy-script-call:target}
+template_configuration_state_script = ${template-configuration-state-script:target}
+template_default_slave_virtualhost = ${template-default-slave-virtualhost:target}
+template_empty = ${template-empty:target}
+template_graceful_script = ${template-graceful-script:target}
+template_log_access = ${template-log-access:target}
+template_not_found_html = ${template-not-found-html:target}
+template_rotate_script = ${template-rotate-script:target}
+template_slave_introspection_httpd_nginx = ${template-slave-introspection-httpd-nginx:target}
+template_trafficserver_logging_yaml = ${template-trafficserver-logging-yaml:target}
+template_trafficserver_records_config = ${template-trafficserver-records-config:target}
+template_trafficserver_storage_config = ${template-trafficserver-storage-config:target}
+template_validate_script = ${template-validate-script:target}
+template_wrapper = ${template-wrapper:output}
+
+# directories
+bin_directory = ${buildout:bin-directory}
+
+# files
+sixtunnel = ${6tunnel:location}
+nginx = ${nginx-output:nginx}
+nginx_mime = ${nginx-output:mime}
+caddy = ${caddy:output}
+haproxy_executable = ${haproxy:location}/sbin/haproxy
+rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
+curl = ${curl:location}
+dash = ${dash:location}
+gzip = ${gzip:location}
+logrotate = ${logrotate:location}
+openssl = ${openssl:location}/bin/openssl
+openssl_cnf = ${openssl:location}/etc/ssl/openssl.cnf
+trafficserver = ${trafficserver:location}
+sha256sum = ${coreutils:location}/bin/sha256sum
+kedifa = ${:bin_directory}/kedifa
+kedifa-updater = ${:bin_directory}/kedifa-updater
+kedifa-csr = ${:bin_directory}/kedifa-csr
+xz_location = ${xz-utils:location}
+htpasswd = ${:bin_directory}/htpasswd
+
+[template]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/template.cfg
+mode = 0644
+context =
+  section software_parameter_dict software-parameter-section
+
+[profile-caddy-frontend]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-frontend.cfg.in
+mode = 0644
+
+[profile-caddy-replicate]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
+mode = 0644
+
+[profile-kedifa]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-kedifa.cfg.in
+mode = 0644
+
+[download-template]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 640
+
+[profile-slave-list]
+<=download-template
+
+[profile-replicate-publish-slave-information]
+<=download-template
+
+[profile-caddy-frontend-configuration]
+<=download-template
+
+[template-not-found-html]
+<=download-template
+
+[template-default-slave-virtualhost]
+<=download-template
+
+[template-backend-haproxy-configuration]
+<=download-template
+
+[template-log-access]
+<=download-template
+
+[template-empty]
+<=download-template
+
+[template-slave-introspection-httpd-nginx]
+<=download-template
+
+[template-wrapper]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/wrapper.in
+output = ${buildout:directory}/template-wrapper.cfg
+mode = 0644
+
+[template-trafficserver-records-config]
+<=download-template
+
+[template-trafficserver-storage-config]
+<=download-template
+
+[template-trafficserver-logging-yaml]
+<=download-template
+
+[template-rotate-script]
+<=download-template
+
+[template-caddy-lazy-script-call]
+<=download-template
+
+[template-graceful-script]
+<=download-template
+
+[template-validate-script]
+<=download-template
+
+[template-configuration-state-script]
+<=download-template
+
+[template-backend-haproxy-rsyslogd-conf]
+<=download-template
 
 [versions]
 # Modern KeDiFa requires zc.lockfile

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -4,21 +4,22 @@
 {%- set backend_slave_list = [] %}
 {%- set part_list = [] %}
 {%- set cache_port = caddy_configuration.get('cache-port') %}
-{%- set cache_access = "http://%s:%s" % (local_ipv4, cache_port) %}
-{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) %}
-{%- set backend_haproxy_http_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['http-port']) %}
-{%- set backend_haproxy_https_url = 'http://%s:%s' % (local_ipv4, backend_haproxy_configuration['https-port']) %}
+{%- set cache_access = "http://%s:%s" % (instance_parameter_dict['ipv4-random'], cache_port) %}
+{%- set ssl_cache_access = "http://%s:%s/HTTPS" % (instance_parameter_dict['ipv4-random'], cache_port) %}
+{%- set backend_haproxy_http_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['http-port']) %}
+{%- set backend_haproxy_https_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['https-port']) %}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
-{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': local_ipv4, 'http_port': http_port, 'https_port': https_port} %}
+{%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %}
 {%- set slave_log_dict = {} %}
-{%- if extra_slave_instance_list %}
 {%- set slave_instance_information_list = [] %}
-{%-   set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) %}
+{%- set slave_instance_list = instance_parameter_dict['slave-instance-list'] %}
+{%- if configuration['extra_slave_instance_list'] %}
+{%-   do slave_instance_list.extend(json_module.loads(configuration['extra_slave_instance_list'])) %}
 {%- endif %}
 {%- if master_key_download_url %}
-{%-   do kedifa_updater_mapping.append((master_key_download_url, master_certificate, apache_certificate)) %}
+{%-   do kedifa_updater_mapping.append((master_key_download_url, caddy_configuration['master-certificate'], apache_certificate)) %}
 {%- else %}
-{%-   do kedifa_updater_mapping.append(('notreadyyet', master_certificate, apache_certificate)) %}
+{%-   do kedifa_updater_mapping.append(('notreadyyet', caddy_configuration['master-certificate'], apache_certificate)) %}
 {%- endif %}
 {%- if kedifa_configuration['slave_kedifa_information'] %}
 {%-   set slave_kedifa_information = json_module.loads(kedifa_configuration['slave_kedifa_information']) %}
@@ -30,7 +31,7 @@ recipe = slapos.recipe.template:jinja2
 extensions = jinja2.ext.do
 extra-context =
 context =
-    raw common_profile {{ common_profile }}
+    raw profile_common {{ profile_common }}
     ${:extra-context}
 
 # empty sections if no slaves are available
@@ -53,7 +54,7 @@ context =
 {%-   if slave_ciphers %}
 {%-     set slave_cipher_list = ' '.join(slave_ciphers) %}
 {%-   else %}
-{%-     set slave_cipher_list = ciphers.strip() %}
+{%-     set slave_cipher_list = configuration['ciphers'].strip() %}
 {%-   endif %}
 {%-   do slave_instance.__setitem__('cipher_list', slave_cipher_list) %}
 {#-   Manage common instance parameters #}
@@ -94,16 +95,17 @@ context =
 {%-   set slave_publish_dict = {} %}
 {%-   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%-   set slave_logrotate_section = slave_reference + "-logs" %}
+{%-   set slave_log_directory_section = slave_reference + "-log-directory" %}
 {%-   set slave_password_section = slave_reference + "-password" %}
 {%-   set slave_htpasswd_section = slave_reference + "-htpasswd" %}
 {%-   set slave_ln_section = slave_reference + "-ln" %}
 {#-   extend parts #}
 {%-   do part_list.extend([slave_ln_section]) %}
-{%-   do part_list.extend([slave_logrotate_section, slave_section_title]) %}
+{%-   do part_list.extend([slave_section_title]) %}
 {%-   set slave_log_folder = '${logrotate-directory:logrotate-backup}/' + slave_reference + "-logs" %}
 {#-   Pass HTTP2 switch #}
-{%-   do slave_instance.__setitem__('enable_http2_by_default', enable_http2_by_default) %}
-{%-   do slave_instance.__setitem__('global_disable_http2', global_disable_http2) %}
+{%-   do slave_instance.__setitem__('enable_http2_by_default', configuration['enable-http2-by-default']) %}
+{%-   do slave_instance.__setitem__('global_disable_http2', configuration['global-disable-http2']) %}
 {#-   Pass backend timeout values #}
 {%-   for key in ['backend-connect-timeout', 'backend-connect-retries', 'request-timeout', 'authenticate-to-backend'] %}
 {%-     if slave_instance.get(key, '') == '' %}
@@ -128,7 +130,7 @@ context =
 {%-   set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
 {%-   do slave_publish_dict.__setitem__('slave-reference', slave_reference) %}
-{%-   do slave_publish_dict.__setitem__('public-ipv4', public_ipv4) %}
+{%-   do slave_publish_dict.__setitem__('public-ipv4', configuration['public-ipv4']) %}
 {%-   do slave_publish_dict.__setitem__('backend-client-caucase-url', backend_client_caucase_url) %}
 {#-   Set slave domain if none was defined #}
 {%-   if slave_instance.get('custom_domain', None) == None %}
@@ -150,11 +152,15 @@ context =
 {{ slave_reference }} = {{ '${' + slave_htpasswd_section + ':file}' }}
 
 {#-   Set slave logrotate entry #}
+[{{slave_log_directory_section}}]
+recipe = slapos.cookbook:mkdirectory
+log-directory = {{ '${slave-log-directory-dict:' + slave_reference.lower() + '}' }}
+
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
 name = ${:_buildout_section_name_}
 log = {{slave_parameter_dict.get('access_log')}} {{slave_parameter_dict.get('error_log')}} {{slave_parameter_dict.get('backend_log')}}
-backup = {{ slave_log_folder }}
+backup = {{ '${' + slave_log_directory_section + ':log-directory}' }}
 rotate-num = {{ dumps('' ~ configuration['rotate-num']) }}
 # disable delayed compression, as log filenames shall be stable
 delaycompress =
@@ -164,8 +170,8 @@ delaycompress =
 [{{slave_ln_section}}]
 recipe = plone.recipe.command
 stop-on-error = false
-update-command = ${:command}
-command = ln -sf {{slave_parameter_dict.get('error_log')}} {{ slave_log_folder }}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} {{ slave_log_folder }}/access.log && ln -sf {{slave_parameter_dict.get('backend_log')}} {{ slave_log_folder }}/backend.log
+log-directory = {{ '${' + slave_logrotate_section + ':backup}' }}
+command = ln -sf {{slave_parameter_dict.get('error_log')}} ${:log-directory}/error.log && ln -sf {{slave_parameter_dict.get('access_log')}} ${:log-directory}/access.log && ln -sf {{slave_parameter_dict.get('backend_log')}} ${:log-directory}/backend.log
 
 {#-   Set password for slave #}
 
@@ -176,10 +182,12 @@ bytes = 8
 
 [{{ slave_htpasswd_section }}]
 recipe = plone.recipe.command
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 file = {{ caddy_configuration_directory }}/.{{ slave_reference }}.htpasswd
-command = {{ frontend_configuration['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} {{ '${' + slave_password_section + ':passwd}' }}
-update-command = ${:command}
+{#- update-command is not needed, as if the ${:password} would change, the whole part will be recalculated #}
+password = {{ '${' + slave_password_section + ':passwd}' }}
+command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_reference.lower() }} ${:password}
 
 {#-   ################################################## #}
 {#-   Set Slave Certificates if needed                   #}
@@ -224,7 +232,7 @@ cert-content = {{ dumps(slave_instance.get('ssl_crt') + '\n' + slave_instance.ge
 extra-context =
     key content :cert-content
 {%-   else %}
-{%-     do kedifa_updater_mapping.append((key_download_url, certificate, master_certificate)) %}
+{%-     do kedifa_updater_mapping.append((key_download_url, certificate, caddy_configuration['master-certificate'])) %}
 {%-   endif %}
 {#-   BBB: SlapOS Master non-zero knowledge END #}
 
@@ -233,9 +241,9 @@ extra-context =
 
 [{{ slave_configuration_section_name }}]
 certificate = {{ certificate }}
-https_port = {{ dumps('' ~ https_port) }}
-http_port = {{ dumps('' ~ http_port) }}
-local_ipv4 = {{ dumps('' ~ local_ipv4) }}
+https_port = {{ dumps('' ~ configuration['port']) }}
+http_port = {{ dumps('' ~ configuration['plain_http_port']) }}
+local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
 {%-   for key, value in slave_instance.iteritems() %}
 {%-     if value is not none %}
 {{ key }} = {{ dumps('' ~ value) }}
@@ -283,7 +291,7 @@ config-frequency = 720
 
 {#-   ###############################  #}
 {#-   Publish Slave Information        #}
-{%-   if not extra_slave_instance_list %}
+{%-   if not configuration['extra_slave_instance_list'] %}
 {%-     set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') %}
 {%-     do part_list.append(publish_section_title) %}
 [{{ publish_section_title }}]
@@ -299,12 +307,6 @@ recipe = slapos.cookbook:publish
 {%-  endif %}
 {%- endfor %} {# Slave iteration ends for slave_instance in slave_instance_list #}
 
-[slave-log-directories]
-<= slave-log-directory-dict
-recipe = slapos.cookbook:mkdirectory
-
-{%- do part_list.append('slave-log-directories') %}
-
 {%- do part_list.append('caddy-log-access') %}
 {%- do part_list.append('slave-introspection') %}
 {#- ############################################## #}
@@ -315,36 +317,36 @@ recipe = slapos.cookbook:wrapper
 ipv4 = ${slap-network-information:local-ipv4}
 ipv6 = ${slap-network-information:global-ipv6}
 wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
-command-line = {{ sixtunnel_executable }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
+command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [tunnel-6to4-base-http_port]
 <= tunnel-6to4-base
-ipv4-port = {{ http_port }}
-ipv6-port = {{ http_port }}
+ipv4-port = {{ configuration['plain_http_port'] }}
+ipv6-port = {{ configuration['plain_http_port'] }}
 
 [tunnel-6to4-base-https_port]
 <= tunnel-6to4-base
-ipv4-port = {{ https_port }}
-ipv6-port = {{ https_port }}
+ipv4-port = {{ configuration['port'] }}
+ipv6-port = {{ configuration['port'] }}
 
 {#- Define log access #}
 
 [caddy-log-access-parameters]
 caddy_log_directory = {{ dumps(caddy_log_directory) }}
 caddy_configuration_directory = {{ dumps(caddy_configuration_directory) }}
-local_ipv4 = {{ dumps(local_ipv4) }}
+local_ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
 global_ipv6 = {{ dumps(global_ipv6) }}
-https_port = {{ dumps(https_port) }}
-http_port = {{ dumps(http_port) }}
+https_port = {{ dumps(configuration['port']) }}
+http_port = {{ dumps(configuration['plain_http_port']) }}
 ip_access_certificate = {{ frontend_configuration.get('ip-access-certificate') }}
-access_log = {{ dumps(access_log) }}
-error_log = {{ dumps(error_log) }}
-not_found_file = {{ dumps(not_found_file) }}
+access_log = {{ dumps(caddy_configuration['access-log']) }}
+error_log = {{ dumps(caddy_configuration['error-log']) }}
+not_found_file = {{ dumps(caddy_configuration['not-found-file']) }}
 
 [caddy-log-access]
 < = jinja2-template-base
-template = {{frontend_configuration.get('template-log-access')}}
+template = {{ software_parameter_dict['template_log_access'] }}
 rendered = {{frontend_configuration.get('log-access-configuration')}}
 extra-context =
     section slave_log_directory slave-log-directory-dict
@@ -352,11 +354,11 @@ extra-context =
     section parameter_dict caddy-log-access-parameters
 
 [slave-introspection-parameters]
-local-ipv4 = {{ dumps(local_ipv4) }}
+local-ipv4 = {{ dumps(instance_parameter_dict['ipv4-random']) }}
 global-ipv6 = {{ dumps(global_ipv6) }}
 https-port = {{ frontend_configuration['slave-introspection-https-port'] }}
 ip-access-certificate = {{ frontend_configuration.get('ip-access-certificate') }}
-nginx-mime = {{ frontend_configuration['nginx_mime'] }}
+nginx-mime = {{ software_parameter_dict['nginx_mime'] }}
 access-log = {{ dumps(caddy_configuration['slave-introspection-access-log']) }}
 error-log = {{ dumps(caddy_configuration['slave-introspection-error-log']) }}
 var = {{ directory['slave-introspection-var'] }}
@@ -364,7 +366,7 @@ pid = {{ caddy_configuration['slave-introspection-pid-file'] }}
 
 [slave-introspection-config]
 <= jinja2-template-base
-template = {{ frontend_configuration['slave-introspection-template'] }}
+template = {{ software_parameter_dict['template_slave_introspection_httpd_nginx'] }}
 rendered = {{ frontend_configuration['slave-introspection-configuration'] }}
 extra-context =
     section slave_log_directory slave-log-directory-dict
@@ -373,7 +375,7 @@ extra-context =
 
 [slave-introspection]
 recipe = slapos.cookbook:wrapper
-command-line = {{ frontend_configuration['nginx'] }}
+command-line = {{ software_parameter_dict['nginx'] }}
   -c ${slave-introspection-config:rendered}
 
 wrapper-path = {{ directory['service'] }}/slave-instrospection-nginx
@@ -384,9 +386,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 {#- Publish information for the instance #}
 [publish-caddy-information]
 recipe = slapos.cookbook:publish.serialised
-public-ipv4 = {{ public_ipv4 }}
-private-ipv4 = {{ local_ipv4 }}
-{%- if extra_slave_instance_list %}
+public-ipv4 = {{ configuration['public-ipv4'] }}
+private-ipv4 = {{ instance_parameter_dict['ipv4-random'] }}
+{%- if configuration['extra_slave_instance_list'] %}
 {#-   sort_keys are important in order to avoid shuffling parameters on each run #}
 slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list, sort_keys=True) }}
 {%- endif %}
@@ -404,11 +406,11 @@ backend-haproxy-statistic-url = {{ statistic_url }}
 
 [kedifa-updater]
 recipe = slapos.cookbook:wrapper
-command-line = {{ kedifa_configuration['kedifa-updater'] }}
+command-line = {{ software_parameter_dict['kedifa-updater'] }}
   --server-ca-certificate {{ kedifa_configuration['ca-certificate'] }}
   --identity {{ kedifa_configuration['certificate'] }}
-  --master-certificate {{ master_certificate }}
-  --on-update "{{ frontend_graceful_reload }}"
+  --master-certificate {{ caddy_configuration['master-certificate'] }}
+  --on-update "{{ caddy_configuration['frontend-graceful-command'] }}"
   ${kedifa-updater-mapping:file}
   {{ kedifa_configuration['kedifa-updater-state-file'] }}
 
@@ -417,8 +419,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [kedifa-updater-run]
 recipe = plone.recipe.command
+{#- Can be stopped on error, as does not rely on self provided service but on service which comes from another partition #}
 stop-on-error = True
-command = {{ kedifa_configuration['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ frontend_graceful_reload }}"
+command = {{ software_parameter_dict['kedifa-updater'] }} --prepare-only ${kedifa-updater-mapping:file} --on-update "{{ caddy_configuration['frontend-graceful-command'] }}"
 update-command = ${:command}
 
 [kedifa-updater-mapping]
@@ -452,7 +455,7 @@ extra-context =
 {%- for key, value in backend_haproxy_configuration.items() %}
 {{ key }} = {{ value }}
 {%- endfor %}
-local-ipv4 = {{ dumps('' ~ local_ipv4) }}
+local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
 global-ipv6 = ${slap-network-information:global-ipv6}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
@@ -467,7 +470,7 @@ csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
 stop-on-error = False
 update-command = ${:command}
 command =
-  {{ bin_directory }}/caucase \
+  {{ software_parameter_dict['bin_directory'] }}/caucase \
     --ca-url {{ backend_haproxy_configuration['caucase-url'] }} \
     --ca-crt {{ backend_haproxy_configuration['cas-ca-certificate'] }} \
     --crl {{ backend_haproxy_configuration['crl'] }} \
@@ -479,9 +482,9 @@ command =
 
 [buildout]
 extends =
-  {{ common_profile }}
-  {{ logrotate_base_instance }}
-  {{ monitor_template }}
+  {{ profile_common }}
+  {{ profile_logrotate_base }}
+  {{ profile_monitor }}
 
 parts +=
     kedifa-updater
@@ -511,7 +514,7 @@ csr_work_path = {{ directory['tmp'] }}/${:_buildout_section_name_}
 stop-on-error = False
 update-command = ${:command}
 command =
-  {{ bin_directory }}/caucase \
+  {{ software_parameter_dict['bin_directory'] }}/caucase \
     --ca-url {{ kedifa_configuration['caucase-url'] }} \
     --ca-crt {{ kedifa_configuration['cas-ca-certificate'] }} \
     --crl {{ kedifa_configuration['crl'] }} \
@@ -524,6 +527,7 @@ recipe = plone.recipe.command
 certificate = {{ directory['caddy-csr_id'] }}/certificate.pem
 key = {{ directory['caddy-csr_id'] }}/key.pem
 
+{#- Can be stopped on error, as does not rely on self provided service #}
 stop-on-error = True
 update-command = ${:command}
 command =
@@ -563,7 +567,7 @@ depends =
   ${store-csr_id:command}
   ${store-backend-haproxy-csr_id:command}
 recipe = slapos.cookbook:wrapper
-command-line = {{ caddy_executable }}
+command-line = {{ software_parameter_dict['caddy'] }}
   -conf ${expose-csr_id-template:rendered}
   -log ${expose-csr_id-configuration:error-log}
   -http2=true

software/caddy-frontend/templates/backend-haproxy-rsyslogd.conf.in

@@ -16,7 +16,7 @@ $Umask 0022
 $WorkDirectory {{ configuration['spool-directory'] }}
 
 # Setup logging per slave, by extracting the slave name from the log stream
-{%- set regex = ".*-backend (.*)-http.*" %}
+{%- set regex = ".*-backend (.*)-http.{0,1}/" %}
 template(name="extract_slave_name" type="string" string="%msg:R,ERE,1,FIELD:{{ regex }}--end%")
 set $!slave_name = exec_template("extract_slave_name");
 template(name="slave_output" type="string" string="{{ configuration['caddy-log-directory'] }}/%$!slave_name%_backend_log")

software/caddy-frontend/templates/backend-haproxy.cfg.in

@@ -14,8 +14,10 @@ defaults
   timeout connect {{ configuration['backend-connect-timeout'] }}s
   retries {{ configuration['backend-connect-retries'] }}
 
+{%- set SCHEME_PREFIX_MAPPING = { 'http': 'http_backend', 'https': 'https_backend'} %}
 {%- macro frontend_entry(slave_instance, scheme, wildcard) %}
 {#-   wildcard switch allows to put dangerous entries in the end, as haproxy parses with first match #}
+{%-   if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['hostname'] and slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['port'] %}
 {%-     set host_list = (slave_instance.get('server-alias') or  '').split() %}
 {%-     if slave_instance.get('custom_domain') not in host_list %}
 {%-       do host_list.append(slave_instance.get('custom_domain')) %}
@@ -36,6 +38,7 @@ defaults
 {%-     if matched['count'] > 0 %}
   use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}
 {%-     endif %}
+{%-   endif %}
 {%- endmacro %}
 
 # statistic
@@ -49,46 +52,44 @@ frontend statistic
 
 frontend http-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'http', False) }}
 {%- endfor %}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'http', True) }}
 {%- endfor %}
 
 frontend https-backend
   bind {{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'https', False) }}
 {%- endfor %}
-{%- for slave_instance in backend_slave_list %}
+{%- for slave_instance in backend_slave_list -%}
 {{ frontend_entry(slave_instance, 'https', True) }}
 {%- endfor %}
 
 {%- for slave_instance in backend_slave_list %}
-{%-   for (scheme, prefix) in [('http', 'http_backend'), ('https', 'https_backend')] %}
+{%-   for (scheme, prefix) in SCHEME_PREFIX_MAPPING.items() %}
 {%-     set info_dict = slave_instance[prefix] %}
+{%-     if info_dict['hostname'] and info_dict['port'] %}
+{%-       set ssl_list = [] %}
 {%-       if info_dict['scheme'] == 'https' %}
-{%-       set ssl = [] %}
 {%-         if slave_instance['authenticate-to-backend'] %}
-{%-         set ssl = ['crt %s' % (configuration['certificate'],)] %}
+{%-           do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
 {%-         endif %}
-{%-       do ssl.append('ssl verify') %}
+{%-         do ssl_list.append('ssl verify') %}
 {%-         set path_to_ssl_proxy_ca_crt = slave_instance.get('path_to_ssl_proxy_ca_crt') %}
 {%-         if slave_instance['ssl_proxy_verify'] %}
 {%-           if path_to_ssl_proxy_ca_crt %}
-{%-           do ssl.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
+{%-             do ssl_list.append('required ca-file %s' % (path_to_ssl_proxy_ca_crt,)) %}
 {%-           else %}
 {#-           Backend SSL shall be verified, but not CA provided, disallow connection #}
 {#-           Simply dropping hostname from the dict will result with ignoring it... #}
 {%-           do info_dict.__setitem__('hostname', '') %}
 {%-           endif %}
 {%-         else %}
-{%-         do ssl.append('none') %}
+{%-           do ssl_list.append('none') %}
 {%-         endif %}
-{%-       set ssl = ' '.join(ssl) %}
-{%-     else %}
-{%-       set ssl = '' %}
 {%-       endif %}
 
 backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
@@ -99,10 +100,11 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
   timeout server {{ slave_instance['request-timeout'] }}s
   timeout connect {{ slave_instance['backend-connect-timeout'] }}s
   retries {{ slave_instance['backend-connect-retries'] }}
-  server backend {{ hostname }}:{{ port }} {{ ssl }}
+  server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
 {%-         if path %}
   http-request set-path {{ path }}%[path]
 {%-         endif %}
 {%-       endif %}
+{%-     endif %}
 {%-   endfor %}
 {%- endfor %}

software/caddy-frontend/templates/caddy-backend-url-validator.in

-#!${dash:location}/bin/dash
-config="https://example.com {\n  proxy / $1 {\n  }\n}"
-
-echo -e $config | ${caddy:output} -conf stdin -validate > /dev/null 2>&1

software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in

@@ -75,7 +75,7 @@ log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
 {% endfor %}
 
 [buildout]
-extends = {{ common_profile }}
+extends = {{ profile_common }}
 parts =
 {% for part in part_list %}
 {{ '  %s' % part }}

software/caddy-frontend/test/test.py

@@ -48,7 +48,6 @@ from slapos.recipe.librecipe import generateHashFromFiles
 import xml.etree.ElementTree as ET
 import urlparse
 import socket
-import sqlite3
 
 
 try:
@@ -358,7 +357,7 @@ class TestDataMixin(object):
         [backend_haproxy_wrapper_path] + hash_file_list
       )
     for rejected_slave_publish_path in glob.glob(os.path.join(
-      self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave')):
+      self.instance_path, '*', 'etc', 'nginx-rejected-slave.conf')):
       partition_id = rejected_slave_publish_path.split('/')[-3]
       rejected_slave_pem_path = os.path.join(
         self.instance_path, partition_id, 'etc', 'rejected-slave.pem')
@@ -1772,7 +1771,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+ ' \
                  r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2}.\d{3}\] ' \
-                 r'http-backend _Url-http\/backend ' \
+                 r'http-backend _Url-http\/_Url-backend ' \
                  r'\d+/\d+\/\d+\/\d+\/\d+ ' \
                  r'200 \d+ - - ---- ' \
                  r'\d\/\d\/\d\/\d\/\d \d\/\d ' \
@@ -1801,15 +1800,18 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
       self.instance_path, '*', 'etc', 'backend-haproxy.cfg'))[0]
     with open(backend_configuration_file) as fh:
       content = fh.read()
-      self.assertTrue("""backend _Url-http
+    self.assertIn("""backend _Url-http
   timeout server 12s
   timeout connect 5s
-  retries 3""" in content)
-      self.assertTrue("""  timeout queue 60s
+  retries 3""", content)
+    self.assertIn("""  timeout queue 60s
   timeout server 12s
   timeout client 12s
   timeout connect 5s
-  retries 3""" in content)
+  retries 3""", content)
+    # check that no needless entries are generated
+    self.assertIn("backend _Url-http\n", content)
+    self.assertNotIn("backend _Url-https\n", content)
 
   def test_auth_to_backend(self):
     parameter_dict = self.assertSlaveBase('auth-to-backend')
@@ -6787,14 +6789,34 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
 
   def test(self):
     self.instance_parameter_dict.update({
+      # master partition parameters
       '-frontend-quantity': 3,
       '-sla-2-computer_guid': self.slap._computer_id,
+      '-sla-3-computer_guid': self.slap._computer_id,
       '-frontend-2-state': 'stopped',
       '-frontend-2-software-release-url': self.frontend_2_sr,
-      '-sla-3-computer_guid': self.slap._computer_id,
       '-frontend-3-state': 'stopped',
       '-frontend-3-software-release-url': self.frontend_3_sr,
       '-kedifa-software-release-url': self.kedifa_sr,
+      'automatic-internal-kedifa-caucase-csr': False,
+      'automatic-internal-backend-client-caucase-csr': False,
+      # all nodes partition parameters
+      'apache-certificate': self.certificate_pem,
+      'apache-key': self.key_pem,
+      'domain': 'example.com',
+      'enable-http2-by-default': True,
+      'global-disable-http2': True,
+      'mpm-graceful-shutdown-timeout': 2,
+      'public-ipv4': '255.255.255.255',
+      're6st-verification-url': 're6st-verification-url',
+      'backend-connect-timeout': 2,
+      'backend-connect-retries': 1,
+      'ciphers': 'ciphers',
+      'request-timeout': 100,
+      'authenticate-to-backend': True,
+      # specific parameters
+      '-frontend-config-1-ram-cache-size': '512K',
+      '-frontend-config-2-ram-cache-size': '256K',
     })
 
     # re-request instance with updated parameters
@@ -6806,43 +6828,196 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
     except Exception:
       pass
 
-    # inspect slapproxy, that the master correctly requested other partitions
-    sqlitedb_file = os.path.join(
-      os.path.abspath(
-        os.path.join(
-          self.slap.instance_directory, os.pardir
-        )
-      ), 'var', 'proxy.db'
-    )
-    connection = sqlite3.connect(sqlitedb_file)
-
-    def dict_factory(cursor, row):
-      d = {}
-      for idx, col in enumerate(cursor.description):
-        d[col[0]] = row[idx]
-      return d
-    connection.row_factory = dict_factory
-    cursor = connection.cursor()
-
-    cursor.execute(
-      "select partition_reference, software_release "
-      "from partition14 where slap_state='busy';")
-
-    requested_partition_information = cursor.fetchall()
+    computer = self.slap._slap.registerComputer('local')
+    # state of parameters of all instances
+    partition_parameter_dict_dict = {}
+    for partition in computer.getComputerPartitionList():
+      if partition.getState() == 'destroyed':
+        continue
+      parameter_dict = partition.getInstanceParameterDict()
+      instance_title = parameter_dict['instance_title']
+      if '_' in parameter_dict:
+        # "flatten" the instance parameter
+        parameter_dict = json.loads(parameter_dict['_'])
+      partition_parameter_dict_dict[instance_title] = parameter_dict
+      parameter_dict[
+        'X-software_release_url'] = partition.getSoftwareRelease().getURI()
 
     base_software_url = self.getSoftwareURL()
+
+    # drop some very varying parameters
+    def assertKeyWithPop(d, k):
+      self.assertIn(k, d)
+      d.pop(k)
+    assertKeyWithPop(
+      partition_parameter_dict_dict['caddy-frontend-1'],
+      'master-key-download-url')
+    assertKeyWithPop(
+      partition_parameter_dict_dict['caddy-frontend-2'],
+      'master-key-download-url')
+    assertKeyWithPop(
+      partition_parameter_dict_dict['caddy-frontend-3'],
+      'master-key-download-url')
+    assertKeyWithPop(
+      partition_parameter_dict_dict['testing partition 0'],
+      'timestamp')
+    assertKeyWithPop(
+      partition_parameter_dict_dict['testing partition 0'],
+      'ip_list')
+
+    monitor_password = partition_parameter_dict_dict[
+      'caddy-frontend-1'].pop('monitor-password')
+    self.assertEqual(
+      monitor_password,
+      partition_parameter_dict_dict[
+        'caddy-frontend-2'].pop('monitor-password')
+    )
+    self.assertEqual(
+      monitor_password,
+      partition_parameter_dict_dict[
+        'caddy-frontend-3'].pop('monitor-password')
+    )
+    self.assertEqual(
+      monitor_password,
+      partition_parameter_dict_dict[
+        'kedifa'].pop('monitor-password')
+    )
+
+    backend_client_caucase_url = u'http://[%s]:8990' % (self._ipv6_address,)
+    kedifa_caucase_url = u'http://[%s]:15090' % (self._ipv6_address,)
+    expected_partition_parameter_dict_dict = {
+      'caddy-frontend-1': {
+        'X-software_release_url': base_software_url,
+        u'apache-certificate': unicode(self.certificate_pem),
+        u'apache-key': unicode(self.key_pem),
+        u'authenticate-to-backend': u'True',
+        u'backend-client-caucase-url': backend_client_caucase_url,
+        u'backend-connect-retries': u'1',
+        u'backend-connect-timeout': u'2',
+        u'ciphers': u'ciphers',
+        u'cluster-identification': u'testing partition 0',
+        u'domain': u'example.com',
+        u'enable-http2-by-default': u'True',
+        u'extra_slave_instance_list': u'[]',
+        u'frontend-name': u'caddy-frontend-1',
+        u'global-disable-http2': u'True',
+        u'kedifa-caucase-url': kedifa_caucase_url,
+        u'monitor-cors-domains': u'monitor.app.officejs.com',
+        u'monitor-httpd-port': 8411,
+        u'monitor-username': u'admin',
+        u'mpm-graceful-shutdown-timeout': u'2',
+        u'plain_http_port': '11080',
+        u'port': '11443',
+        u'public-ipv4': u'255.255.255.255',
+        u'ram-cache-size': u'512K',
+        u're6st-verification-url': u're6st-verification-url',
+        u'request-timeout': u'100',
+        u'slave-kedifa-information': u'{}'
+      },
+      'caddy-frontend-2': {
+        'X-software_release_url': self.frontend_2_sr,
+        u'apache-certificate': unicode(self.certificate_pem),
+        u'apache-key': unicode(self.key_pem),
+        u'authenticate-to-backend': u'True',
+        u'backend-client-caucase-url': backend_client_caucase_url,
+        u'backend-connect-retries': u'1',
+        u'backend-connect-timeout': u'2',
+        u'ciphers': u'ciphers',
+        u'cluster-identification': u'testing partition 0',
+        u'domain': u'example.com',
+        u'enable-http2-by-default': u'True',
+        u'extra_slave_instance_list': u'[]',
+        u'frontend-name': u'caddy-frontend-2',
+        u'global-disable-http2': u'True',
+        u'kedifa-caucase-url': kedifa_caucase_url,
+        u'monitor-cors-domains': u'monitor.app.officejs.com',
+        u'monitor-httpd-port': 8412,
+        u'monitor-username': u'admin',
+        u'mpm-graceful-shutdown-timeout': u'2',
+        u'plain_http_port': u'11080',
+        u'port': u'11443',
+        u'public-ipv4': u'255.255.255.255',
+        u'ram-cache-size': u'256K',
+        u're6st-verification-url': u're6st-verification-url',
+        u'request-timeout': u'100',
+        u'slave-kedifa-information': u'{}'
+      },
+      'caddy-frontend-3': {
+        'X-software_release_url': self.frontend_3_sr,
+        u'apache-certificate': unicode(self.certificate_pem),
+        u'apache-key': unicode(self.key_pem),
+        u'authenticate-to-backend': u'True',
+        u'backend-client-caucase-url': backend_client_caucase_url,
+        u'backend-connect-retries': u'1',
+        u'backend-connect-timeout': u'2',
+        u'ciphers': u'ciphers',
+        u'cluster-identification': u'testing partition 0',
+        u'domain': u'example.com',
+        u'enable-http2-by-default': u'True',
+        u'extra_slave_instance_list': u'[]',
+        u'frontend-name': u'caddy-frontend-3',
+        u'global-disable-http2': u'True',
+        u'kedifa-caucase-url': kedifa_caucase_url,
+        u'monitor-cors-domains': u'monitor.app.officejs.com',
+        u'monitor-httpd-port': 8413,
+        u'monitor-username': u'admin',
+        u'mpm-graceful-shutdown-timeout': u'2',
+        u'plain_http_port': u'11080',
+        u'port': u'11443',
+        u'public-ipv4': u'255.255.255.255',
+        u're6st-verification-url': u're6st-verification-url',
+        u'request-timeout': u'100',
+        u'slave-kedifa-information': u'{}'
+      },
+      'kedifa': {
+        'X-software_release_url': self.kedifa_sr,
+        u'caucase_port': u'15090',
+        u'cluster-identification': u'testing partition 0',
+        u'kedifa_port': u'15080',
+        u'monitor-cors-domains': u'monitor.app.officejs.com',
+        u'monitor-httpd-port': u'8402',
+        u'monitor-username': u'admin',
+        u'slave-list': []
+      },
+      'testing partition 0': {
+        '-frontend-2-software-release-url': self.frontend_2_sr,
+        '-frontend-2-state': 'stopped',
+        '-frontend-3-software-release-url': self.frontend_3_sr,
+        '-frontend-3-state': 'stopped',
+        '-frontend-config-1-ram-cache-size': '512K',
+        '-frontend-config-2-ram-cache-size': '256K',
+        '-frontend-quantity': '3',
+        '-kedifa-software-release-url': self.kedifa_sr,
+        '-sla-2-computer_guid': 'local',
+        '-sla-3-computer_guid': 'local',
+        'X-software_release_url': base_software_url,
+        'apache-certificate': unicode(self.certificate_pem),
+        'apache-key': unicode(self.key_pem),
+        'authenticate-to-backend': 'True',
+        'automatic-internal-backend-client-caucase-csr': 'False',
+        'automatic-internal-kedifa-caucase-csr': 'False',
+        'backend-connect-retries': '1',
+        'backend-connect-timeout': '2',
+        'caucase_port': '15090',
+        'ciphers': 'ciphers',
+        'domain': 'example.com',
+        'enable-http2-by-default': 'True',
+        'full_address_list': [],
+        'global-disable-http2': 'True',
+        'instance_title': 'testing partition 0',
+        'kedifa_port': '15080',
+        'mpm-graceful-shutdown-timeout': '2',
+        'plain_http_port': '11080',
+        'port': '11443',
+        'public-ipv4': '255.255.255.255',
+        're6st-verification-url': 're6st-verification-url',
+        'request-timeout': '100',
+        'root_instance_title': 'testing partition 0',
+        'slap_software_type': 'RootSoftwareInstance',
+        'slave_instance_list': []
+      }
+    }
     self.assertEqual(
-      requested_partition_information,
-      [
-        {'software_release': base_software_url,
-         'partition_reference': 'testing partition 0'},
-        {'software_release': self.kedifa_sr,
-         'partition_reference': 'kedifa'},
-        # that one is base, as expected
-        {'software_release': base_software_url,
-         'partition_reference': 'caddy-frontend-1'},
-        {'software_release': self.frontend_2_sr,
-         'partition_reference': 'caddy-frontend-2'},
-        {'software_release': self.frontend_3_sr,
-         'partition_reference': 'caddy-frontend-3'}]
+      expected_partition_parameter_dict_dict,
+      partition_parameter_dict_dict
     )

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlave.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlave.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -13,13 +13,10 @@ T-2/var/log/httpd/_dummy-cached_access_log
 T-2/var/log/httpd/_dummy-cached_backend_log
 T-2/var/log/httpd/_dummy-cached_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
-T-2/var/log/httpd/_enable-http2-default_backend_log
 T-2/var/log/httpd/_enable-http2-default_error_log
 T-2/var/log/httpd/_enable-http2-false_access_log
-T-2/var/log/httpd/_enable-http2-false_backend_log
 T-2/var/log/httpd/_enable-http2-false_error_log
 T-2/var/log/httpd/_enable-http2-true_access_log
-T-2/var/log/httpd/_enable-http2-true_backend_log
 T-2/var/log/httpd/_enable-http2-true_error_log
 T-2/var/log/monitor-httpd-access.log
 T-2/var/log/monitor-httpd-error.log

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
 T-2/var/log/httpd/_auth-to-backend_backend_log
 T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
-T-2/var/log/httpd/_ciphers_backend_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log
 T-2/var/log/httpd/_custom_domain_backend_log
@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
 T-2/var/log/httpd/_disabled-cookie-list_backend_log
 T-2/var/log/httpd/_disabled-cookie-list_error_log
 T-2/var/log/httpd/_empty_access_log
-T-2/var/log/httpd/_empty_backend_log
 T-2/var/log/httpd/_empty_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
 T-2/var/log/httpd/_enable-http2-default_backend_log
@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
-T-2/var/log/httpd/_monitor-ipv4-test_backend_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
-T-2/var/log/httpd/_monitor-ipv6-test_backend_log
 T-2/var/log/httpd/_monitor-ipv6-test_error_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -22,7 +22,6 @@ T-2/var/log/httpd/_auth-to-backend_access_log
 T-2/var/log/httpd/_auth-to-backend_backend_log
 T-2/var/log/httpd/_auth-to-backend_error_log
 T-2/var/log/httpd/_ciphers_access_log
-T-2/var/log/httpd/_ciphers_backend_log
 T-2/var/log/httpd/_ciphers_error_log
 T-2/var/log/httpd/_custom_domain_access_log
 T-2/var/log/httpd/_custom_domain_backend_log
@@ -43,7 +42,6 @@ T-2/var/log/httpd/_disabled-cookie-list_access_log
 T-2/var/log/httpd/_disabled-cookie-list_backend_log
 T-2/var/log/httpd/_disabled-cookie-list_error_log
 T-2/var/log/httpd/_empty_access_log
-T-2/var/log/httpd/_empty_backend_log
 T-2/var/log/httpd/_empty_error_log
 T-2/var/log/httpd/_enable-http2-default_access_log
 T-2/var/log/httpd/_enable-http2-default_backend_log
@@ -79,10 +77,8 @@ T-2/var/log/httpd/_https-only_access_log
 T-2/var/log/httpd/_https-only_backend_log
 T-2/var/log/httpd/_https-only_error_log
 T-2/var/log/httpd/_monitor-ipv4-test_access_log
-T-2/var/log/httpd/_monitor-ipv4-test_backend_log
 T-2/var/log/httpd/_monitor-ipv4-test_error_log
 T-2/var/log/httpd/_monitor-ipv6-test_access_log
-T-2/var/log/httpd/_monitor-ipv6-test_backend_log
 T-2/var/log/httpd/_monitor-ipv6-test_error_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_access_log
 T-2/var/log/httpd/_prefer-gzip-encoding-to-backend-https-only_backend_log

software/erp5/test/setup.py

@@ -33,7 +33,7 @@ with open("README.md") as f:
 
 setup(name=name,
       version=version,
-      description="Test for SlapOS' ERP5 software releae",
+      description="Test for SlapOS' ERP5 software release",
       long_description=long_description,
       long_description_content_type='text/markdown',
       maintainer="Nexedi",
@@ -50,8 +50,9 @@ setup(name=name,
         'mysqlclient',
         'backports.lzma',
         'cryptography',
+        'pexpect',
         'pyOpenSSL',
+        'typing; python_version<"3"',
       ],
-      zip_safe=True,
       test_suite='test',
     )

software/erp5/test/test/test_balancer.py

-from . import ERP5InstanceTestCase
-from . import setUpModule
-from slapos.testing.utils import findFreeTCPPort
-
-from BaseHTTPServer import HTTPServer
-from BaseHTTPServer import BaseHTTPRequestHandler
-import OpenSSL.SSL
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization, hashes
-from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography import x509
-from cryptography.x509.oid import NameOID
+import glob
 import hashlib
 import json
-import multiprocessing
+import logging
 import os
-import requests
+import re
 import shutil
+import socket
 import subprocess
 import tempfile
 import time
+import urlparse
+from BaseHTTPServer import BaseHTTPRequestHandler
+from typing import Any, Dict, Optional
+
+import idna
+import mock
+import OpenSSL.SSL
+import pexpect
+import psutil
+import requests
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from slapos.testing.testcase import ManagedResource
+from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
+                                  findFreeTCPPort)
+
+from . import ERP5InstanceTestCase, setUpModule
 
 setUpModule  # pyflakes
 
-class TestHandler(BaseHTTPRequestHandler):
+
+class EchoHTTPServer(ManagedHTTPServer):
+  """An HTTP Server responding with the request path and incoming headers,
+  encoded in json.
+  """
+  class RequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):
+      # type: () -> None
       self.send_response(200)
       self.send_header("Content-Type", "application/json")
-    response = {
+      response = json.dumps(
+          {
               'Path': self.path,
               'Incoming Headers': self.headers.dict
-    }
-    response = json.dumps(response, indent=2)
+          },
+          indent=2,
+      )
       self.end_headers()
       self.wfile.write(response)
 
-class TestFrontendXForwardedFor(ERP5InstanceTestCase):
-  __partition_reference__ = 'xff'
-  http_server_process = None
-  frontend_caucase_dir = None
-  frontend_caucased_process = None
-  backend_caucase_dir = None
-  backend_caucased_process = None
+    log_message = logging.getLogger(__name__ + '.HeaderEchoHandler').info
 
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    return 'balancer'
 
-  @classmethod
-  def setUpClass(cls):
-    # start a dummy web server echoing headers.
-    http_server_port = findFreeTCPPort(cls._ipv4_address)
-    server = HTTPServer(
-      (cls._ipv4_address, http_server_port),
-      TestHandler)
-    cls.http_server_process = multiprocessing.Process(
-      target=server.serve_forever, name='HTTPServer')
-    cls.http_server_process.start()
-    cls.http_server_netloc = '%s:%s' % (cls._ipv4_address, http_server_port)
-
-    # start a caucased and generate a valid client certificate.
-    cls.computer_partition_root_path = os.path.abspath(os.curdir)
-    cls.frontend_caucase_dir = tempfile.mkdtemp()
-    frontend_caucased_dir = os.path.join(cls.frontend_caucase_dir, 'caucased')
-    os.mkdir(frontend_caucased_dir)
-    frontend_user_dir = os.path.join(cls.frontend_caucase_dir, 'user')
-    os.mkdir(frontend_user_dir)
-    frontend_service_dir = os.path.join(cls.frontend_caucase_dir, 'service')
-    os.mkdir(frontend_service_dir)
-    frontend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
-    cls.frontend_caucased_url = 'http://' + frontend_caucased_netloc
-
-    cls.user_certificate = frontend_user_key = os.path.join(frontend_user_dir, 'client.key.pem')
-    frontend_user_csr = os.path.join(frontend_user_dir, 'client.csr.pem')
+class CaucaseService(ManagedResource):
+  """A caucase service.
+  """
+  url = None # type: str
+  directory = None # type: str
+  _caucased_process = None # type: subprocess.Popen
 
-    key = rsa.generate_private_key(
-      public_exponent=65537,
-      key_size=2048,
-      backend=default_backend()
+  def open(self):
+    # type: () -> None
+    # start a caucased and server certificate.
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
     )
-    with open(frontend_user_key, 'wb') as f:
-      f.write(key.private_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PrivateFormat.TraditionalOpenSSL,
-        encryption_algorithm=serialization.NoEncryption(),
-      ))
+    caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
 
-    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
-      x509.NameAttribute(NameOID.COMMON_NAME, u'user'),
-    ])).sign(key, hashes.SHA256(), default_backend())
-    with open(frontend_user_csr, 'wb') as f:
-      f.write(csr.public_bytes(serialization.Encoding.PEM))
+    self.directory = tempfile.mkdtemp()
+    caucased_dir = os.path.join(self.directory, 'caucased')
+    os.mkdir(caucased_dir)
+    os.mkdir(os.path.join(caucased_dir, 'user'))
+    os.mkdir(os.path.join(caucased_dir, 'service'))
 
-    cls.software_release_root_path = os.path.join(
-       cls.slap._software_root,
-       hashlib.md5(cls.getSoftwareURL()).hexdigest(),
-    )
-    caucased_path = os.path.join(cls.software_release_root_path, 'bin', 'caucased')
-    caucase_path = os.path.join(cls.software_release_root_path, 'bin', 'caucase')
-    cls.frontend_caucased_process = subprocess.Popen(
+    backend_caucased_netloc = '%s:%s' % (self._cls._ipv4_address, findFreeTCPPort(self._cls._ipv4_address))
+    self.url = 'http://' + backend_caucased_netloc
+    self._caucased_process = subprocess.Popen(
         [
             caucased_path,
-        '--db', os.path.join(frontend_caucased_dir, 'caucase.sqlite'),
-        '--server-key', os.path.join(frontend_caucased_dir, 'server.key.pem'),
-        '--netloc', frontend_caucased_netloc,
+            '--db', os.path.join(caucased_dir, 'caucase.sqlite'),
+            '--server-key', os.path.join(caucased_dir, 'server.key.pem'),
+            '--netloc', backend_caucased_netloc,
             '--service-auto-approve-count', '1',
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
     )
-    for _ in range(10):
+    for _ in range(30):
       try:
-        if requests.get(cls.frontend_caucased_url).status_code == 200:
+        if requests.get(self.url).status_code == 200:
           break
       except Exception:
         pass
@@ -118,173 +100,591 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
     else:
       raise RuntimeError('caucased failed to start.')
 
-    cau_args = [
-      caucase_path,
-      '--ca-url', cls.frontend_caucased_url,
-      '--ca-crt', os.path.join(frontend_user_dir, 'service-ca-crt.pem'),
-      '--crl', os.path.join(frontend_user_dir, 'service.crl'),
-      '--user-ca-crt', os.path.join(frontend_user_dir, 'user-ca-crt.pem'),
-      '--user-crl', os.path.join(frontend_user_dir, 'user.crl'),
-    ]
+  def close(self):
+    # type: () -> None
+    self._caucased_process.terminate()
+    self._caucased_process.wait()
+    shutil.rmtree(self.directory)
 
-    cas_args = [
-      caucase_path,
-      '--ca-url', cls.frontend_caucased_url,
-      '--ca-crt', os.path.join(frontend_service_dir, 'service-ca-crt.pem'),
-      '--crl', os.path.join(frontend_service_dir, 'service.crl'),
-      '--user-ca-crt', os.path.join(frontend_service_dir, 'user-ca-crt.pem'),
-      '--user-crl', os.path.join(frontend_service_dir, 'user.crl'),
+  @property
+  def ca_crt_path(self):
+    # type: () -> str
+    """Path of the CA certificate from this caucase.
+    """
+    ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
+    if not os.path.exists(ca_crt_path):
+      with open(ca_crt_path, 'w') as f:
+        f.write(
+            requests.get(urlparse.urljoin(
+                self.url,
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    return ca_crt_path
+
+class BalancerTestCase(ERP5InstanceTestCase):
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'balancer'
+
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    return {
+        'tcpv4-port': 8000,
+        'computer-memory-percent-threshold': 100,
+        # XXX what is this ? should probably not be needed here
+        'name': cls.__name__,
+        'monitor-passwd': 'secret',
+        'apachedex-configuration': '--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet',
+        'apachedex-promise-threshold': 100,
+        'haproxy-server-check-path': '/',
+        'zope-family-dict': {
+            'default': ['dummy_http_server'],
+        },
+        'dummy_http_server': [[cls.getManagedResource("backend_web_server", EchoHTTPServer).netloc, 1, False]],
+        'backend-path-dict': {
+            'default': '',
+        },
+        'ssl-authentication-dict': {},
+        'ssl': {
+            'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
+        }
+      }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    # type: () -> Dict
+    return {'_': json.dumps(cls._getInstanceParameterDict())}
+
+  def setUp(self):
+    self.default_balancer_url = json.loads(
+        self.computer_partition.getConnectionParameterDict()['_'])['default']
+
+
+class SlowHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which reply after 3 seconds.
+  """
+  class RequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+      # type: () -> None
+      self.send_response(200)
+      self.send_header("Content-Type", "text/plain")
+      time.sleep(3)
+      self.end_headers()
+      self.wfile.write("OK\n")
+
+    log_message = logging.getLogger(__name__ + '.SlowHandler').info
+
+
+class TestAccessLog(BalancerTestCase, CrontabMixin):
+  """Check access logs emitted by balancer
+  """
+  __partition_reference__ = 'l'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestAccessLog, cls)._getInstanceParameterDict()
+    # use a slow server instead
+    parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
+    return parameter_dict
+
+  def test_access_log_format(self):
+    # type: () -> None
+    requests.get(
+        urlparse.urljoin(self.default_balancer_url, '/url_path'),
+        verify=False,
+    )
+    with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
+      access_line = access_log_file.read()
+    self.assertIn('/url_path', access_line)
+
+    # last \d is the request time in micro seconds, since this SlowHTTPServer
+    # sleeps for 3 seconds, it should take between 3 and 4 seconds to process
+    # the request - but our test machines can be slow sometimes, so we tolerate
+    # it can take up to 20 seconds.
+    match = re.match(
+        r'([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
+        access_line
+    )
+    self.assertTrue(match)
+    assert match
+    request_time = int(match.groups()[-1])
+    self.assertGreater(request_time, 3 * 1000 * 1000)
+    self.assertLess(request_time, 20 * 1000 * 1000)
+
+  def test_access_log_apachedex_report(self):
+    # type: () -> None
+    # make a request so that we have something in the logs
+    requests.get(self.default_balancer_url, verify=False)
+
+    # crontab for apachedex is executed
+    self._executeCrontabAtDate('generate-apachedex-report', '23:59')
+    # it creates a report for the day
+    apachedex_report, = glob.glob(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'monitor',
+            'private',
+            'apachedex',
+            'ApacheDex-*.html',
+        ))
+    with open(apachedex_report, 'r') as f:
+      report_text = f.read()
+    self.assertIn('APacheDEX', report_text)
+    # having this table means that apachedex could parse some lines.
+    self.assertIn('<h2>Hits per status code</h2>', report_text)
+
+  def test_access_log_rotation(self):
+    # type: () -> None
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    # make a request so that we have something in the logs
+    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+
+    # slow query crontab depends on crontab for log rotation
+    # to be executed first.
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.computer_partition_root_path,
+        'srv',
+        'backup',
+        'logrotate',
+        'apache-access.log-20500101',
+    )
+    self.assertTrue(os.path.exists(rotated_log_file))
+
+    requests.get(self.default_balancer_url, verify=False).raise_for_status()
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+
+class BalancerCookieHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which can set balancer cookie.
+
+  This server set cookie when requested /set-cookie path.
+
+  The reply body is the name used when registering this resource
+  using getManagedResource. This way we can assert which
+  backend replied.
+  """
+
+  @property
+  def RequestHandler(self):
+    server = self
+    class RequestHandler(BaseHTTPRequestHandler):
+      def do_GET(self):
+        # type: () -> None
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        if self.path == '/set_cookie':
+          # the balancer tells the backend what's the name of the balancer cookie with
+          # the X-Balancer-Current-Cookie header.
+          self.send_header('Set-Cookie', '%s=anything' % self.headers['X-Balancer-Current-Cookie'])
+          # The name of this cookie is SERVERID
+          assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
+        self.end_headers()
+        self.wfile.write(server._name)
+      log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
+
+    return RequestHandler
+
+
+class TestBalancer(BalancerTestCase):
+  """Check balancing capabilities
+  """
+  __partition_reference__ = 'b'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
+
+    # use two backend servers
+    parameter_dict['dummy_http_server'] = [
+        [cls.getManagedResource("backend_web_server1", BalancerCookieHTTPServer).netloc, 1, False],
+        [cls.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).netloc, 1, False],
     ]
+    return parameter_dict
 
-    caucase_process = subprocess.Popen(
-      cau_args + [
-        '--mode', 'user',
-        '--send-csr', frontend_user_csr,
-      ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
+  def test_balancer_round_robin(self):
+    # requests are by default balanced to both servers
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {'backend_web_server1', 'backend_web_server2'}
     )
-    result = caucase_process.communicate()
-    csr_id = result[0].split()[0]
 
-    subprocess.check_call(
-      cau_args + [
-        '--mode', 'user',
-        '--get-crt', csr_id, frontend_user_key,
-      ],
+  def test_balancer_server_down(self):
+    # if one backend is down, it is excluded from balancer
+    self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
+    self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
+        {'backend_web_server1',}
+    )
+
+  def test_balancer_set_cookie(self):
+    # if backend provides a "SERVERID" cookie, balancer will overwrite it with the
+    # backend selected by balancing algorithm
+    self.assertIn(
+        requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
+        ('default-0', 'default-1'),
+    )
+
+  def test_balancer_respects_sticky_cookie(self):
+    # if request is made with the sticky cookie, the client stick on one balancer
+    cookies = dict(SERVERID='default-1')
+    self.assertEqual(
+        {requests.get(self.default_balancer_url, verify=False, cookies=cookies).text for _ in range(10)},
+        {'backend_web_server2',}
     )
 
-    cls.client_certificate = frontend_service_key = os.path.join(frontend_service_dir, 'crt.pem')
-    frontend_service_csr = os.path.join(frontend_service_dir, 'csr.pem')
+    # if that backend becomes down, requests are balanced to another server
+    self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
+    self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
+    self.assertEqual(
+        requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
+        'backend_web_server1')
+
+
+class TestHTTP(BalancerTestCase):
+  """Check HTTP protocol
+  """
+  __partition_reference__ = 'h'
+
+  def test_http_version(self):
+    # type: () -> None
+    # https://stackoverflow.com/questions/37012486/python-3-x-how-to-get-http-version-using-requests-library/37012810
+    self.assertEqual(
+        requests.get(self.default_balancer_url, verify=False).raw.version, 11)
+
+  def test_keep_alive(self):
+    # type: () -> None
+    # when doing two requests, connection is established only once
+    session = requests.Session()
+    session.verify = False
+
+    # do a first request, which establish a first connection
+    session.get(self.default_balancer_url).raise_for_status()
+
+    # "break" new connection method and check we can make another request
+    with mock.patch(
+        "requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
+    ) as new_conn:
+      session.get(self.default_balancer_url).raise_for_status()
+    new_conn.assert_not_called()
+
+    parsed_url = urlparse.urlparse(self.default_balancer_url)
+    # check that we have an open file for the ip connection
+    self.assertTrue([
+        c for c in psutil.Process(os.getpid()).connections()
+        if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
+        and c.raddr.port == parsed_url.port
+    ])
+
+
+class TestTLS(BalancerTestCase):
+  """Check TLS
+  """
+  __partition_reference__ = 's'
+
+  def _getServerCertificate(self, hostname, port):
+    # type: (Optional[str], Optional[int]) -> Any
+    hostname_idna = idna.encode(hostname)
+    sock = socket.socket()
+
+    sock.connect((hostname, port))
+    ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
+    ctx.check_hostname = False
+    ctx.verify_mode = OpenSSL.SSL.VERIFY_NONE
+
+    sock_ssl = OpenSSL.SSL.Connection(ctx, sock)
+    sock_ssl.set_connect_state()
+    sock_ssl.set_tlsext_host_name(hostname_idna)
+    sock_ssl.do_handshake()
+    cert = sock_ssl.get_peer_certificate()
+    crypto_cert = cert.to_cryptography()
+    sock_ssl.close()
+    sock.close()
+    return crypto_cert
+
+  def test_certificate_validates_with_caucase_ca(self):
+    # type: () -> None
+    caucase = self.getManagedResource("caucase", CaucaseService)
+    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path)
+
+  def test_certificate_renewal(self):
+    # type: () -> None
+    caucase = self.getManagedResource("caucase", CaucaseService)
+    balancer_parsed_url = urlparse.urlparse(self.default_balancer_url)
+    certificate_before_renewal = self._getServerCertificate(
+        balancer_parsed_url.hostname,
+        balancer_parsed_url.port)
+
+    # run caucase updater 90 days in the future, so that certificate is
+    # renewed.
+    caucase_updater = os.path.join(
+        self.computer_partition_root_path,
+        'etc',
+        'service',
+        'caucase-updater',
+    )
+    process = pexpect.spawnu(
+       "faketime +90days %s" % caucase_updater,
+        env=dict(os.environ, PYTHONPATH=''),
+    )
+    logger = self.logger
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+    process.logfile = DebugLogFile()
+    process.expect(u"Renewing .*\nNext wake-up.*")
+    process.terminate()
+    process.wait()
+
+    # wait for server to use new certificate
+    for _ in range(30):
+      certificate_after_renewal = self._getServerCertificate(
+          balancer_parsed_url.hostname,
+          balancer_parsed_url.port)
+      if certificate_after_renewal.not_valid_before > certificate_before_renewal.not_valid_before:
+        break
+      time.sleep(.5)
+
+    self.assertGreater(
+        certificate_after_renewal.not_valid_before,
+        certificate_before_renewal.not_valid_before,
+    )
+
+    # requests are served properly after cert renewal
+    requests.get(self.default_balancer_url, verify=caucase.ca_crt_path).raise_for_status()
+
+
+class ContentTypeHTTPServer(ManagedHTTPServer):
+  """An HTTP Server which reply with content type from path.
+
+  For example when requested http://host/text/plain it will reply
+  with Content-Type: text/plain header.
+
+  The body is always "OK"
+  """
+
+  class RequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+      # type: () -> None
+      self.send_response(200)
+      if self.path == '/':
+        return self.end_headers()
+      content_type = self.path[1:]
+      self.send_header("Content-Type", content_type)
+      self.end_headers()
+      self.wfile.write("OK")
+
+    log_message = logging.getLogger(__name__ + '.ContentTypeHTTPServer').info
+
+
+class TestContentEncoding(BalancerTestCase):
+  """Test how responses are gzip encoded or not depending on content type header.
+  """
+  __partition_reference__ = 'ce'
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
+    parameter_dict['dummy_http_server'] = [
+        [cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
+    ]
+    return parameter_dict
+
+  def test_gzip_encoding(self):
+    # type: () -> None
+    for content_type in (
+        'text/cache-manifest',
+        'text/html',
+        'text/plain',
+        'text/css',
+        'application/hal+json',
+        'application/json',
+        'application/x-javascript',
+        'text/xml',
+        'application/xml',
+        'application/rss+xml',
+        'text/javascript',
+        'application/javascript',
+        'image/svg+xml',
+        'application/x-font-ttf',
+        'application/font-woff',
+        'application/font-woff2',
+        'application/x-font-opentype',
+        'application/wasm',):
+      resp = requests.get(urlparse.urljoin(self.default_balancer_url, content_type), verify=False)
+      self.assertEqual(resp.headers['Content-Type'], content_type)
+      self.assertEqual(
+          resp.headers['Content-Encoding'],
+          'gzip',
+          '%s uses wrong encoding: %s' % (content_type, resp.headers['Content-Encoding']))
+      self.assertEqual(resp.text, 'OK')
+
+  def test_no_gzip_encoding(self):
+    # type: () -> None
+    resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
+    self.assertNotIn('Content-Encoding', resp.headers)
+    self.assertEqual(resp.text, 'OK')
+
+
+class CaucaseClientCertificate(ManagedResource):
+  """A client certificate issued by a caucase services.
+  """
+
+  ca_crt_file = None # type: str
+  crl_file = None # type: str
+  csr_file = None # type: str
+  cert_file = None # type: str
+  key_file = None # type: str
+
+  def open(self):
+    # type: () -> None
+    self.tmpdir = tempfile.mkdtemp()
+    self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
+    self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
+    self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
+    self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
+    self.key_file = os.path.join(self.tmpdir, 'key.pem')
+
+  def close(self):
+    # type: () -> None
+    shutil.rmtree(self.tmpdir)
+
+  @property
+  def _caucase_path(self):
+    # type: () -> str
+    """path of caucase executable.
+    """
+    software_release_root_path = os.path.join(
+        self._cls.slap._software_root,
+        hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
+    )
+    return os.path.join(software_release_root_path, 'bin', 'caucase')
+
+  def request(self, common_name, caucase):
+    # type: (str, CaucaseService) -> None
+    """Generate certificate and request signature to the caucase service.
+
+    This overwrite any previously requested certificate for this instance.
+    """
+    cas_args = [
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+    ]
 
     key = rsa.generate_private_key(
         public_exponent=65537,
         key_size=2048,
         backend=default_backend()
     )
-    with open(frontend_service_key, 'wb') as f:
-      f.write(key.private_bytes(
+    with open(self.key_file, 'wb') as f:
+      f.write(
+          key.private_bytes(
               encoding=serialization.Encoding.PEM,
               format=serialization.PrivateFormat.TraditionalOpenSSL,
               encryption_algorithm=serialization.NoEncryption(),
           ))
 
-    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
-      x509.NameAttribute(NameOID.COMMON_NAME, u'service'),
-    ])).sign(key, hashes.SHA256(), default_backend())
-    with open(frontend_service_csr, 'wb') as f:
+    csr = x509.CertificateSigningRequestBuilder().subject_name(
+        x509.Name([
+            x509.NameAttribute(
+                NameOID.COMMON_NAME,
+                common_name,
+            ),
+        ])).sign(
+            key,
+            hashes.SHA256(),
+            default_backend(),
+        )
+    with open(self.csr_file, 'wb') as f:
       f.write(csr.public_bytes(serialization.Encoding.PEM))
 
-    caucase_process = subprocess.Popen(
+    csr_id = subprocess.check_output(
       cas_args + [
-        '--send-csr', frontend_service_csr,
+          '--send-csr', self.csr_file,
       ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
-    )
-    result = caucase_process.communicate()
-    csr_id = result[0].split()[0]
+    ).split()[0]
+    assert csr_id
 
-    for _ in range(10):
+    for _ in range(30):
       if not subprocess.call(
         cas_args + [
-          '--get-crt', csr_id, frontend_service_key,
+            '--get-crt', csr_id, self.cert_file,
         ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
       ) == 0:
         break
       else:
         time.sleep(1)
     else:
       raise RuntimeError('getting service certificate failed.')
+    with open(self.cert_file) as f:
+      assert 'BEGIN CERTIFICATE' in f.read()
 
-    # start a caucased and server certificate.
-    cls.backend_caucase_dir = tempfile.mkdtemp()
-    backend_caucased_dir = os.path.join(cls.backend_caucase_dir, 'caucased')
-    os.mkdir(backend_caucased_dir)
-    backend_caucased_netloc = '%s:%s' % (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address))
-    cls.backend_caucased_url = 'http://' + backend_caucased_netloc
-    cls.backend_caucased_process = subprocess.Popen(
-      [
-        caucased_path,
-        '--db', os.path.join(backend_caucased_dir, 'caucase.sqlite'),
-        '--server-key', os.path.join(backend_caucased_dir, 'server.key.pem'),
-        '--netloc', backend_caucased_netloc,
-        '--service-auto-approve-count', '1',
-      ],
-      stdout=subprocess.PIPE,
-      stderr=subprocess.STDOUT,
-    )
-    for _ in range(10):
-      try:
-        if requests.get(cls.backend_caucased_url).status_code == 200:
-          break
-      except Exception:
-        pass
-      time.sleep(1)
-    else:
-      raise RuntimeError('caucased failed to start.')
+  def revoke(self, caucase):
+    # type: (str, CaucaseService) -> None
+    """Revoke the client certificate on this caucase instance.
+    """
+    subprocess.check_call([
+        self._caucase_path,
+        '--ca-url', caucase.url,
+        '--ca-crt', self.ca_crt_file,
+        '--crl', self.crl_file,
+        '--revoke-crt', self.cert_file, self.key_file,
+    ])
 
-    super(TestFrontendXForwardedFor, cls).setUpClass()
+
+class TestFrontendXForwardedFor(BalancerTestCase):
+  __partition_reference__ = 'xff'
 
   @classmethod
-  def getInstanceParameterDict(cls):
-    return {
-      '_': json.dumps({
-        'tcpv4-port': 3306,
-        'computer-memory-percent-threshold': 100,
-        # XXX what is this ? should probably not be needed here
-        'name': cls.__name__,
-        'monitor-passwd': 'secret',
-        'apachedex-configuration': '',
-        'apachedex-promise-threshold': 100,
-        'haproxy-server-check-path': '/',
-        'zope-family-dict': {
-          'default': ['dummy_http_server'],
-          'default-auth': ['dummy_http_server'],
-        },
-        'dummy_http_server': [[cls.http_server_netloc, 1, False]],
-        'backend-path-dict': {
-          'default': '/',
-          'default-auth': '/',
-        },
-        'ssl-authentication-dict': {
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
+    certificate = cls.getManagedResource('client_certificate', CaucaseClientCertificate)
+    certificate.request(u'shared frontend', frontend_caucase)
+
+    parameter_dict = super(TestFrontendXForwardedFor, cls)._getInstanceParameterDict()
+    # add another "-auth" backend, that will have ssl-authentication enabled
+    parameter_dict['zope-family-dict']['default-auth'] = ['dummy_http_server']
+    parameter_dict['backend-path-dict']['default-auth'] = '/'
+    parameter_dict['ssl-authentication-dict'] = {
         'default': False,
         'default-auth': True,
-        },
-        'ssl': {
-          'caucase-url': cls.backend_caucased_url,
-          'frontend-caucase-url-list': [cls.frontend_caucased_url],
-        },
-      })
     }
-
-  @classmethod
-  def _cleanup(cls, snapshot_name):
-    if cls.http_server_process:
-      cls.http_server_process.terminate()
-    if cls.frontend_caucased_process:
-      cls.frontend_caucased_process.terminate()
-    if cls.frontend_caucase_dir:
-      shutil.rmtree(cls.frontend_caucase_dir)
-    if cls.backend_caucased_process:
-      cls.backend_caucased_process.terminate()
-    if cls.backend_caucase_dir:
-      shutil.rmtree(cls.backend_caucase_dir)
-
-    super(TestFrontendXForwardedFor, cls)._cleanup(snapshot_name)
+    parameter_dict['ssl']['frontend-caucase-url-list'] = [frontend_caucase.url]
+    return parameter_dict
 
   def test_x_forwarded_for_added_when_verified_connection(self):
+    # type: () -> None
+    client_certificate = self.getManagedResource('client_certificate', CaucaseClientCertificate)
+
     for backend in ('default', 'default-auth'):
       balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])[backend]
       result = requests.get(
         balancer_url,
         headers={'X-Forwarded-For': '1.2.3.4'},
-        cert=self.client_certificate,
+        cert=(client_certificate.cert_file, client_certificate.key_file),
         verify=False,
       ).json()
       self.assertEqual(result['Incoming Headers'].get('x-forwarded-for').split(', ')[0], '1.2.3.4')
 
   def test_x_forwarded_for_stripped_when_not_verified_connection(self):
+    # type: () -> None
     balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default']
     result = requests.get(
       balancer_url,
@@ -299,3 +699,103 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
         headers={'X-Forwarded-For': '1.2.3.4'},
         verify=False,
       )
+
+
+class TestClientTLS(BalancerTestCase):
+  __partition_reference__ = 'c'
+
+  @classmethod
+  def _getInstanceParameterDict(cls):
+    # type: () -> Dict
+    frontend_caucase1 = cls.getManagedResource('frontend_caucase1', CaucaseService)
+    certificate1 = cls.getManagedResource('client_certificate1', CaucaseClientCertificate)
+    certificate1.request(u'client_certificate1', frontend_caucase1)
+
+    frontend_caucase2 = cls.getManagedResource('frontend_caucase2', CaucaseService)
+    certificate2 = cls.getManagedResource('client_certificate2', CaucaseClientCertificate)
+    certificate2.request(u'client_certificate2', frontend_caucase2)
+
+    parameter_dict = super(TestClientTLS, cls)._getInstanceParameterDict()
+    parameter_dict['ssl-authentication-dict'] = {
+        'default': True,
+    }
+    parameter_dict['ssl']['frontend-caucase-url-list'] = [
+        frontend_caucase1.url,
+        frontend_caucase2.url,
+    ]
+    return parameter_dict
+
+  def test_refresh_crl(self):
+    # type: () -> None
+
+    logger = self.logger
+
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from caucase_updater: %s", msg)
+      def flush(self):
+        pass
+
+    for client_certificate_name, caucase_name in (
+        ('client_certificate1', 'frontend_caucase1'),
+        ('client_certificate2', 'frontend_caucase2'),
+    ):
+      client_certificate = self.getManagedResource(client_certificate_name,
+                                                   CaucaseClientCertificate)
+
+      # when client certificate can be authenticated, backend receive the CN of
+      # the client certificate in "remote-user" header
+      def _make_request():
+        return requests.get(
+            self.default_balancer_url,
+            cert=(client_certificate.cert_file, client_certificate.key_file),
+            verify=False,
+        ).json()
+
+      self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
+                       client_certificate_name)
+
+      # when certificate is revoked, updater service should update the CRL
+      # used by balancer from the caucase service used for client certificates
+      # (ie. the one used by frontend).
+      caucase = self.getManagedResource(caucase_name, CaucaseService)
+      client_certificate.revoke(caucase)
+
+      # until the CRL is updated, the client certificate is still accepted.
+      self.assertEqual(_make_request()['Incoming Headers'].get('remote-user'),
+                       client_certificate_name)
+
+      # We have two services, in charge of updating CRL and CA certificates for
+      # each frontend CA
+      caucase_updater_list = glob.glob(
+          os.path.join(
+              self.computer_partition_root_path,
+              'etc',
+              'service',
+              'caucase-updater-*',
+          ))
+      self.assertEqual(len(caucase_updater_list), 2)
+
+      # find the one corresponding to this caucase
+      for caucase_updater_candidate in caucase_updater_list:
+        with open(caucase_updater_candidate) as f:
+          if caucase.url in f.read():
+            caucase_updater = caucase_updater_candidate
+            break
+      else:
+        self.fail("Could not find caucase updater script for %s" % caucase.url)
+
+      # simulate running updater service in the future, to confirm that it fetches
+      # the new CRL and make sure balancer uses that new CRL.
+      process = pexpect.spawnu(
+          "faketime +1day %s" % caucase_updater,
+          env=dict(os.environ, PYTHONPATH=''),
+      )
+
+      process.logfile = DebugLogFile()
+      process.expect(u"Got new CRL.*Next wake-up at.*")
+      process.terminate()
+      process.wait()
+
+      with self.assertRaisesRegexp(Exception, 'certificate revoked'):
+        _make_request()

software/erp5/test/test/test_erp5.py

@@ -31,6 +31,7 @@ import glob
 import urlparse
 import socket
 import time
+import tempfile
 
 import psutil
 import requests
@@ -43,7 +44,7 @@ setUpModule # pyflakes
 class TestPublishedURLIsReachableMixin(object):
   """Mixin that checks that default page of ERP5 is reachable.
   """
-  def _checkERP5IsReachable(self, url):
+  def _checkERP5IsReachable(self, url, verify):
     # What happens is that instanciation just create the services, but does not
     # wait for ERP5 to be initialized. When this test run ERP5 instance is
     # instanciated, but zope is still busy creating the site and haproxy replies
@@ -51,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # erp5 site is not created, with 500 when mysql is not yet reachable, so we
     # retry in a loop until we get a succesful response.
     for i in range(1, 60):
-      r = requests.get(url, verify=False)  # XXX can we get CA from caucase already ?
+      r = requests.get(url, verify=verify)
       if r.status_code != requests.codes.ok:
         delay = i * 2
         self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
@@ -62,19 +63,36 @@ class TestPublishedURLIsReachableMixin(object):
 
     self.assertIn("ERP5", r.text)
 
+  def _getCaucaseServiceCACertificate(self):
+    ca_cert = tempfile.NamedTemporaryFile(
+        prefix="ca.crt.pem",
+        mode="w",
+        delete=False,
+    )
+    ca_cert.write(
+        requests.get(
+            urlparse.urljoin(
+                self.getRootPartitionConnectionParameterDict()['caucase-http-url'],
+                '/cas/crt/ca.crt.pem',
+            )).text)
+    self.addCleanup(os.unlink, ca_cert.name)
+    return ca_cert.name
+
   def test_published_family_default_v6_is_reachable(self):
     """Tests the IPv6 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
   def test_published_family_default_v4_is_reachable(self):
     """Tests the IPv4 URL published by the root partition is reachable.
     """
     param_dict = self.getRootPartitionConnectionParameterDict()
     self._checkERP5IsReachable(
-      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
+      urlparse.urljoin(param_dict['family-default'], param_dict['site-id']),
+      self._getCaucaseServiceCACertificate())
 
 
 class TestDefaultParameters(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):

software/erp5testnode/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
 
 [template-default]
 filename = instance-default.cfg
-md5sum = 40364ff26e9284cea97a58f3cd8c75e3
+md5sum = e553b582cd51db90fcd5634bd055f042

software/erp5testnode/instance-default.cfg

@@ -23,6 +23,7 @@ parts =
 recipe = slapos.cookbook:publish
 url = $${shellinabox-frontend:url}
 frontend-url = $${testnode-frontend:connection-secure_access}
+log-frontend-url = $${testnode-log-frontend:connection-secure_access}
 
 [pwgen]
 recipe = slapos.cookbook:generate.password
@@ -63,6 +64,8 @@ httpd-log-directory = $${basedirectory:log}
 httpd-software-directory = $${directory:software}
 httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
 httpd-key-file = $${rootdirectory:etc}/httpd-private.key
+frontend-url = $${testnode-frontend:connection-secure_access}
+log-frontend-url = $${testnode-log-frontend:connection-secure_access}
 
 configuration-file = $${rootdirectory:etc}/erp5testnode.cfg
 log-file = $${basedirectory:log}/erp5testnode.log
@@ -212,22 +215,31 @@ recipe = collective.recipe.template
 input = inline: **
 output = $${directory:srv}/exporter.exclude
 
-[testnode-frontend]
+[request-shared-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = Test Node Frontend $${testnode:test-node-title}
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-software-access-port}
 config-https-only = true
-#software-type = custom-personal
 return = domain secure_access
 
+[testnode-frontend]
+<= request-shared-frontend
+name = Test Node Frontend $${testnode:test-node-title}
+config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-software-access-port}
+
+[testnode-log-frontend]
+<= request-shared-frontend
+name = Test Node Logs Frontend $${testnode:test-node-title}
+config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port}
+
+
 [promises]
 recipe =
 instance-promises =
   $${shellinabox-frontend-listen-promise:name}
+  $${testnode-log-frontend-promise:name}
 
 [shellinabox-frontend-listen-promise]
 <= monitor-promise-base
@@ -236,6 +248,12 @@ name = $${:_buildout_section_name_}.py
 config-hostname = $${shellinabox-frontend:hostname}
 config-port = $${shellinabox-frontend:port}
 
+[testnode-log-frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${testnode-log-frontend:connection-secure_access}
+
 [slap-parameter]
 node-quantity = 1
 test-suite-master-url =

software/fluentd/software.cfg

@@ -24,4 +24,3 @@ gems +=
 
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe = 0.2.2+slapos001

software/gitlab/software.cfg

@@ -405,6 +405,5 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-rubygemsrecipe = 0.2.2+slapos002
 slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/helloworld/software.cfg

@@ -29,6 +29,9 @@ parts =
   helloweb-go
 
 
+[python]
+part = python3
+
 # Macro for jinja templates. The filename is set in buildout.hash.cfg
 # in the section using this template
 [jinja-template]
@@ -54,4 +57,3 @@ context =
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe  = 0.2.2+slapos001

software/kvm/buildout.hash.cfg

@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = 4a6f149177a453a13436f320f6841518
+md5sum = a97b8462955dd422a30fbe02d6906172
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -505,9 +505,9 @@
             },
             "boot-image-url-list": {
               "title": "Boot image list",
-              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+              "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
               "type": "string",
-              "textarea": "true"
+              "textarea": true
             },
             "boot-image-url-select": {
               "title": "Boot image",

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -60,19 +60,53 @@
               ],
               "default": "fr"
             },
-            "nbd-host": {
-              "title": "NBD hostname or IP",
-              "description": "hostname (or IP) of the NBD server containing the boot image.",
-              "type": "string",
-              "format": "internet-address"
+            "boot-image-url-select": {
+              "title": "Boot image",
+              "type": "array",
+              "oneOf": [
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/0a6aee1d9aafc1ed095105c052f9fdd65ed00ea9274188c9cd0072c8e6838ab40e246d45a1e6956d74ef1b04a1fc042151762f25412e9ff0cbf49418eef7992e#a3ebc76aec372808ad80000108a2593a"
+                  ],
+                  "title": "Debian Buster 10.5 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/ce5ddfdbdaccdf929b7fe321212356347d82a02f6b7733427282b416f113d91e587682b003e9d376ac189c3b731595c50c236962aadf2720c16d9f36913577c0#23bf2a2d60271e553e63525e794415f1"
+                  ],
+                  "title": "Centos 8.2004 Minimal x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/302c990c6d69575ff24c96566e5c7e26bf36908abb0cd546e22687c46fb07bf8dba595bf77a9d4fd9ab63e75c0437c133f35462fd41ea77f6f616140cd0e5e6a#f3a306f40e4a313fb5a584d73b3dee8f"
+                  ],
+                  "title": "Ubuntu Focal 20.04.1 Live Server x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6635269a7eb6fbd6b85fda40cd94f14a27bf53cb1fc82ffcce9fe386a025a43e1ab681db7e8cec50416bfbfc90262f0d95273686a101c74b3f17646f0a34c85b#3708a59af6cf820a95cafe0ae73ac399"
+                  ],
+                  "title": "openSUSE Leap 15.2 NET x86_64"
                 },
-            "nbd-port": {
-              "title": "NBD port",
-              "description": "Port of the NBD server containing the boot image.",
-              "type": "integer",
-              "default": 1024,
-              "minimum": 1,
-              "maximum": 65535
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/fc17e8c6ae0790162f4beb8fa6226d945cff638429588999b3a08493ff27b280dc2939fba825ae04be1d9082ea8d7c3c002c5e4c39fbbcf88b8ab5104619e28a#ebcdb2223a77f098af3923fe1fa180aa"
+                  ],
+                  "title": "Arch Linux 2020.09.01 x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/c5a511f349a1146b615e6fab9c24f9be4362046adcf24f0ff82c470d361fac5f6628895e2110ebf8ff87db49d4c413a0a332699da6b1bec64275e0c17a15b999#ca7a1e555c04b4d9a549065fa2ddf713"
+                  ],
+                  "title": "Fedora Server 32-1.6 netinst x86_64"
+                },
+                {
+                  "const": [
+                    "https://shacache.nxdcdn.com/6c355def68b3c0427f21598cb054ffc893568902f205601ac60f192854769b31bc9cff8eeb6ce99ef975a8fb887d8d3e56fc6cd5ea5cb4b3bba1175c520047cb#57088b77f795ca44b00971e44782ee23"
+                  ],
+                  "title": "FreeBSD 12.1 RELEASE bootonly x86_64"
+                }
+              ]
             }
           },
           "type": "object"

software/kvm/instance-kvm-input-schema.json

@@ -368,9 +368,9 @@
     },
     "boot-image-url-list": {
       "title": "Boot image list",
-      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
+      "description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM. Note: Maximum 3 ISOs are supported.",
       "type": "string",
-      "textarea": "true"
+      "textarea": true
     },
     "boot-image-url-select": {
       "title": "Boot image",

software/kvm/template/template-kvm-run.in

@@ -371,7 +371,8 @@ for nbd_ip, nbd_port in nbd_list:
           '-drive',
           'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
 else:
-  index = 0
+  # Note: Do not get tempted to use virtio-scsi-pci, as it does not work with
+  #       Debian installation CDs, rendering it uninstallable
   if boot_image_url_select_json_config:
     # Support boot-image-url-select
     with open(boot_image_url_select_json_config) as fh:
@@ -381,11 +382,9 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   if boot_image_url_list_json_config:
     # Support boot-image-url-list
     with open(boot_image_url_list_json_config) as fh:
@@ -395,16 +394,12 @@ else:
         link = os.path.join(image_config['destination-directory'], image['link'])
         if os.path.exists(link) and os.path.islink(link):
           kvm_argument_list.extend([
-            '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (link, index),
-            '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-            '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+            '-drive',
+            'file=%s,media=cdrom' % (link,)
           ])
-          index += 1
   # Always add by default the default image
   kvm_argument_list.extend([
-      '-drive', 'file=%s,media=cdrom,if=none,id=cdrom%s' % (default_cdrom_iso, index),
-      '-device', 'virtio-scsi-pci,id=scsi%s' % (index,),
-      '-device', 'scsi-cd,bus=scsi%s.0,drive=cdrom%s' % (index, index)
+      '-drive', 'file=%s,media=cdrom' % default_cdrom_iso
   ])
 
 

software/kvm/test/test.py

@@ -604,7 +604,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -618,12 +618,10 @@ class TestBootImageUrlList(InstanceTestCase):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/%s/image_001,media=cdrom,if=none,id=cdrom0' % (
-          self.image_directory,),
-        'file=/srv/%s/image_002,media=cdrom,if=none,id=cdrom1' % (
-          self.image_directory,),
+        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
+        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -647,7 +645,7 @@ class TestBootImageUrlList(InstanceTestCase):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -786,7 +784,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # check that the image is NOT YET available in kvm
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 
@@ -800,12 +798,10 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # now the image is available in the kvm, and its above default image
     self.assertEqual(
       [
-        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom0',
-        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom,'
-        'if=none,id=cdrom1',
+        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
+        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
         'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-        'media=cdrom,if=none,id=cdrom2'
+        'media=cdrom'
       ],
       getRunningImageList()
     )
@@ -834,7 +830,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # again only default image is available in the running process
     self.assertEqual(
       ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
-       'media=cdrom,if=none,id=cdrom0'],
+       'media=cdrom'],
       getRunningImageList()
     )
 

software/neoppod/stress-testing/software.cfg

@@ -78,6 +78,8 @@ packages +=
   ca-certificates file g++ libc6-dev make patch python
 # speed up build by using the following components from the OS
   git liblzma-dev libssl-dev pkg-config python-dev
+# for pygolang
+  python-greenlet-dev
 # extra requirements for NEO
   libnetfilter-queue-dev nftables
 # extra requirements for this SR

software/plantuml/software.cfg

@@ -15,6 +15,9 @@ parts =
   slapos-cookbook
   instance
 
+[python]
+part = python3
+
 [instance]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}

software/proftpd/software.cfg

@@ -12,6 +12,8 @@ parts =
   proftpd-config-file
   instance-profile
 
+[python]
+part = python3
 
 [download-file-base]
 recipe = slapos.recipe.build:download

software/proftpd/test/test.py

@@ -27,9 +27,9 @@
 
 import os
 import shutil
-import urlparse
+from urllib.parse import urlparse
 import tempfile
-import StringIO
+import io
 import subprocess
 
 import pysftp
@@ -58,7 +58,7 @@ class ProFTPdTestCase(SlapOSInstanceTestCase):
     cnopts.hostkeys = None
 
     parameter_dict = self.computer_partition.getConnectionParameterDict()
-    sftp_url = urlparse.urlparse(parameter_dict['url'])
+    sftp_url = urlparse(parameter_dict['url'])
 
     return pysftp.Connection(
         hostname or sftp_url.hostname,
@@ -95,7 +95,7 @@ class TestSFTPOperations(ProFTPdTestCase):
   def test_simple_sftp_session(self):
     with self._getConnection() as sftp:
       # put a file
-      with tempfile.NamedTemporaryFile() as f:
+      with tempfile.NamedTemporaryFile(mode='w') as f:
         f.write("Hello FTP !")
         f.flush()
         sftp.put(f.name, remotepath='testfile')
@@ -117,14 +117,14 @@ class TestSFTPOperations(ProFTPdTestCase):
   def test_uploaded_file_not_visible_until_fully_uploaded(self):
     test_self = self
 
-    class PartialFile(StringIO.StringIO):
+    class PartialFile(io.StringIO):
       def read(self, *args):
         # file is not visible yet
         test_self.assertNotIn('destination', os.listdir(test_self.upload_dir))
         # it's just a hidden file
         test_self.assertEqual(
             ['.in.destination.'], os.listdir(test_self.upload_dir))
-        return StringIO.StringIO.read(self, *args)
+        return super().read(*args)
 
     with self._getConnection() as sftp:
       sftp.sftp_client.putfo(PartialFile("content"), "destination")
@@ -136,7 +136,7 @@ class TestSFTPOperations(ProFTPdTestCase):
     test_self = self
     with self._getConnection() as sftp:
 
-      class ErrorFile(StringIO.StringIO):
+      class ErrorFile(io.StringIO):
         def read(self, *args):
           # at this point, file is already created on server
           test_self.assertEqual(
@@ -152,17 +152,17 @@ class TestSFTPOperations(ProFTPdTestCase):
 
   def test_user_cannot_escape_home(self):
     with self._getConnection() as sftp:
-      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+      with self.assertRaises(PermissionError):
         sftp.listdir('..')
-      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+      with self.assertRaises(PermissionError):
         sftp.listdir('/')
-      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+      with self.assertRaises(PermissionError):
         sftp.listdir('/tmp/')
 
 
 class TestUserManagement(ProFTPdTestCase):
   def test_user_can_be_added_from_script(self):
-    with self.assertRaisesRegexp(AuthenticationException,
+    with self.assertRaisesRegex(AuthenticationException,
                                  'Authentication failed'):
       self._getConnection(username='bob', password='secret')
 
@@ -177,12 +177,12 @@ class TestBan(ProFTPdTestCase):
   def test_client_are_banned_after_5_wrong_passwords(self):
     # Simulate failed 5 login attempts
     for i in range(5):
-      with self.assertRaisesRegexp(AuthenticationException,
+      with self.assertRaisesRegex(AuthenticationException,
                                    'Authentication failed'):
         self._getConnection(password='wrong')
 
     # after that, even with a valid password we cannot connect
-    with self.assertRaisesRegexp(SSHException, 'Connection reset by peer'):
+    with self.assertRaisesRegex(SSHException, 'Connection reset by peer'):
       self._getConnection()
 
     # ban event is logged
@@ -190,7 +190,7 @@ class TestBan(ProFTPdTestCase):
                            'var',
                            'log',
                            'proftpd-ban.log')) as ban_log_file:
-      self.assertRegexpMatches(
+      self.assertRegex(
           ban_log_file.readlines()[-1],
           'login from host .* denied due to host ban')
 
@@ -203,7 +203,7 @@ class TestInstanceParameterPort(ProFTPdTestCase):
 
   def test_instance_parameter_port(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
-    sftp_url = urlparse.urlparse(parameter_dict['url'])
+    sftp_url = urlparse(parameter_dict['url'])
     self.assertEqual(self.free_port, sftp_url.port)
     self.assertTrue(self._getConnection())
 

software/repman/buildout.hash.cfg

@@ -14,11 +14,11 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 027cdcfe251b7bba9b779fee890a9162
+md5sum = fd0df2c4a91c1e478b72cbaeb6c021c3
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 657ecdb1dfbbcf53e4e7932b3b5708c4
+md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = 189ccee60d0fb53e29431a45e0816bc1
+md5sum = 21a29a41768b2370d671d3086b3ef2bb
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in

software/repman/instance-mariadb.cfg.jinja2.in

@@ -156,7 +156,7 @@ wrapper = ${directory:controller}/mariadb
 {% do part_list.append("supervisord-mariadb") %}
 
 [odbc-ini-text]
-text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
+text = {{ dumps(base64.b64encode( slapparameter_dict.get('odbc-ini', '').encode() )) }}
 
 [{{ section('odbc-ini') }}]
 < = jinja2-template-base

software/repman/instance-repman.cfg.jinja2.in

@@ -359,7 +359,7 @@ update-command = ${:command}
 [replication-manager-reload]
 recipe = slapos.recipe.template:jinja2
 template = {{ template_repman_manager_sh }}
-cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
+cluster-list = {{ dumps( list(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict)) ) }}
 context =
   section parameter_dict repman
   key username repman-parameter:username

software/repman/instance.cfg.in

@@ -101,3 +101,4 @@ template = {{ template_mariadb }}
 filename = instance-mariadb.cfg
 extra-context =
     section parameter_dict template-mariadb-parameters
+    import base64 base64

software/repman/software.cfg

@@ -29,6 +29,9 @@ parts =
   template-mysqld-wrapper
   gowork
   
+[python]
+part = python3
+
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/instance.cfg
@@ -150,4 +153,3 @@ link-binary =
 # Pin versions of eggs used that are not already pinned by stack/slapos.cfg
 [versions]
 slapos.recipe.template = 4.4
-rubygemsrecipe  = 0.2.2+slapos001

software/seleniumserver/test/test.py

@@ -34,8 +34,10 @@ import unittest
 import urlparse
 import base64
 import hashlib
+import logging
 import contextlib
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from BaseHTTPServer import BaseHTTPRequestHandler
+
 from io import BytesIO
 
 import paramiko
@@ -48,30 +50,22 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase
+from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
 
 setUpModule, SeleniumServerTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
 
-class WebServerMixin(object):
-  """Mixin class which provides a simple web server reachable at self.server_url
-  """
-  def setUp(self):
-    """Start a minimal web server.
-    """
-    class TestHandler(BaseHTTPRequestHandler):
+
+class WebServer(ManagedHTTPServer):
+  class RequestHandler(BaseHTTPRequestHandler):
     """Request handler for our test server.
 
     The implemented server is:
       - submit q and you'll get a page with q as title
       - upload a file and the file content will be displayed in div.uploadedfile
     """
-      def log_message(self, *args, **kw):
-        if SeleniumServerTestCase._debug:
-          BaseHTTPRequestHandler.log_message(self, *args, **kw)
-
     def do_GET(self):
       self.send_response(200)
       self.end_headers()
@@ -111,18 +105,14 @@ class WebServerMixin(object):
         </html>
       ''' % (form['q'].value, file_data))
 
-    super(WebServerMixin, self).setUp()
-    ip = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
-    port = findFreeTCPPort(ip)
-    server = HTTPServer((ip, port), TestHandler)
-    self.server_process = multiprocessing.Process(target=server.serve_forever)
-    self.server_process.start()
-    self.server_url = 'http://%s:%s/' % (ip, port)
+    log_message = logging.getLogger(__name__ + '.WebServer').info
 
-  def tearDown(self):
-    self.server_process.terminate()
-    self.server_process.join()
-    super(WebServerMixin, self).tearDown()
+
+class WebServerMixin(object):
+  """Mixin class which provides a simple web server reachable at self.server_url
+  """
+  def setUp(self):
+    self.server_url = self.getManagedResource('web_server', WebServer).url
 
 
 class BrowserCompatibilityMixin(WebServerMixin):

software/slapos-master/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = 2ef0ddc206c6b0982a37cfc21f23e423
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = ef86e09e44ac67a9b15939df0ab4a466
+md5sum = 4998e62351f54700ee23a2ca8cd89329
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -7,9 +7,6 @@
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
 per partition. No more (undefined result), no less (IndexError).
 -#}
-# TODO: insert varnish between apache & haproxy.
-# And think of a way to specify which urls goe through varnish, which go
-# directly to haproxy. (maybe just passing literal configuration file chunk)
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set apache_ip_list = [ipv4] -%}
 {% if ipv6_set -%}

software/slapos-sr-testing/software-py3.cfg

@@ -6,11 +6,15 @@ extends =
 part = python3
 
 [eggs]
-eggs -=
-# plantuml is not Py3-compatible
+eggs +=
+# plantuml 0.3.0 is only available for Python 3
   ${slapos.test.plantuml-setup:egg}
 
 [template]
 extra =
+  ${slapos.test.helloworld-setup:setup}
   ${slapos.test.monitor-setup:setup}
+  ${slapos.test.plantuml-setup:setup}
   ${slapos.test.powerdns-setup:setup}
+  ${slapos.test.proftpd-setup:setup}
+  ${slapos.test.repman-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -189,7 +189,6 @@ eggs =
   ${slapos.test.jstestnode-setup:egg}
   ${slapos.test.kvm-setup:egg}
   ${slapos.test.monitor-setup:egg}
-  ${slapos.test.plantuml-setup:egg}
   ${slapos.test.powerdns-setup:egg}
   ${slapos.test.proftpd-setup:egg}
   ${slapos.test.re6stnet-setup:egg}
@@ -256,11 +255,8 @@ extra =
   ${slapos.test.erp5-setup:setup}
   ${slapos.test.htmlvalidatorserver-setup:setup}
   ${slapos.test.slapos-master-setup:setup}
-  ${slapos.test.plantuml-setup:setup}
-  ${slapos.test.proftpd-setup:setup}
   ${slapos.test.re6stnet-setup:setup}
   ${slapos.test.seleniumserver-setup:setup}
-  ${slapos.test.helloworld-setup:setup}
   ${slapos.test.jstestnode-setup:setup}
   ${slapos.test.jupyter-setup:setup}
   ${slapos.test.nextcloud-setup:setup}
@@ -270,7 +266,6 @@ extra =
   ${slapos.test.gitlab-setup:setup}
   ${slapos.test.cloudooo-setup:setup}
   ${slapos.test.dream-setup:setup}
-  ${slapos.test.repman-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always
@@ -283,7 +278,7 @@ forcediphttpsadapter = 1.0.1
 httplib2 = 0.11.3
 image = 1.5.25
 paramiko = 2.4.2
-plantuml = 0.1.1
+plantuml = 0.3.0
 pysftp = 0.2.9
 requests-toolbelt = 0.8.0
 selenium = 3.141.0
@@ -319,3 +314,4 @@ funcsigs = 1.0.2
 mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
+typing = 3.7.4.3

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = a04806bdebd611c7f6698b4bef7af70d
+md5sum = 2df601dd3ccb3ba38b3aee7243b7f8e5

software/slapos-testing/instance.cfg

@@ -70,6 +70,10 @@ repository = ${slapos.toolbox-repository:location}
 <= download-source
 repository = ${slapos.rebootstrap-repository:location}
 
+[rubygemsrecipe]
+<= download-source
+repository = ${rubygemsrecipe-repository:location}
+
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper
@@ -77,7 +81,7 @@ wrapper-path = $${create-directory:bin}/runTestSuite
 command-line =
   ${buildout:bin-directory}/runTestSuite
   --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
-  --source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location}
+  --source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location},$${slapos.rebootstrap:location},$${rubygemsrecipe:location}
 
 # Notes about environment:
 # * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests

software/slapos-testing/software.cfg

@@ -97,6 +97,11 @@ depends =
 egg = slapos.rebootstrap[test]
 setup = ${slapos.rebootstrap-repository:location}
 
+[rubygemsrecipe-setup]
+<= setup-develop-egg
+egg = rubygemsrecipe
+setup = ${rubygemsrecipe-repository:location}
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -118,6 +123,7 @@ eggs =
   ${slapos.toolbox-setup:egg}
   ${slapos.libnetworkcache-setup:egg}
   ${slapos.rebootstrap-setup:egg}
+  ${rubygemsrecipe-setup:egg}
   zope.testing
   supervisor
 entry-points =
@@ -186,6 +192,10 @@ repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.rebootstrap.git
 
+[rubygemsrecipe-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/rubygemsrecipe.git
+
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}

software/theia/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 397fcb3279029af3055b23525d147445
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = d058e73c3d90ac3da44734c2d47eac95
+md5sum = c7aa84922a1b80fd8a4c3d96f6ac7e25
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt

stack/erp5/buildout.hash.cfg

@@ -78,7 +78,7 @@ md5sum = 68b329da9893e34099c7d8ad5cb9c940
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 82dc695e212be124d60ceb1143e56b0d
+md5sum = 0920a53b10d3811a5f49930adffb62d8
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -90,7 +90,7 @@ md5sum = 2f3ddd328ac1c375e483ecb2ef5ffb57
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 0097e49b5bd7ad4978c722c1cdd27d6c
+md5sum = ecf119142e6b5cd85a2ba397552d2142
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -7,9 +7,6 @@
 XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
 per partition. No more (undefined result), no less (IndexError).
 -#}
-# TODO: insert varnish between apache & haproxy.
-# And think of a way to specify which urls goe through varnish, which go
-# directly to haproxy. (maybe just passing literal configuration file chunk)
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set apache_ip_list = [ipv4] -%}
 {% if ipv6_set -%}
@@ -21,19 +18,52 @@ per partition. No more (undefined result), no less (IndexError).
 recipe = slapos.recipe.template:jinja2
 mode = 644
 
+[balancer-csr-request-config]
+< = jinja2-template-base
+template = inline:
+    [req]
+    prompt = no
+    req_extensions = req_ext
+    distinguished_name = dn
+    [ dn ]
+    CN = example.com
+    [ req_ext ]
+    subjectAltName = @alt_names
+    [ alt_names ]
+    IP.1 =  {{ ipv4 }}
+    {% if ipv6_set -%}
+    IP.2 =  {{ ipv6 }}
+    {% endif %}
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
+
+[balancer-csr-request]
+recipe = plone.recipe.command
+command = {{ parameter_dict["openssl"] }}/bin/openssl req \
+  -newkey rsa:2048 \
+  -batch \
+  -new \
+  -nodes \
+  -keyout '${apache-conf-ssl:key}' \
+  -config '${balancer-csr-request-config:rendered}' \
+  -out '${:csr}'
+stop-on-error = true
+csr = ${directory:etc}/${:_buildout_section_name_}.csr.pem
+
+
 {{ caucase.updater(
      prefix='caucase-updater',
      buildout_bin_directory=parameter_dict['bin-directory'],
      updater_path='${directory:services-on-watch}/caucase-updater',
      url=ssl_parameter_dict['caucase-url'],
      data_dir='${directory:srv}/caucase-updater',
-     crt_path='${apache-conf-ssl:caucase-cert}',
+     crt_path='${apache-conf-ssl:cert}',
      ca_path='${directory:srv}/caucase-updater/ca.crt',
      crl_path='${directory:srv}/caucase-updater/crl.pem',
-     key_path='${apache-conf-ssl:caucase-key}',
+     key_path='${apache-conf-ssl:key}',
      on_renew='${apache-graceful:output}',
      max_sleep=ssl_parameter_dict.get('max-crl-update-delay', 1.0),
      template_csr_pem=ssl_parameter_dict.get('csr'),
+     template_csr=None if ssl_parameter_dict.get('csr') else '${balancer-csr-request:csr}',
      openssl=parameter_dict['openssl'] ~ '/bin/openssl',
 )}}
 {% do section('caucase-updater') -%}
@@ -179,9 +209,6 @@ hash-files = ${haproxy-cfg:rendered}
 [apache-conf-ssl]
 cert = ${directory:apache-conf}/apache.crt
 key = ${directory:apache-conf}/apache.pem
-# XXX caucase certificate is not supported by caddy for now
-caucase-cert = ${directory:apache-conf}/apache-caucase.crt
-caucase-key = ${directory:apache-conf}/apache-caucase.pem
 {% if frontend_caucase_url_list -%}
 depends = ${caucase-updater-housekeeper-run:recipe}
 ca-cert-dir = ${directory:apache-ca-cert-dir}
@@ -204,19 +231,6 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
-[apache-ssl]
-{% if ssl_parameter_dict.get('key') -%}
-key = ${apache-ssl-key:rendered}
-cert = ${apache-ssl-cert:rendered}
-{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
-{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
-{% else %}
-recipe = plone.recipe.command
-command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
-key = ${apache-conf-ssl:key}
-cert = ${apache-conf-ssl:cert}
-{%- endif %}
-
 [apache-conf-parameter-dict]
 backend-list = {{ dumps(apache_dict.values()) }}
 zope-virtualhost-monster-backend-dict = {{ dumps(zope_virtualhost_monster_backend_dict) }}
@@ -228,8 +242,8 @@ access-log = ${directory:log}/apache-access.log
 # Apache 2.4's default value (60 seconds) can be a bit too short
 timeout = 300
 # Basic SSL server configuration
-cert = ${apache-ssl:cert}
-key = ${apache-ssl:key}
+cert = ${apache-conf-ssl:cert}
+key = ${apache-conf-ssl:key}
 cipher =
 ssl-session-cache = ${directory:log}/apache-ssl-session-cache
 {% if frontend_caucase_url_list -%}

stack/erp5/instance-erp5.cfg.in

@@ -176,11 +176,16 @@ return =
 {%- if test_runner_enabled %}
   test-runner-address-list
 {% endif %}
-{% set bt5_default_list = 'erp5_full_text_mroonga_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_run_my_doc' -%}
+{% set bt5_default_list = [
+  'erp5_full_text_mroonga_catalog',
+  'erp5_configurator_standard',
+  'erp5_configurator_maxma_demo',
+  'erp5_configurator_run_my_doc',
+] -%}
 {% if has_jupyter -%}
-{%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
+{%   do bt5_default_list.append('erp5_data_notebook') -%}
 {% endif -%}
-config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
+config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
 config-caucase-url = {{ dumps(caucase_url) }}

stack/lamp/buildout.cfg

@@ -4,20 +4,13 @@ ignore-existing = true
 
 parts =
   slapos-cookbook
-  apache-php
   php-redis
   php-imagick
   php-apcu
-  mariadb
-  mroonga-mariadb
   dropbear
   eggs
   instance
 
-  instance-apache-php
-  template-mariadb
-  instance-lamp
-
 extends =
   buildout.hash.cfg
   ../../component/curl/buildout.cfg

stack/slapos.cfg

@@ -188,8 +188,9 @@ regex = 2020.9.27
 requests = 2.24.0
 scandir = 1.10.0
 setuptools-dso = 1.7
+rubygemsrecipe  = 0.3.0
 six = 1.12.0
-slapos.cookbook = 1.0.152
+slapos.cookbook = 1.0.167
 slapos.core = 1.6.2
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
@@ -197,7 +198,7 @@ slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
 slapos.recipe.build = 0.46
 slapos.recipe.cmmi = 0.16
-slapos.toolbox = 0.111
+slapos.toolbox = 0.112
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
@@ -254,7 +255,7 @@ enum34 = 1.1.10
 
 # Required by:
 # slapos.toolbox==0.94
-erp5.util = 0.4.68
+erp5.util = 0.4.69
 
 # Required by:
 # slapos.toolbox==0.94