Update Release Candidate

component/ZEO/buildout.cfg

@@ -65,9 +65,9 @@ egg     = ZEO
 eggs    = ${:egg}
 
 egg-versions =
-  ZEO = 5.2.2
-  trollius = 2.2.post1
-  futures = 3.2.0
+  ZEO = 5.2.3
+  trollius = 2.2.1
+  futures = 3.3.0
 
 
 # ZEO4-wc2 is ZEO4 version with patches for wendelin.core 2 to work correctly.
@@ -76,5 +76,9 @@ egg-versions =
 # See https://github.com/zopefoundation/ZEO/pull/161 for the reference.
 [ZEO4-wc2]
 <= ZEO4
-revisions = bf80d23d3506
+setup   = ${ZEO4-wc2-repository:location}
+
+[ZEO4-wc2-repository]
+<= ZEO4-repository
+revision = bf80d23d3506
 location = ${buildout:parts-directory}/ZEO4-wc2

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.22.tar.gz
-md5sum = 4d6d5debca0d1bcf51293fb58914f1a3
+url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.24.tar.gz
+md5sum = 493011d81eea0f3700543fc6a80acc03
 configure-command = true
 # for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
 # otherwise use "generic".

component/headless-chromium/buildout.cfg

 [buildout]
 extends =
-  ../binutils/buildout.cfg
-  ../bison/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../gperf/buildout.cfg
-  ../ninja/buildout.cfg
-  ../freetype/buildout.cfg
-  ../cmake/buildout.cfg
-  ../git/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../cups/buildout.cfg
+# Build dependencies:
   ../coreutils/buildout.cfg
+  ../curl/buildout.cfg
   ../depot_tools/buildout.cfg
-  ../findutils/buildout.cfg
-  ../fontconfig/buildout.cfg
-  ../gettext/buildout.cfg
-  ../glib/buildout.cfg
-  ../gtk-2/buildout.cfg
-  ../libexpat/buildout.cfg
-  ../libffi/buildout.cfg
-  ../libpng/buildout.cfg
-  ../libxml2/buildout.cfg
-  ../mesa/buildout.cfg
+  ../git/buildout.cfg
+  ../gperf/buildout.cfg
+  ../pkgconfig/buildout.cfg
+# Runtime dependencies:
   ../nspr/buildout.cfg
   ../nss/buildout.cfg
-  ../pcre/buildout.cfg
-  ../sqlite3/buildout.cfg
-  ../xorg/buildout.cfg
-  ../zlib/buildout.cfg
 
 parts =
-  chromium
+  headless-chromium-wrapper
 
-[gconf]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
-md5sum = 2b16996d0e4b112856ee5c59130e822c
-configure-options = --disable-orbit --disable-static
-environment =
-  PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-
-[chromedriver]
-recipe = hexagonit.recipe.download
-url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
-md5sum = a72088c0a6b018ded2c0fff616da8f65
-
-[chromium-download]
+# The Chromium project recommends using their own `fetch' tool rather
+# than doing a `git clone', but this is a little more flexible and works
+# fine.
+#
+# Setting depth=1000 is a middle ground between cloning no history
+# (causes some scripts to break) and cloning the full history (takes a
+# really long time).
+#
+# Note: we should add a `depth' option to slapos.recipe.build:gitclone
+# and then migrate this part to use that recipe at some point.
+[chromium-source]
 recipe = plone.recipe.command
-# This revision is 56.0.2924.122. Because the chromedriver only support some certain
-# version. Which version 56 is in the middle of 55-57.
-revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
-path = ${buildout:parts-directory}/${:_buildout_section_name_}
 command =
-  set -e
-  PATH=${depot_tools:location}:${git:location}/bin:$PATH
-  [ -d ${:path} ] && rm -r ${:path}
-  mkdir -p ${:path}
-  cd ${:path}
-  # Do never use `fetch` with the `--no-history` option unless you find an
-  # option to fetch directly at the wanted revision. `--no-history` could
-  # reduce the download size significantly but even if it retrieves enough
-  # commits at the time you test this section, development continues upstream
-  # and at some point the next command (gclient) would break.
-  # ...
-  # This command only could work in an empty dir.
-  fetch --nohooks chromium
-  gclient sync --revision ${:revision} --with_branch_heads
+  export PATH=$PATH:${git:location}/bin
+  git clone ${:repository} ${:location} \
+    --branch ${:version} \
+    --depth 1000
+repository = https://chromium.googlesource.com/chromium/src.git
 stop-on-error = true
 
-[chromium]
+# We should place the .gclient file in the parent directory of the
+# checkout/repository itself.
+location = ${:gclient-location}/${:name}
+gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+# Theoretically the checkout can be put anywhere as long as you specify
+# "name" appropriately in the .gclient file, but in practice it seems
+# that some automated tools break. It's safest to put it in a directory
+# called "src".
+name = src
+
+# There's nothing special about version 92.0.4515.107. It just happened
+# to be the current Chromium stable version at the time of writing.
+version = 92.0.4515.107
+
+
+[headless-chromium]
 recipe = slapos.recipe.cmmi
-path = ${chromium-download:path}/src
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
+path = ${chromium-source:location}
+location = ${:path}/out/headless
+
+# Configuration file for GN, the tool to build the actual compilation
+# configuration file.
+default-gclient-config =
+  solutions = [
+    {
+      "name": "${chromium-source:name}",
+      "url": "${chromium-source:repository}",
+      "managed": False,
+      "custom_deps": {},
+      "custom_vars": {},
+    },
+  ]
+
+# Configuration for a headless build.
+build-config-options =
+  import("//build/args/headless.gn")
+  is_debug = false
+  symbol_level = 0
+  blink_symbol_level = 0
+
+  # We need to unbundle the build toolchain in order to set our own
+  # LDFLAGS.
+  custom_toolchain = "//build/toolchain/linux/unbundle:default"
+  host_toolchain = "//build/toolchain/linux/unbundle:default"
+  current_os = "linux"
+  current_cpu = "x64"
+
+# Chromium bundles its own LLVM toolchain, so we might as well use it.
+llvm-toolchain = ${:path}/third_party/llvm-build/Release+Asserts/bin
+
 configure-command =
-  gclient runhooks
+# Sync build dependencies---this is a little finnicky.
+  echo '${:default-gclient-config}' \
+    > ${chromium-source:gclient-location}/.gclient
+  gclient sync --no-history
+# Generate build configuration files.
   mkdir -p ${:location}
-  echo 'use_udev = true
-  is_debug = false
-  enable_nacl = false' > ${:location}/args.gn
+  echo '${:build-config-options}' > ${:location}/args.gn
   gn gen ${:location}
 
-# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
-make-binary = ninja -C ${:location} chrome
+# You can run the headless Chromium shell using
+# ${:binary} --remote-debugging-port=1234
+make-binary =
+  autoninja -C ${:location} headless_shell
+# By building our own version of Chromedriver, we can ensure version
+# compatibility. The build is quite cheap compared to Chromium, anyway.
+  autoninja -C ${:location} chromedriver
 environment =
-  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
-  PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
-  CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
-  LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
+  PATH=${depot_tools:location}:${gperf:location}/bin:${pkgconfig:location}/bin:${coreutils:location}/bin:${git:location}/bin:${curl:location}/bin:%(PATH)s
+  LDFLAGS="-Wl,-rpath=${nss:location}/lib,-rpath=${nspr:location}/lib"
+  CC="${:llvm-toolchain}/clang"
+  CXX="${:llvm-toolchain}/clang++"
+  AR="${:llvm-toolchain}/llvm-ar"
+  NM="${:llvm-toolchain}/llvm-nm"
+
+# Expose devtools frontend location.
+devtools-frontend = ${:location}/gen/third_party/devtools-frontend/src/front_end
+
+binary = ${:location}/headless_shell
+chromedriver = ${:location}/chromedriver
+promises =
+  ${:binary}
+  ${:chromedriver}
+
+
+# At runtime, Chromium tries to dynamically load the NSS certificate
+# database from "libnssckbi.so". But Chromium does this through NSPR,
+# which doesn't know where SlapOS installed NSS. Since we don't want to
+# modify the NSPR component from the Chromium component, we just set
+# LD_LIBRARY_PATH in a wrapper script so that NSPR knows where NSS is.
+#
+# Alternatively, we could patch crypto/nss_util.cc in the Chromium
+# source code to use an absolute path for libnssckbi.so, but this is not
+# as future-proof against new versions of Chromium.
+[headless-chromium-wrapper]
+recipe = slapos.recipe.template:jinja2
+template =
+  inline:#!/bin/sh
+  export LD_LIBRARY_PATH="{{ nss_location }}/lib:$LD_LIBRARY_PATH"
+  exec {{ chromium_binary }} "$@"
+rendered = ${buildout:bin-directory}/headless-chromium
+context =
+  key nss_location nss:location
+  key chromium_binary headless-chromium:binary

component/jupyter-py2/buildout.cfg

@@ -95,63 +95,25 @@ tornado = 4.4.2
 widgetsnbextension = 2.0.0
 # numpy >= 1.13.1 is required for numpy.core.multiarray
 numpy = 1.13.1
-
-
-# Required by:
-# tornado==4.4.2
 certifi = 2020.6.20
 
-# Required by:
-# notebook==4.3.2
 # nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
 nbconvert = 4.1.0
-
-# Required by:
-# ipython==5.3.0
 pathlib2 = 2.2.1
-
-# Required by:
-# statsmodels==0.8.0
 patsy = 0.4.1
-
-# Required by:
-# ipython==5.3.0
 pexpect = 4.2.1
-
-# Required by:
-# ipython==5.3.0
 pickleshare = 0.7.4
-
-# Required by:
-# pathlib2==2.2.1
 scandir = 1.5
-
-# Required by:
-# statsmodels==0.8.0
 scipy = 0.19.0
-
-# Required by:
-# tornado==4.4.2
 singledispatch = 3.4.0.3
-
-# Required by:
-# prompt-toolkit==1.0.13
 wcwidth = 0.1.7
-
 jupyter = 1.0.0
 jupyter-console = 5.1.0
-
-# Required by:
-# jupyter==1.0.0
 qtconsole = 4.3.0
-
 et-xmlfile = 1.0.1
 h5py = 2.7.1
 mpmath = 1.0.0
 openpyxl = 2.5.2
 sympy = 1.1.1
 xlrd = 1.1.0
-
-# Required by:
-# openpyxl==2.5.2
 jdcal = 1.4

component/libsecret/buildout.cfg

+[buildout]
+parts = libsecret
+extends =
+  ../gnutls/buildout.cfg
+  ../libxslt/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../glib/buildout.cfg
+  ../gettext/buildout.cfg
+  ../libuuid/buildout.cfg
+  ../xz-utils/buildout.cfg
+
+
+[libsecret]
+recipe = slapos.recipe.cmmi
+url = https://download.gnome.org/sources/libsecret/0.20/libsecret-0.20.4.tar.xz
+md5sum = bf92f48afab2891f644f311e0f37683f
+shared = true
+configure-options = 
+  --with-libgcrypt-prefix=${libgcrypt:location}
+  --with-libintl-prefix=${gettext:location}
+  --disable-gtk-doc-html
+  --disable-manpages
+  --disable-vala
+
+environment = 
+  PATH=${xz-utils:location}/bin/:${pkgconfig:location}/bin:${libxslt:location}/bin:${glib:location}/bin:${libgcrypt:location}/bin/:%(PATH)s
+  PKG_CONFIG_PATH=${:pkg-config-path-depends}
+  LDFLAGS=-Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libuuid:location}/lib
+pkg-config-path-depends = ${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${libuuid:location}/lib/pkgconfig
+pkg-config-path = @@LOCATION@@/lib/pkgconfig:${:pkg-config-path-depends}

component/nginx/buildout.cfg

@@ -24,6 +24,7 @@ configure-options=
   --with-http_v2_module
   --with-http_gzip_static_module
   --with-http_realip_module
+  --with-http_sub_module
   --with-mail
   --with-mail_ssl_module
   --with-ld-opt="-L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
@@ -85,7 +86,7 @@ revision = 3d3a204177d3a7ab8a2858e04e792a6d11bf133f
 git-executable = ${git:location}/bin/git
 
 [nginx-push-stream]
-<= nginx
+<= nginx-common
 configure-options=
   --with-ipv6
   --with-http_ssl_module

component/nxdtest/buildout.cfg

@@ -20,6 +20,6 @@ egg     = nxdtest
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = b5a74214
+revision = 0ad45a9c
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git

component/pytest/buildout.cfg

@@ -15,3 +15,4 @@ eggs    = ${pytest:eggs}
 
 [versions]
 pytest  = 4.6.11:whl
+pytest-timeout = 1.4.2

component/qemu-kvm/buildout.cfg

@@ -91,11 +91,11 @@ md5sum = 6097fdb9cbab47c96471274b9044e983
 # XXX: This is not the latest version because
 #      Debian does not provide a stable URL for it.
 <= debian-amd64-netinst-base
-version = 10.8.0
-md5sum = e221f43f4fdd409250908fc4305727d4
+version = 10.10.0
+md5sum = c7d0e562e589e853b5d00563b4311720
 
 [debian-amd64-testing-netinst.iso]
 <= debian-amd64-netinst-base
-release = bullseye_di_alpha3
-version = bullseye-DI-alpha3
-md5sum = bff147077791586fa7c102267da9f2d2
+release = bullseye_di_rc3
+version = bullseye-DI-rc3
+md5sum = 405917de7062c58357a3673c9901f0c4

component/tesseract/buildout.cfg

@@ -26,6 +26,9 @@ md5sum = 51fe2bcbff1bbce77a25d180fd247f7d
 pkg_config_depends = ${leptonica:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${lcms2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
 pre-configure =
   autoreconf -ivf -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -Wno-portability
+# XXX workaround path on slaprunner with a double slash
+# https://github.com/tesseract-ocr/tesseract/issues/3527
+configure-options = --prefix=$(python -c 'print("""@@LOCATION@@""".replace("//", "/"))')
 
 environment =
   PATH=${pkgconfig:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${patch:location}/bin:%(PATH)s

component/trafficserver/buildout.cfg

@@ -23,12 +23,10 @@ min_version = 8
 
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-9.0.1.tar.bz2
-md5sum = 98bb2de25f332715339ade87530a9f5a
+url = http://apache.claz.org/trafficserver/trafficserver-9.0.2.tar.bz2
+md5sum = 4df67ada24665116bafedd71503215cb
 shared = true
 patch-options = -p1
-patches =
-  https://patch-diff.githubusercontent.com/raw/apache/trafficserver/pull/7577.patch
 configure-options =
   --with-openssl=${openssl:location}
   --with-pcre=${pcre:location}

component/wendelin.core/test-common.cfg

@@ -7,6 +7,7 @@ extends =
 # appropriate ZODB and versions of other components.
 
     ../pytest/buildout.cfg
+    ../scipy/buildout.cfg
     ../gdb/buildout.cfg
     ../../stack/nxdtest.cfg
     buildout.cfg
@@ -15,22 +16,25 @@ parts =
 # keep neoppod first and in parts so that ZODB is built correctly
     neoppod
 
-    wendelin.core
-
 #   for instance
     wendelin.core-python
     slapos-cookbook
     instance.cfg
 
 
-# bin/python s python interpreter with wendelin.core and all other eggs.
+# test-dependent eggs that must come through in-tree recipes.
+[wendelin.core]
+depends +=  ${scipy:egg}
+
+
+# bin/python is python interpreter with wendelin.core and all other eggs.
 [wendelin.core-python]
 <= python-interpreter
 eggs =
-    wendelin.core[test]
-    pygolang[pyx.build]
+    ${wendelin.core:egg}[test]
+    ${pygolang:egg}[pyx.build]
     neoppod[tests]
-    ZEO[test]
+    ${ZEO:egg}[test]
 
 # env.sh for that python + go to be on $PATH
 [wendelin.core-env.sh]

component/zodbtools/test-common.cfg

@@ -10,8 +10,6 @@ extends =
     buildout-dev.cfg
 
 parts =
-    zodbtools
-
 #   for instance
     zodbtools-python
     slapos-cookbook
@@ -21,7 +19,7 @@ parts =
 # bin/python is preinstalled with sys.path to zodbtools & friends.
 [zodbtools-python]
 <= python-interpreter
-eggs = zodbtools[test]
+eggs = ${zodbtools:egg}[test]
 
 # env.sh for zodbtools's python to be on $PATH.
 [zodbtools-env.sh]

software/caddy-frontend/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg.in
-md5sum = 04015a7a552285984d091293ef573fb9
+md5sum = 1dfbd20c77fb3c1f01005a8a920d2ed9
 
 [profile-common]
 filename = instance-common.cfg.in
@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-caddy-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 8507a2ace2f789b92c522cc62ca5aace
+md5sum = 51087ac7615bd7cc01e60eb23701f625
 
 [profile-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 8beb438d06bbb0f917d13e182fb12d17
+md5sum = b6fc5a004a1235ffad3af0b4cb0e661f
 
 [profile-slave-list]
 _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
-md5sum = 613f777a08373088cbaf7f51fd18ea70
+md5sum = 9bb51f663f69d66b5b3708bf892dd3e6
 
 [profile-replicate-publish-slave-information]
 _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
@@ -72,10 +72,6 @@ md5sum = d022455a8610bac2dd51101edb035987
 _update_hash_filename_ = templates/trafficserver/logging.yaml.jinja2
 md5sum = 368b271215a92594ca9e2fa3102d484f
 
-[template-nginx-eventsource-slave-virtualhost]
-_update_hash_filename_ = templates/nginx-eventsource-slave.conf.in
-md5sum = 217a6c801b8330b0b825f7b8b4c77184
-
 [template-caddy-lazy-script-call]
 _update_hash_filename_ = templates/apache-lazy-script-call.sh.in
 md5sum = 77d60840591de67b64ab3572e46273a0

software/caddy-frontend/instance-apache-frontend.cfg.in

-{%- if instance_parameter_dict['slap-software-type'] == software_type -%}
 {% import "caucase" as caucase with context %}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 [buildout]
@@ -11,6 +10,7 @@ parts =
   directory
   logrotate-entry-caddy
   caddy-frontend
+  caddyprofiledeps
   switch-caddy-softwaretype
   caucase-updater
   caucase-updater-promise
@@ -97,9 +97,11 @@ expose-csr_id-var = ${:var}/expose-csr_id
 slave-introspection-var = ${:var}/slave-introspection
 
 [switch-caddy-softwaretype]
-recipe = slapos.cookbook:softwaretype
-single-default = ${dynamic-custom-personal-profile-slave-list:rendered}
-single-custom-personal = ${dynamic-custom-personal-profile-slave-list:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+default = dynamic-custom-personal-profile-slave-list:rendered
+RootSoftwareInstance = ${:default}
+single-default = dynamic-custom-personal-profile-slave-list:rendered
+single-custom-personal = dynamic-custom-personal-profile-slave-list:rendered
 
 [frontend-configuration]
 ip-access-certificate = ${self-signed-ip-access:certificate}
@@ -114,7 +116,7 @@ slave-introspection-domain = ${slave-introspection-frontend:connection-domain}
 # Self Signed certificate for HTTPS IP accesses to the frontend
 recipe = plone.recipe.command
 update-command = ${:command}
-ipv6 = ${slap-network-information:global-ipv6}
+ipv6 = ${slap-configuration:ipv6-random}
 ipv4 = {{instance_parameter_dict['ipv4-random']}}
 certificate = ${caddy-directory:master-autocert-dir}/ip-access-${:ipv6}-${:ipv4}.crt
 {#- Can be stopped on error, as does not rely on self provided service #}
@@ -138,7 +140,7 @@ command =
 # Self Signed certificate for HTTPS access to the frontend with fallback certificate
 recipe = plone.recipe.command
 update-command = ${:command}
-ipv6 = ${slap-network-information:global-ipv6}
+ipv6 = ${slap-configuration:ipv6-random}
 ipv4 = {{instance_parameter_dict['ipv4-random']}}
 certificate = ${caddy-directory:master-autocert-dir}/fallback-access.crt
 {#- Can be stopped on error, as does not rely on self provided service #}
@@ -282,6 +284,7 @@ software_type = single-custom-personal
 organization = {{ slapparameter_dict['cluster-identification'] }}
 organizational-unit = {{ instance_parameter_dict['configuration.frontend-name'] }}
 backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
+partition_ipv6 =  ${slap-configuration:ipv6-random}
 extra-context =
     key caddy_configuration_directory caddy-directory:slave-configuration
     key backend_client_caucase_url :backend-client-caucase-url
@@ -293,7 +296,7 @@ extra-context =
     key caddy_log_directory caddy-directory:slave-log
     key expose_csr_id_organization :organization
     key expose_csr_id_organizational_unit :organizational-unit
-    key global_ipv6 slap-network-information:global-ipv6
+    key global_ipv6 slap-configuration:ipv6-random
     key empty_template software-release-path:template-empty
     key template_default_slave_configuration software-release-path:template-default-slave-virtualhost
     key software_type :software_type
@@ -328,8 +331,8 @@ extra-context =
     section frontend_configuration frontend-configuration
     key http_port configuration:plain_http_port
     key https_port configuration:port
+    key global_ipv6 slap-configuration:ipv6-random
     key local_ipv4 :local_ipv4
-    key global_ipv6 slap-network-information:global-ipv6
     key error_log caddy-configuration:error-log
     key not_found_file caddy-configuration:not-found-file
     key username monitor-instance-parameter:username
@@ -904,7 +907,7 @@ recipe = slapos.cookbook:requestoptional
 name = Slave Introspection Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
+config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.slave-introspection-https-port'] }}/
 config-https-only = true
 return = domain secure_access
 
@@ -914,7 +917,7 @@ recipe = slapos.cookbook:requestoptional
 name = Backend Haproxy Statistic Frontend {{ instance_parameter_dict['configuration.frontend-name'] }}
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config-url = https://[${slap-network-information:global-ipv6}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
+config-url = https://[${slap-configuration:ipv6-random}]:{{ instance_parameter_dict['configuration.backend-haproxy-statistic-port'] }}/
 config-https-only = true
 return = domain secure_access
 
@@ -1022,5 +1025,3 @@ config-command =
 {%- for key, value in software_parameter_dict.iteritems() %}
 {{ key }} = {{ dumps(value) }}
 {%- endfor %}
-
-{%- endif -%} {# if instance_parameter_dict['slap-software-type'] == software_type #}

software/caddy-frontend/instance-apache-replicate.cfg.in

-{% if instance_parameter_dict['slap-software-type'] in software_type %}
 {% set aibcc_enabled = True %}
 {% import "caucase" as caucase with context %}
 {#- SERVER_POLLUTED_KEY_LIST is a list of keys which comes from various SlapOS Master implementations, which mix request and publish keys on each slave information -#}
@@ -44,13 +43,7 @@ context =
 {% set popen = functools_module.partial(subprocess_module.Popen, stdout=subprocess_module.PIPE, stderr=subprocess_module.STDOUT, stdin=subprocess_module.PIPE) %}
 {% set part_list = [] %}
 {% set single_type_key = 'single-' %}
-{% if instance_parameter_dict['slap-software-type'] == "replicate" %}
-{%   set frontend_type = slapparameter_dict.pop('-frontend-type', 'single-default') %}
-{% elif instance_parameter_dict['slap-software-type'] in ['default', 'RootSoftwareInstance'] %}
-{%   set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
-{% else %}
-{%   set frontend_type = "%s%s" % (single_type_key, instance_parameter_dict['slap-software-type']) %}
-{% endif %}
+{% set frontend_type = "%s%s" % (single_type_key, 'custom-personal') %}
 {% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '1') | int %}
 {% set slave_list_name = 'extra_slave_instance_list' %}
 {% set frontend_list = [] %}
@@ -117,9 +110,7 @@ context =
 {%   set slave_warning_list = [] %}
 {%   set slave_server_alias_unclashed = [] %}
 {%   set slave_type = slave.get('type') %}
-{%   if slave_type == 'eventsource' %}
-{%     do slave_error_list.append('type:eventsource is not implemented') %}
-{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
+{%   if slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
 {#   Check health-check-* #}
@@ -392,12 +383,12 @@ config-url =
 #--
 #-- Publish slave information
 [publish-slave-information]
-recipe = slapos.cookbook:softwaretype
-default = ${dynamic-publish-slave-information:rendered}
-RootSoftwareInstance = ${dynamic-publish-slave-information:rendered}
-replicate = ${dynamic-publish-slave-information:rendered}
-custom-personal = ${dynamic-publish-slave-information:rendered}
-custom-group = ${dynamic-publish-slave-information:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+default = dynamic-publish-slave-information:rendered
+RootSoftwareInstance = ${:default}
+replicate = dynamic-publish-slave-information:rendered
+custom-personal = dynamic-publish-slave-information:rendered
+custom-group = dynamic-publish-slave-information:rendered
 
 [request-kedifa]
 <= slap-connection
@@ -909,6 +900,4 @@ parts =
   caucased-backend-client-promise
 {% for part in part_list %}
 {{ '  %s' % part }}
-{% endfor %}
-#      publish-information
+{% endfor %}
\ No newline at end of file
-{% endif %}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-04/schema",
   "properties": {
     "custom_domain": {
-      "description": "Custom Domain to use for the website",
+      "description": "Custom Domain to use for the website. Shall contain only letters, numbers and -, and can look like example.com, first2.example.com special-site.example.com.",
       "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$",
       "title": "Custom Domain",
       "type": "string"
@@ -15,14 +15,13 @@
     },
     "type": {
       "default": "",
-      "description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
+      "description": "Type of slave. If redirect, the slave will redirect to the given URL. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
       "enum": [
         "",
         "zope",
         "redirect",
         "notebook",
-        "websocket",
-        "eventsource"
+        "websocket"
       ],
       "title": "Backend Type",
       "type": "string"

software/caddy-frontend/instance-slave-caddy-simplified-input-schema.json

@@ -9,7 +9,7 @@
     },
     "type": {
       "default": "",
-      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
+      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster.",
       "enum": [
         "",
         "zope"

software/caddy-frontend/instance.cfg.in

@@ -2,6 +2,7 @@
 extends = {{ software_parameter_dict['profile_common'] }}
 
 parts =
+  caddyprofiledeps
   switch-softwaretype
 
 [caddyprofiledeps]
@@ -14,8 +15,8 @@ extensions = jinja2.ext.do
 extra-context =
 context =
     import json_module json
-    key slapparameter_dict instance-parameter:configuration
-    section instance_parameter_dict instance-parameter
+    key slapparameter_dict slap-configuration:configuration
+    section instance_parameter_dict slap-configuration
     section software_parameter_dict software-parameter-section
     ${:extra-context}
 caucase-jinja2-library = {{ software_parameter_dict['caucase_jinja2_library'] }}
@@ -23,14 +24,14 @@ import-list =
   file caucase :caucase-jinja2-library
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${dynamic-profile-caddy-replicate:rendered}
-RootSoftwareInstance = ${dynamic-profile-caddy-replicate:rendered}
-custom-personal = ${dynamic-profile-caddy-replicate:rendered}
-single-default = ${dynamic-profile-caddy-frontend:rendered}
-single-custom-personal = ${dynamic-profile-caddy-frontend:rendered}
-replicate = ${dynamic-profile-caddy-replicate:rendered}
-kedifa = ${dynamic-profile-kedifa:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+default = dynamic-profile-caddy-replicate:rendered
+RootSoftwareInstance = ${:default}
+custom-personal = dynamic-profile-caddy-replicate:rendered
+single-default = dynamic-profile-caddy-frontend:rendered
+single-custom-personal = dynamic-profile-caddy-frontend:rendered
+replicate = dynamic-profile-caddy-replicate:rendered
+kedifa = dynamic-profile-kedifa:rendered
 
 [software-parameter-section]
 {% for key,value in software_parameter_dict.iteritems() %}
@@ -64,7 +65,7 @@ filename = instance-kedifa.cfg
 extra-context =
   raw software_type kedifa
 
-[instance-parameter]
+[slap-configuration]
 # Fetches parameters defined in SlapOS Master for this instance.
 # Always the same.
 recipe = slapos.cookbook:slapconfiguration.serialised

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

-{%- if software_type == slap_software_type %}
 {%- set kedifa_updater_mapping = [] %}
 {%- set cached_server_dict = {} %}
 {%- set backend_slave_list = [] %}
@@ -187,7 +186,7 @@ context =
 {%-   set furled = furl_module.furl(frontend_configuration['slave-introspection-secure_access']) %}
 {%-   do furled.set(username = slave_reference.lower()) %}
 {%-   do furled.set(password = '${'+ slave_password_section +':passwd}') %}
-{%-   do furled.set(path = slave_reference.lower() + '/') %}
+{%-   do furled.set(path = slave_reference + '/') %}
 {#-   We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
 {%-   set slave_log_access_url = urlparse_module.unquote(furled.tostr()) %}
 {%-   do slave_publish_dict.__setitem__('log-access', slave_log_access_url) %}
@@ -237,7 +236,7 @@ context =
 {#-   Set slave logrotate entry #}
 [{{slave_log_directory_section}}]
 recipe = slapos.cookbook:mkdirectory
-log-directory = {{ '${slave-log-directory-dict:' + slave_reference.lower() + '}' }}
+log-directory = {{ '${slave-log-directory-dict:' + slave_reference + '}' }}
 
 [{{slave_logrotate_section}}]
 <= logrotate-entry-base
@@ -399,8 +398,8 @@ recipe = slapos.cookbook:publish
 {#- Define IPv6 to IPV4 tunneling #}
 [tunnel-6to4-base]
 recipe = slapos.cookbook:wrapper
-ipv4 = ${slap-network-information:local-ipv4}
-ipv6 = ${slap-network-information:global-ipv6}
+ipv4 = ${slap-configuration:ipv4-random}
+ipv6 = ${slap-configuration:ipv6-random}
 wrapper-path = {{ directory['service'] }}/6tunnel-${:ipv6-port}
 command-line = {{ software_parameter_dict['sixtunnel'] }}/bin/6tunnel -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
@@ -509,7 +508,7 @@ extra-context =
 {{ key }} = {{ value }}
 {%- endfor %}
 local-ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
-global-ipv6 = ${slap-network-information:global-ipv6}
+global-ipv6 = ${slap-configuration:ipv6-random}
 request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
 backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
 backend-connect-retries =  {{ dumps('' ~ configuration['backend-connect-retries']) }}
@@ -583,12 +582,12 @@ update-command = ${:command}
 command =
   if ! [ -f ${:key} ] && ! [ -f ${:certificate} ] ; then
     openssl req -new -newkey rsa:2048 -sha256 -subj \
-      "/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-network-information:global-ipv6}" \
+      "/O={{ expose_csr_id_organization }}/OU={{ expose_csr_id_organizational_unit }}/CN=${slap-configuration:ipv6-random}" \
       -days 5 -nodes -x509 -keyout ${:key} -out ${:certificate}
   fi
 
 [expose-csr_id-configuration]
-ip = ${slap-network-information:global-ipv6}
+ip = ${slap-configuration:ipv6-random}
 port = 17001
 key = ${certificate-csr_id:key}
 certificate = ${certificate-csr_id:certificate}
@@ -659,5 +658,4 @@ commands =
 module = check_command_execute
 name = ${:_buildout_section_name_}.py
 config-command =
-  ${logrotate:wrapper-path} -d
-{%- endif %} {# if software_type == slap_software_type #}
+  ${logrotate:wrapper-path} -d
\ No newline at end of file

software/caddy-frontend/templates/nginx-eventsource-slave.conf.in

-{% set url = slave_parameter.get('url') %}
-{% set https_url = slave_parameter.get('https-url', url) %}
-{% if url.startswith("http://") or url.startswith("https://") %}
-{%   set upstream = url.split("/")[2] %}
-{%   set https_upstream = https_url.split("/")[2] %}
-{%   set protocol = url.split("/")[0] %}
-{%   set https_protocol = https_url.split("/")[0] %}
-{%   set proxy_pass = '%s//%s' % (protocol, slave_parameter.get('slave_reference')) %}
-{%   set https_proxy_pass = '%s//https_%s' % (protocol, slave_parameter.get('slave_reference')) %}
-
-# TODO-Caddy upstream {{ slave_parameter.get('slave_reference') }} {
-# TODO-Caddy  server {{ upstream }};
-# TODO-Caddy 
-
-# TODO-Caddy pstream https_{{ slave_parameter.get('slave_reference') }} {
-# TODO-Caddy  server {{ https_upstream }};
-# TODO-Caddy 
-
-
-# TODO-Caddy server {
-# TODO-Caddy  listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_http_port'] }};
-# TODO-Caddy 
-# TODO-Caddy  server_name {{ slave_parameter.get('custom_domain') }};
-# TODO-Caddy 
-# TODO-Caddy  error_log {{ slave_parameter.get('error_log') }} error; 
-# TODO-Caddy  access_log {{ slave_parameter.get('access_log') }} custom;
-# TODO-Caddy  
-# TODO-Caddy  location /pub {
-# TODO-Caddy    push_stream_publisher;
-# TODO-Caddy    push_stream_channels_path $arg_id;
-# TODO-Caddy    # store messages in memory
-# TODO-Caddy    push_stream_store_messages  off;
-# TODO-Caddy 
-# TODO-Caddy    # Message size limit
-# TODO-Caddy    # client_max_body_size MUST be equal to client_body_buffer_size or
-# TODO-Caddy    # you will be sorry.
-# TODO-Caddy    client_max_body_size 16k;
-# TODO-Caddy    client_body_buffer_size 16k;
-# TODO-Caddy 
-# TODO-Caddy   }
-# TODO-Caddy 
-# TODO-Caddy   location ~ /sub/(.*) {
-# TODO-Caddy     # activate subscriber mode for this location
-# TODO-Caddy     add_header "Access-Control-Allow-Origin" "*";
-# TODO-Caddy     add_header 'Access-Control-Allow-Credentials' 'false';
-# TODO-Caddy     add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
-# TODO-Caddy     add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
-# TODO-Caddy 
-# TODO-Caddy     push_stream_subscriber eventsource;
-# TODO-Caddy     # positional channel path
-# TODO-Caddy     push_stream_channels_path $1;
-# TODO-Caddy 
-# TODO-Caddy     # content-type
-# TODO-Caddy     default_type "text/event-stream; charset=utf-8";
-# TODO-Caddy  }
-# TODO-Caddy 
-# TODO-Caddy 
-
-# TODO-Caddy server {
-# TODO-Caddy  listen {{ slave_parameter['local_ipv4'] }}:{{ slave_parameter['nginx_https_port'] }} ssl;
-# TODO-Caddy 
-# TODO-Caddy  server_name {{ slave_parameter.get('custom_domain') }};
-# TODO-Caddy 
-# TODO-Caddy  error_log {{ slave_parameter.get('error_log') }} error;
-# TODO-Caddy  access_log {{ slave_parameter.get('access_log') }} custom;
-# TODO-Caddy 
-# TODO-Caddy  ssl on;
-# TODO-Caddy  
-# TODO-Caddy  ssl_session_timeout 5m;
-# TODO-Caddy  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
-# TODO-Caddy  ssl_ciphers 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5';
-# TODO-Caddy  ssl_prefer_server_ciphers on;
-# TODO-Caddy  ssl_session_cache shared:SSL:10m;
-
-# TODO-Caddy  location /pub {
-# TODO-Caddy    push_stream_publisher;
-# TODO-Caddy    push_stream_channels_path $arg_id;
-# TODO-Caddy    # store messages in memory
-# TODO-Caddy    push_stream_store_messages  off;
-# TODO-Caddy 
-# TODO-Caddy    # Message size limit
-# TODO-Caddy    # client_max_body_size MUST be equal to client_body_buffer_size or
-# TODO-Caddy    # you will be sorry.
-# TODO-Caddy    client_max_body_size 16k;
-# TODO-Caddy    client_body_buffer_size 16k;
-# TODO-Caddy 
-# TODO-Caddy   }
-# TODO-Caddy 
-# TODO-Caddy   location ~ /sub/(.*) {
-# TODO-Caddy     # activate subscriber mode for this location
-# TODO-Caddy     add_header "Access-Control-Allow-Origin" "*";
-# TODO-Caddy     add_header 'Access-Control-Allow-Credentials' 'false';
-# TODO-Caddy     add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
-# TODO-Caddy     add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
-# TODO-Caddy 
-# TODO-Caddy     push_stream_subscriber eventsource;
-# TODO-Caddy     # positional channel path
-# TODO-Caddy     push_stream_channels_path $1;
-# TODO-Caddy 
-# TODO-Caddy     # content-type
-# TODO-Caddy     default_type "text/event-stream; charset=utf-8";
-# TODO-Caddy  }
-# TODO-Caddy}
-{% endif %}

software/caddy-frontend/test/test.py

@@ -1479,10 +1479,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
         'websocket-path-list': '////ws//// /with%20space/',
         'websocket-transparent': 'false',
       },
-      # 'type-eventsource': {
-      #   'url': cls.backend_url,
-      #   'type': 'eventsource',
-      # },
       'type-redirect': {
         'url': cls.backend_url,
         'type': 'redirect',
@@ -1817,7 +1813,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
   def test_server_polluted_keys_removed(self):
     buildout_file = os.path.join(
-      self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
+      self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
     for line in [
       q for q in open(buildout_file).readlines()
       if q.startswith('config-slave-list') or q.startswith(
@@ -3261,54 +3257,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     )
     self.assertFalse('x-real-ip' in j['Incoming Headers'])
 
-  @skip('Feature postponed')
-  def test_type_eventsource(self):
-    # Caddy: For event source, if I understand
-    #        https://github.com/mholt/caddy/issues/1355 correctly, we could use
-    #        Caddy as a proxy in front of nginx-push-stream . If we have a
-    #        "central shared" caddy instance, can it handle keeping connections
-    #        opens for many clients ?
-    parameter_dict = self.parseSlaveParameterDict('type-eventsource')
-    self.assertLogAccessUrlWithPop(parameter_dict)
-    self.assertEqual(
-      {
-        'domain': 'typeeventsource.nginx.example.com',
-        'replication_number': '1',
-        'url': 'http://typeeventsource.nginx.example.com',
-        'site_url': 'http://typeeventsource.nginx.example.com',
-        'secure_access': 'https://typeeventsource.nginx.example.com',
-        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
-      },
-      parameter_dict
-    )
-
-    result = fakeHTTPSResult(
-      parameter_dict['domain'], 'pub',
-      #  NGINX_HTTPS_PORT
-    )
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
-
-    self.assertEqual(
-      '',
-      result.content
-    )
-    headers = result.headers.copy()
-    self.assertKeyWithPop('Expires', headers)
-    self.assertKeyWithPop('Date', headers)
-    self.assertEqual(
-      {
-        'X-Nginx-PushStream-Explain': 'No channel id provided.',
-        'Content-Length': '0',
-        'Cache-Control': 'no-cache, no-store, must-revalidate',
-        'Connection': 'keep-alive',
-        'Server': 'nginx'
-      },
-      headers
-    )
-
   def test_type_redirect(self):
     parameter_dict = self.assertSlaveBase('type-redirect')
 
@@ -3629,7 +3577,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
   def test_enable_cache_server_alias(self):
@@ -3671,7 +3619,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
     result = fakeHTTPResult(
@@ -3788,7 +3736,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
     # BEGIN: Check that squid.log is correctly filled in
@@ -3990,7 +3938,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
     # check stale-if-error support is really respected if not present in the
@@ -4133,7 +4081,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
     try:
@@ -4180,7 +4128,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertNotEqual(via, None)
     self.assertRegexpMatches(
       via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9.0.1\)$'
+      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/9\.0\.[0-9]+\)$'
     )
 
   def test_enable_http2_false(self):
@@ -4642,7 +4590,7 @@ class TestReplicateSlaveOtherDestroyed(SlaveHttpFrontendTestCase):
     self.slap.waitForInstance(self.instance_max_retry)
 
     buildout_file = os.path.join(
-      self.getMasterPartitionPath(), 'buildout-switch-softwaretype.cfg')
+      self.getMasterPartitionPath(), 'instance-caddy-replicate.cfg')
     with open(buildout_file) as fh:
       buildout_file_content = fh.read()
       node_1_present = re.search(

software/cloudooo/software-common.cfg

@@ -100,10 +100,5 @@ url = ${:_profile_base_location_}/${:filename}
 mode = 640
 
 [versions]
-# Required by:
-# cloudooo==1.2.6.dev0
 argparse = 1.4.0
-
-# Required by:
-# cloudooo==1.2.6.dev0
 pyPdf = 1.13

software/erp5testnode/buildout.hash.cfg

@@ -14,8 +14,8 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 307663d73ef3ef94b02567ecd322252e
+md5sum = d9c7c3acb975d44c6c66ca32c550ac58
 
 [template-default]
-filename = instance-default.cfg
-md5sum = 24cc143b1886d443a4c29dcb8147a01c
+filename = instance-default.cfg.jinja.in
+md5sum = ffb6b74b55e7ca01666254353ae1cebe

software/erp5testnode/instance-default.cfg → software/erp5testnode/instance-default.cfg.jinja.in

-[buildout]
+{%- set slapparameter_dict = dict(default_parameter_dict, **slapparameter_dict) -%}
+
 
+[buildout]
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -18,6 +20,7 @@ parts =
   resiliency-exclude-file
   shellinabox-frontend-reload
   promises
+  testnode-compatibility
 
 [monitor-publish]
 recipe = slapos.cookbook:publish
@@ -35,20 +38,24 @@ slapos-directory = $${directory:slapos}
 working-directory = $${directory:testnode}
 test-suite-directory = $${directory:test-suite}
 shared-part-list = $${directory:shared}
-proxy-host = $${slap-network-information:local-ipv4}
+proxy-host = {{ partition_ipv4 }}
 proxy-port = 5000
 log-directory = $${directory:log}
 srv-directory = $${rootdirectory:srv}
 software-directory = $${directory:software}
 run-directory = $${directory:run}
-test-node-title = $${slap-parameter:test-node-title}
-node-quantity = $${slap-parameter:node-quantity}
-ipv4-address = $${slap-network-information:local-ipv4}
-ipv6-address = $${slap-network-information:global-ipv6}
-test-suite-master-url = $${slap-parameter:test-suite-master-url}
-instance-dict = $${slap-parameter:instance-dict}
-software-path-list = $${slap-parameter:software-path-list}
-keep-log-days = $${slap-parameter:keep-log-days}
+test-node-title = {{ slapparameter_dict['test-node-title'] }}
+node-quantity = {{ slapparameter_dict['node-quantity'] }}
+ipv4-address = {{ partition_ipv4 }}
+ipv6-address = {{ partition_ipv6 }}
+test-suite-master-url = {{ slapparameter_dict['test-suite-master-url'] }}
+instance-dict = {{ slapparameter_dict['instance-dict'] }}
+{%- if isinstance(slapparameter_dict['software-path-list'], str) %}
+software-path-list = {{ slapparameter_dict['software-path-list'] }}
+{%- else %}
+software-path-list = {{ json.dumps(slapparameter_dict['software-path-list']) }}
+{%- endif %}
+keep-log-days = {{ slapparameter_dict['keep-log-days'] }}
 git-binary = ${git:location}/bin/git
 slapos-binary = ${buildout:bin-directory}/slapos
 testnode = ${buildout:bin-directory}/testnode
@@ -59,7 +66,7 @@ httpd-conf-file = $${rootdirectory:etc}/httpd.conf
 httpd-wrapper = $${rootdirectory:bin}/httpd
 httpd-port = 9080
 httpd-software-access-port = 9081
-httpd-ip = $${slap-network-information:global-ipv6}
+httpd-ip = {{ partition_ipv6 }}
 httpd-log-directory = $${basedirectory:log}
 httpd-software-directory = $${directory:software}
 httpd-cert-file = $${rootdirectory:etc}/httpd-public.crt
@@ -117,7 +124,7 @@ template = inline:
       /
     }
   }
-ipv6 = $${slap-network-information:global-ipv6}
+ipv6 = {{ partition_ipv6 }}
 hostname = [$${:ipv6}]
 port = 8080
 username = testnode
@@ -200,7 +207,7 @@ run = $${rootdirectory:var}/run
 [directory]
 recipe = slapos.cookbook:mkdirectory
 slapos = $${rootdirectory:srv}/slapos
-testnode = $${rootdirectory:srv}/testnode
+testnode = $${buildout:directory}/t
 shared = $${rootdirectory:srv}/shared
 test-suite = $${rootdirectory:srv}/test_suite
 log = $${basedirectory:log}/testnode
@@ -209,6 +216,16 @@ software = $${rootdirectory:srv}/software
 shellinabox = $${rootdirectory:srv}/shellinabox
 ca-dir = $${rootdirectory:srv}/ca
 
+[testnode-compatibility]
+# Remove old ~/srv/testnode
+recipe = slapos.recipe.build
+update =
+  import os
+  from zc.buildout.rmtree import rmtree
+  old_testnode_path = self.buildout['rootdirectory']['srv'] + '/testnode'
+  if os.path.exists(old_testnode_path):
+    rmtree(old_testnode_path)
+
 [resiliency-exclude-file]
 # Generate rdiff exclude file in case of resiliency
 recipe = collective.recipe.template
@@ -239,6 +256,7 @@ config-url = https://[$${testnode:httpd-ip}]:$${testnode:httpd-port}
 recipe =
 instance-promises =
   $${shellinabox-frontend-listen-promise:name}
+# $${shellinabox-frontend-available-promise:name}
   $${testnode-log-frontend-promise:name}
 
 [shellinabox-frontend-listen-promise]
@@ -248,15 +266,18 @@ name = $${:_buildout_section_name_}.py
 config-host = $${shellinabox-frontend:hostname}
 config-port = $${shellinabox-frontend:port}
 
+## This promise fails in a test suite
+## due to ports conflict with the testnode of the test suite
+# [shellinabox-frontend-available-promise]
+# <= monitor-promise-base
+# module = check_url_available
+# name = $${:_buildout_section_name_}.py
+# config-url = https://$${shellinabox-frontend-config:hostname}:$${shellinabox-frontend-config:port}
+# config-username = $${shellinabox-frontend-config:username}
+# config-password = $${shellinabox-frontend-config:passwd}
+
 [testnode-log-frontend-promise]
 <= monitor-promise-base
 module = check_url_available
 name = $${:_buildout_section_name_}.py
 config-url = $${testnode-log-frontend:connection-secure_access}
-
-[slap-parameter]
-node-quantity = 1
-test-suite-master-url =
-instance-dict =
-software-path-list = ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"]
-keep-log-days = 15

software/erp5testnode/instance.cfg

@@ -6,6 +6,34 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
 [switch_softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = ${template-default:output}
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = $${:default}
+default = instance-default:rendered
+
+[instance-default]
+recipe = slapos.recipe.template:jinja2
+template = ${template-default:output}
+rendered = $${buildout:directory}/instance-default.cfg
+context =
+  import json json
+  jsonkey default_parameter_dict :default-parameters
+  key slapparameter_dict slap-configuration:configuration
+  key partition_ipv6 slap-configuration:ipv6-random
+  key partition_ipv4 slap-configuration:ipv4-random
+default-parameters =
+  {
+    "node-quantity": 1,
+    "test-suite-master-url": "",
+    "instance-dict": "",
+    "software-path-list": ["https://lab.nexedi.com/nexedi/slapos/raw/1.0.181/software/seleniumrunner/software.cfg"],
+    "keep-log-days": 15
+  }

software/erp5testnode/software.cfg

@@ -25,20 +25,20 @@ parts =
 [eggs]
 recipe = zc.recipe.egg
 eggs =
-  erp5.util[testnode]
+  erp5.util[testnode,benchmark,scalability_tester]
   ${lxml-python:egg}
 
 
 [template]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-default]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-default.cfg
-output = ${buildout:directory}/template-default.cfg
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template-default.cfg.jinja
 mode = 0644
 
 [versions]
@@ -52,23 +52,9 @@ soupsieve = 1.8
 waitress = 1.2.1
 z3c.etestbrowser = 2.0.1
 zope.testbrowser = 5.3.2
-
-# Required by:
-# zope.testbrowser==5.3.2
 WSGIProxy2 = 0.4.6
-
-# Required by:
-# WebTest==2.0.33
 beautifulsoup4 = 4.7.1
-
-# Required by:
-# zope.testbrowser==5.3.2
 zope.cachedescriptors = 4.3.1
-
-# Required by:
-# zope.schema==4.9.3
 zope.event = 4.4
-
-# Required by:
-# zope.testbrowser==5.3.2
 zope.schema = 4.9.3
+slapos.tool.nosqltester = 0.0.4.dev-r45972

software/erp5testnode/test/README.md

+Tests for erp5testnode software release

software/erp5testnode/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.erp5testnode'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' erp5testnode",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/erp5testnode/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestnodeTest(SlapOSInstanceTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {"test-node-title": "hello"}
+
+  def test(self):
+    connexion_parameters = self.computer_partition.getConnectionParameterDict()
+    self.assertIn('url', connexion_parameters)
+    self.assertIn('frontend-url', connexion_parameters)
+    self.assertIn('log-frontend-url', connexion_parameters)

software/fluentd/software.cfg

@@ -23,4 +23,4 @@ context =
 [fluentd]
 gems +=
   fluent-plugin-wendelin==0.4
-  fluent-plugin-bin
+  fluent-plugin-bin==0.2

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = dc3f318e8a3aa7a59f9394118543e9e3
+md5sum = 47e8092404feaf3f28ee6469523689ac
 
 [watcher]
 _update_hash_filename_ = watcher.in
@@ -54,7 +54,7 @@ md5sum = 0f1ec4077dab586cc003ae13f689eda2
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 64ec65b2daa0648453022f3afcbc4da3
+md5sum = 6b34d4b96ae0067977fa509046d71231
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in

software/gitlab/instance-gitlab.cfg.in

@@ -5,7 +5,6 @@
 #      throughput compared to tcp over loopback).
 [buildout]
 extends =
-  {{ gitlab_parameters_cfg }}
   {{ monitor_template }}
 parts =
     directory
@@ -50,26 +49,15 @@ offline = true
 ##################################
 
 [instance-parameter]
-# std stuff to fetch slapos instance parameters
-recipe  = slapos.cookbook:slapconfiguration
-computer= ${slap-connection:computer-id}
-partition=${slap-connection:partition-id}
-url     = ${slap-connection:server-url}
-key     = ${slap-connection:key-file}
-cert    = ${slap-connection:cert-file}
+{#- There are dangerous keys like recipe, etc #}
+{#- XXX: Some other approach would be useful #}
+{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
+{%- for key, value in instance_parameter_dict.iteritems() -%}
+{%-   if key not in DROP_KEY_LIST %}
+{{ key }} = {{ value }}
+{%-   endif -%}
+{%- endfor %}
 
-# autogenerated gitlab instance parameters
-<= gitlab-parameters
-
-# adjust/override some default settings:
-
-# automatically load all available CPUs
-configuration.unicorn_worker_processes  = {{ multiprocessing.cpu_count() + 1 }}
-configuration.nginx_worker_processes    = {{ multiprocessing.cpu_count() }}
-
-
-# gitlab non-native parameters
-configuration.icp_license  =
 
 
 # for convenience

software/gitlab/instance.cfg.in

 # GitLab "switch-softwaretype" instance
 [buildout]
-parts = switch-softwaretype
+extends =
+  ${gitlab-parameters.cfg:target}
+parts =
+    switch-softwaretype
 
 # std stuff for slapos instance
 eggs-directory = ${buildout:eggs-directory}
@@ -9,14 +12,46 @@ offline = true
 
 
 [switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-gitlab  = $${instance-gitlab.cfg:rendered}
-gitlab-export   = $${instance-gitlab-export.cfg:rendered}
-gitlab-test = $${instance-gitlab-test.cfg:rendered}
-default = $${:gitlab}
+recipe = slapos.cookbook:switch-softwaretype
+gitlab = instance-gitlab.cfg:rendered
+gitlab-export = instance-gitlab-export.cfg:rendered
+gitlab-test = instance-gitlab-test.cfg:rendered
+RootSoftwareInstance = $${:gitlab}
 # TODO -import, -pull-backup
 
 
+[worker-processes]
+recipe = slapos.recipe.build
+init =
+  import multiprocessing
+  cpu_count = multiprocessing.cpu_count()
+  # automatically load all available CPUs
+  options['unicorn-worker-processes'] = cpu_count + 1
+  options['nginx-worker-processes'] = cpu_count
+
+
+[slap-configuration]
+# std stuff to fetch slapos instance parameters
+recipe  = slapos.cookbook:slapconfiguration
+computer= $${slap-connection:computer-id}
+partition=$${slap-connection:partition-id}
+url     = $${slap-connection:server-url}
+key     = $${slap-connection:key-file}
+cert    = $${slap-connection:cert-file}
+
+# autogenerated gitlab instance parameters
+<= gitlab-parameters
+
+# adjust/override some default settings:
+
+configuration.unicorn_worker_processes = $${worker-processes:unicorn-worker-processes}
+configuration.nginx_worker_processes   = $${worker-processes:nginx-worker-processes}
+
+
+# gitlab non-native parameters
+configuration.icp_license  =
+
+
 # macro: render instance-*.cfg from instance-*.cfg.in
 [instance-cfg]
 recipe  = slapos.recipe.template:jinja2
@@ -25,13 +60,13 @@ rendered= $${buildout:directory}/$${:_buildout_section_name_}
 context =
     import os os
     import pwd pwd
-    import multiprocessing multiprocessing
 
     key bin_directory           buildout:bin-directory
     key eggs_directory          buildout:eggs-directory
     key develop_eggs_directory  buildout:develop-eggs-directory
     raw gitlab_repository_location          ${gitlab-repository:location}
     raw gitlab_shell_repository_location    ${gitlab-shell-repository:location}
+    section instance_parameter_dict slap-configuration
 
 # program binaries
     raw bash_bin                    ${bash:location}/bin/bash
@@ -67,7 +102,6 @@ context =
 # config files
     raw database_yml_in             ${database.yml.in:target}
     raw gitconfig_in                ${gitconfig.in:target}
-    raw gitlab_parameters_cfg       ${gitlab-parameters.cfg:target}
     raw monitor_template            ${monitor2-template:rendered}
     raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
     raw gitlab_unicorn_startup_in   ${gitlab-unicorn-startup.in:target}

software/headless-chromium/README.md

+# Headless Chromium
+This software release compiles and runs a headless Chromium shell and
+exposes an interface to connect to it remotely from another browser.
+
+After deployment, the instance is configured like this:
+
+```
+   Caddy frontend
+     |
+   (HTTPS, IPv6)
+     |
+   Nginx proxy, basic authentication
+     |
+   (HTTP, IPv4)
+     |
+   Chromium shell
+```
+
+The proxy is necessary because Chromium only accepts local connections
+for remote debugging.
+
+## Parameters
+The following instance parameters can be configured:
+
+- target-url:             URL for Chromium to load on startup.
+- remote-debugging-port:  Port for Chromium to listen on.
+- nginx-proxy-port:       Port for Ningx proxy to listen on.
+- monitor-httpd-port:     Port for monitor.
+
+See `instance-headless-chromium-input-schema.json` for default values.

software/headless-chromium/buildout.hash.cfg

+[template-cfg]
+filename = instance.cfg.in
+md5sum = 5dfeeb5eca125dcaa5f9e537f941dd41
+
+[instance-headless-chromium]
+_update_hash_filename_ = instance-headless-chromium.cfg.in
+md5sum = fad685238b26ca20537c12ce7432e7e7
+
+[template-nginx-conf]
+_update_hash_filename_ = templates/nginx.conf.in
+md5sum = c4d09d2b819f624087ef4c38551dfe2f
+
+[template-mime-types]
+_update_hash_filename_ = templates/mime-types.in
+md5sum = 4ef94a7b458d885cd79ba0b930a5727e

software/headless-chromium/instance-headless-chromium-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "target-url": {
+      "description": "URL for Chromium to load on startup.",
+      "title": "Target URL",
+      "type": "string",
+      "default": "https://www.example.com"
+    },
+    "remote-debugging-port": {
+      "description": "Port for Chromium to listen on.",
+      "title": "Remote Debugging Port",
+      "type": "integer",
+      "default": 8081
+    },
+    "nginx-proxy-port": {
+      "description": "Port for Nginx proxy to listen on.",
+      "title": "Nginx Proxy Port",
+      "type": "integer",
+      "default": 8082
+    },
+    "monitor-httpd-port": {
+      "description": "Port for monitor frontend.",
+      "title": "Monitor Httpd Port",
+      "type": "integer",
+      "default": 8083
+    }
+  }
+}

software/headless-chromium/instance-headless-chromium-output-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Values returned by headless Chromium instantiation",
+  "properties": {
+    "frontend-url": {
+      "description": "URL to access remote debugging interface",
+      "type": "string"
+    },
+    "username": {
+      "description": "Username for remote debugging interface",
+      "type": "string"
+    },
+    "password": {
+      "description": "Password for remote debugging interface",
+      "type": "string"
+    },
+    "monitor-base-url": {
+      "description": "Base URL used by monitor",
+      "type": "string"
+    },
+    "monitor-setup-url": {
+      "description": "One-click link to setup and monitor feeds",
+      "type": "string"
+    },
+    "proxy-url": {
+      "description": "Raw IPv6 address used by Nginx proxy",
+      "type": "string"
+    },
+    "remote-debug-url": {
+      "description": "Local IPv4 address used by Chromium",
+      "type": "string"
+    }
+  }
+}

software/headless-chromium/instance-headless-chromium.cfg.in

+{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
+
+[buildout]
+parts =
+  chromium-launcher
+  generate-passwd-file
+  nginx-config
+  nginx-mime-types
+  nginx-launcher
+  logrotate-entry-nginx
+  publish-connection-information
+  frontend-ok-promise
+  frontend-secure-promise
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+extends = {{ parameter_list['template-monitor'] }}
+
+# Create necessary directories.
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+tmp = ${:home}/tmp
+log = ${:home}/log
+etc = ${:home}/etc
+ssl = ${:etc}/ssl
+service = ${:etc}/service
+
+# Options for instance configuration. See README.md for a list of
+# options that can be configured when requesting an instance.
+[headless-chromium]
+ipv4 = {{ partition_ipv4 }}
+ipv6 = {{ partition_ipv6 }}
+remote-debugging-port = {{ parameter_dict['remote-debugging-port'] }}
+url = {{ parameter_dict['target-url'] }}
+remote-debugging-address = ${:ipv4}:${:remote-debugging-port}
+devtools-frontend-root = {{ parameter_list['devtools-frontend'] }}
+
+nginx-port = {{ parameter_dict['nginx-proxy-port'] }}
+proxy-address = [${:ipv6}]:${:nginx-port}
+nginx-config-target = ${directory:etc}/nginx.conf
+nginx-pid-path = ${directory:log}/nginx.pid
+nginx-temp-path = ${directory:tmp}
+nginx-error-log = ${directory:log}/nginx-error.log
+nginx-access-log = ${directory:log}/nginx-access.log
+nginx-htpasswd-file = ${directory:etc}/.htpasswd
+nginx-key-file = ${frontend-instance-certificate:key-file}
+nginx-cert-file = ${frontend-instance-certificate:cert-file}
+nginx-mime-types = ${directory:etc}/mime-types
+
+
+# Create a launcher script in /etc/service for the headless shell
+# executable.
+[chromium-launcher]
+recipe = slapos.recipe.template:jinja2
+template =
+  inline:#!/bin/sh
+
+  export FONTCONFIG_FILE=${font-config:rendered}
+  exec {{ parameter_list['chromium-wrapper'] }} \
+    --remote-debugging-address=${headless-chromium:ipv4} \
+    --remote-debugging-port=${headless-chromium:remote-debugging-port} \
+    ${headless-chromium:url}
+rendered = ${directory:service}/chromium
+
+
+# Configure and launch the proxy server.
+[nginx-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-nginx-config'] }}
+rendered = ${headless-chromium:nginx-config-target}
+mode = 700
+context =
+  section param_headless_chromium headless-chromium
+
+[nginx-mime-types]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-mime-types'] }}
+rendered = ${headless-chromium:nginx-mime-types}
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = {{ parameter_list['nginx-location'] }}/sbin/nginx -c ${headless-chromium:nginx-config-target}
+wrapper-path = ${directory:service}/nginx
+
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${headless-chromium:nginx-error-log} ${headless-chromium:nginx-access-log}
+
+[frontend-instance-password]
+recipe = slapos.cookbook:generate.password
+username = admin
+bytes = 12
+
+[generate-passwd-file]
+recipe = plone.recipe.command
+command =
+  echo -n '${frontend-instance-password:username}:' > ${headless-chromium:nginx-htpasswd-file}
+  openssl passwd -apr1 '${frontend-instance-password:passwd}' >> ${headless-chromium:nginx-htpasswd-file}
+environment =
+  PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+# Generate a self-signed TLS certificate.
+[frontend-instance-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${:key-file} ]
+  then
+    openssl req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${:common-name}" \
+      -newkey rsa:1024 -keyout ${:key-file} \
+      -out ${:cert-file}
+    openssl x509 -addtrust serverAuth \
+      -in ${:cert-file} \
+      -out ${:cert-file}
+  fi
+update-command = ${:command}
+key-file = ${directory:ssl}/${:_buildout_section_name_}.key
+cert-file = ${directory:ssl}/${:_buildout_section_name_}.cert
+common-name = ${headless-chromium:ipv6}
+environment =
+  PATH={{ parameter_list['openssl-location'] }}/bin:%(PATH)s
+
+
+# Generate a fonts.conf file.
+[font-config]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_list['template-fonts-conf'] }}
+rendered = ${directory:etc}/fonts.conf
+context =
+  key cachedir :cache-dir
+  key fonts :fonts
+  key includes :includes
+cache-dir =
+  ${directory:etc}/.fontconfig.cache
+fonts =
+  {{ parameter_list['liberation-fonts-location'] }}
+includes =
+  {{ parameter_list['fontconfig-location'] }}/etc/fonts/conf.d
+
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+<= monitor-publish
+remote-debug-url = http://${headless-chromium:remote-debugging-address}
+proxy-url = https://${headless-chromium:proxy-address}
+frontend-url = ${remote-debugging-frontend:connection-secure_access}
+username = ${frontend-instance-password:username}
+password = ${frontend-instance-password:passwd}
+
+# Request a frontend URL from the CDN for the remote debugging interface.
+[remote-debugging-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Headless Chromium Remote Debugging Frontend
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = https://${headless-chromium:proxy-address}
+config-https-only = true
+config-type = websocket
+config-websocket-path-list = /devtools
+return = domain secure_access
+
+
+# Monitoring: check that the Chromium process is alive and responding to
+# requests through the proxy.
+[monitor-instance-parameter]
+monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
+
+# Promise to make sure the remote debugging frontend returns 200 when
+# queried with the correct credentials.
+[frontend-ok-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-ok.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-username = ${frontend-instance-password:username}
+config-password = ${frontend-instance-password:passwd}
+
+# Promise to make sure that the remote debugging frontend returns 401
+# when queried with no credentials.
+[frontend-secure-promise]
+<= monitor-promise-base
+module = check_url_available
+name = headless-chromium-frontend-secure.py
+url = ${remote-debugging-frontend:connection-secure_access}
+config-url = ${:url}
+config-http-code = 401

software/headless-chromium/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+[profile-common]
+openssl-location = {{ openssl_location }}
+nginx-location = {{ nginx_location }}
+liberation-fonts-location = {{ liberation_fonts_location }}
+fontconfig-location = {{ fontconfig_location }}
+chromium-wrapper = {{ chromium_wrapper }}
+devtools-frontend = {{ devtools_frontend }}
+template-nginx-config = {{ template_nginx_config_target }}
+template-fonts-conf = {{ template_fonts_conf_target }}
+template-monitor = {{ template_monitor }}
+template-mime-types = {{ template_mime_types_target }}
+
+[instance-headless-chromium]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_instance_headless_chromium_target }}
+rendered = ${buildout:directory}/${:filename}
+filename = instance-headless-chromium.cfg
+context =
+  section buildout buildout
+  section parameter_list profile-common
+  key partition_ipv4 slap-configuration:ipv4-random
+  key partition_ipv6 slap-configuration:ipv6-random
+  key slapparameter_dict slap-configuration:configuration
+  jsonkey default_parameter_dict :default-parameters
+default-parameters =
+  {
+    "remote-debugging-port": 8081,
+    "nginx-proxy-port": 8082,
+    "target-url": "https://www.example.com",
+    "monitor-httpd-port": 8083
+  }
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = ${:default}
+default = instance-headless-chromium:rendered
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}

software/headless-chromium/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/headless-chromium/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/fonts/buildout.cfg
+  ../../component/fontconfig/buildout.cfg
+
+parts =
+  slapos-cookbook
+  template-cfg
+
+[python]
+part = python3
+
+[template-cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/template.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+  section buildout buildout
+  key openssl_location openssl:location
+  key nginx_location nginx:location
+  key liberation_fonts_location liberation-fonts:location
+  key fontconfig_location fontconfig:location
+  key chromium_wrapper headless-chromium-wrapper:rendered
+  key devtools_frontend headless-chromium:devtools-frontend
+  key template_nginx_config_target template-nginx-conf:target
+  key template_mime_types_target template-mime-types:target
+  key template_fonts_conf_target template-fonts-conf:output
+  key template_instance_headless_chromium_target instance-headless-chromium:target
+  key template_monitor monitor2-template:rendered
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
+[instance-headless-chromium]
+<= download-base
+
+[template-nginx-conf]
+<= download-base
+
+[template-mime-types]
+<= download-base

software/headless-chromium/software.cfg.json

+{
+  "name": "Headless Chromium",
+  "description": "Headless (stripped-down) Chromium shell",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Standalone headless shell",
+      "request": "instance-headless-chromium-input-schema.json",
+      "response": "instance-headless-chromium-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/headless-chromium/templates/mime-types.in

+types {
+	text/html				html htm shtml;
+	text/css				css;
+	text/xml				xml rss;
+	image/gif				gif;
+	image/jpeg				jpeg jpg;
+	application/x-javascript		js;
+	application/atom+xml			atom;
+
+	text/mathml				mml;
+	text/plain				txt;
+	text/vnd.sun.j2me.app-descriptor	jad;
+	text/vnd.wap.wml			wml;
+	text/x-component			htc;
+
+	image/png				png;
+	image/tiff				tif tiff;
+	image/vnd.wap.wbmp			wbmp;
+	image/x-icon				ico;
+	image/x-jng				jng;
+	image/x-ms-bmp				bmp;
+	image/svg+xml				svg svgz;
+
+	application/java-archive		jar war ear;
+	application/mac-binhex40		hqx;
+	application/msword			doc;
+	application/pdf				pdf;
+	application/postscript			ps eps ai;
+	application/rtf				rtf;
+	application/vnd.ms-excel		xls;
+	application/vnd.ms-powerpoint		ppt;
+	application/vnd.wap.wmlc		wmlc;
+	application/vnd.google-earth.kml+xml	kml;
+	application/vnd.google-earth.kmz	kmz;
+	application/x-7z-compressed		7z;
+	application/x-cocoa			cco;
+	application/x-java-archive-diff		jardiff;
+	application/x-java-jnlp-file		jnlp;
+	application/x-makeself			run;
+	application/x-perl			pl pm;
+	application/x-pilot			prc pdb;
+	application/x-rar-compressed		rar;
+	application/x-redhat-package-manager	rpm;
+	application/x-sea			sea;
+	application/x-shockwave-flash		swf;
+	application/x-stuffit			sit;
+	application/x-tcl			tcl tk;
+	application/x-x509-ca-cert		der pem crt;
+	application/x-xpinstall			xpi;
+	application/xhtml+xml			xhtml;
+	application/zip				zip;
+
+	application/octet-stream		bin exe dll;
+	application/octet-stream		deb;
+	application/octet-stream		dmg;
+	application/octet-stream		eot;
+	application/octet-stream		iso img;
+	application/octet-stream		msi msp msm;
+	application/ogg				ogx;
+
+	audio/midi				mid midi kar;
+	audio/mpeg				mpga mpega mp2 mp3 m4a;
+	audio/ogg				oga ogg spx;
+	audio/x-realaudio			ra;
+	audio/webm				weba;
+
+	video/3gpp				3gpp 3gp;
+	video/mp4				mp4;
+	video/mpeg				mpeg mpg mpe;
+	video/ogg				ogv;
+	video/quicktime				mov;
+	video/webm				webm;
+	video/x-flv				flv;
+	video/x-mng				mng;
+	video/x-ms-asf				asx asf;
+	video/x-ms-wmv				wmv;
+	video/x-msvideo				avi;
+}

software/headless-chromium/templates/nginx.conf.in

+pid {{ param_headless_chromium['nginx-pid-path'] }};
+error_log {{ param_headless_chromium['nginx-error-log'] }};
+
+events {
+  worker_connections 1024;
+}
+
+http {
+  access_log {{ param_headless_chromium['nginx-access-log'] }};
+
+  include {{ param_headless_chromium['nginx-mime-types'] }};
+  default_type application/octet-stream;
+
+  types {
+    text/html html;
+    text/css css;
+    application/javascript js;
+  }
+
+  server {
+    listen {{ param_headless_chromium['proxy-address'] }} ssl;
+
+    # Require username/password to access remote debugging port.
+    auth_basic "Remote Debugging";
+    auth_basic_user_file {{ param_headless_chromium['nginx-htpasswd-file'] }};
+
+    # Use self-signed SSL certificate.
+    ssl_certificate {{ param_headless_chromium['nginx-cert-file'] }};
+    ssl_certificate_key {{ param_headless_chromium['nginx-key-file'] }};
+
+    client_body_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    proxy_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    fastcgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    uwsgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+    scgi_temp_path {{ param_headless_chromium['nginx-temp-path'] }};
+
+    # All websocket connections are served from /devtools.
+    location /devtools {
+      proxy_http_version 1.1;
+      proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_set_header Upgrade "websocket";
+      proxy_set_header Connection "Upgrade";
+    }
+
+    # The DevTools frontend is served from /serve_file/@{version_hash}.
+    location ~ "^\/serve_file\/@[0-9a-f]{5,40}\/(.*)" {
+      alias {{ param_headless_chromium['devtools-frontend-root'] }}/$1;
+    }
+
+    location / {
+      proxy_http_version 1.1;
+
+      # The proxy must set the Host header to an IP address, since the
+      # headless Chromium shell refuses to run otherwise, for security
+      # reasons.
+      # See https://bugs.chromium.org/p/chromium/issues/detail?id=813540.
+      proxy_set_header Host {{ param_headless_chromium['remote-debugging-address'] }};
+      proxy_pass http://{{ param_headless_chromium['remote-debugging-address'] }};
+
+      # The browser security policy will prevent us from loading the
+      # Websocket connection without TLS, so we have to go through the
+      # frontend CDN URL. The tricky thing is that the frontend URL is
+      # not available yet when this file is built; what we do instead is
+      # use the given Host header.
+      sub_filter "ws={{ param_headless_chromium['remote-debugging-address'] }}" "wss=$host";
+      sub_filter_once on;
+      sub_filter_types application/json;
+
+      sub_filter "ws://{{ param_headless_chromium['remote-debugging-address'] }}" "wss://$host";
+      sub_filter_types application/json;
+
+      # We want to use our own DevTools frontend rather than
+      # https://chrome-devtools-frontend.appspot.com. There should be a
+      # --custom-devtools-frontend flag for Chromium, but it doesn't
+      # seem to work with the remote debugging port.
+      sub_filter "chrome-devtools-frontend.appspot.com" "$host";
+      sub_filter_types *;
+    }
+  }
+}

software/headless-chromium/test/README.md

+Tests for headless Chromium software release

software/headless-chromium/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.headless-chromium'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS headless Chromium",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/headless-chromium/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '../software.cfg')))
+
+class TestHeadlessChromium(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = self.requestDefaultInstance().getConnectionParameterDict()
+
+  def test_remote_debugging_port(self):
+    # The headless browser should respond at /json with a nonempty list
+    # of available pages, each of which has a webSocketDebuggerUrl and a
+    # devtoolsFrontendUrl.
+    url = self.connection_parameters['remote-debug-url']
+    response = requests.get('%s/json' % url)
+
+    # Check that request was successful and the response was a nonempty
+    # list.
+    self.assertEqual(requests.codes['ok'], response.status_code)
+    self.assertTrue(len(response.json()) > 0)
+
+    # Check that the first page has the correct fields.
+    first_page = response.json()[0]
+    self.assertIn('webSocketDebuggerUrl', first_page)
+    self.assertIn('devtoolsFrontendUrl', first_page)
+
+  def test_devtools_frontend_ok(self):
+    # The proxy should serve the DevTools frontend from
+    # /serve_file/@{hash}/inspector.html, where {hash} is a 5-32 digit
+    # hash.
+    proxyURL = self.connection_parameters['proxy-url']
+    username = self.connection_parameters['username']
+    password = self.connection_parameters['password']
+    frontend = '/serve_file/@aaaaa/inspector.html'
+
+    response = requests.get(proxyURL + frontend, verify=False,
+                            auth=(username, password))
+    self.assertEqual(requests.codes['ok'], response.status_code)

software/jupyter/software.cfg

@@ -91,78 +91,27 @@ tornado = 6.1
 traitlets = 5.0.5
 webencodings = 0.5.1
 widgetsnbextension = 2.0.0
-
-
-# Required by:
-# tornado==4.4.2
 certifi = 2020.6.20
-
-# Required by:
-# notebook==6.1.5
 Send2Trash = 1.5.0
-
-# Required by:
-# notebook==6.1.5
 argon2-cffi = 20.1.0
-
-# Required by:
-# notebook==6.1.5
 nbconvert = 6.0.7
-
-# Required by:
-# ipython==5.3.0
 pathlib2 = 2.2.1
-
-# Required by:
-# statsmodels==0.11.1
 patsy = 0.5.1
-
-# Required by:
-# ipython==5.3.0
 pexpect = 4.8.0
-
-# Required by:
-# ipython==5.3.0
 pickleshare = 0.7.4
-
-# Required by:
-# notebook==6.1.5
 prometheus-client = 0.9.0
-
-# Required by:
-# pathlib2==2.2.1
 scandir = 1.5
-
-# Required by:
-# statsmodels==0.11.1
 pytz = 2020.4
-
-# Required by:
-# statsmodels==0.11.1
 scipy = 1.0.1
-
-# Required by:
-# tornado==6.1
 singledispatch = 3.4.0.3
-
-# Required by:
-# prompt-toolkit==1.0.13
 wcwidth = 0.1.7
-
 jupyter = 1.0.0
 jupyter-console = 5.1.0
-
-# Required by:
-# jupyter==1.0.0
 qtconsole = 4.3.0
-
 et-xmlfile = 1.0.1
 h5py = 2.7.1
 mpmath = 1.0.0
 openpyxl = 2.5.2
 sympy = 1.1.1
 xlrd = 1.1.0
-
-# Required by:
-# openpyxl==2.5.2
 jdcal = 1.4

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = c115ed9d4ff0f785d79cdcacbb0bd1ad
+md5sum = 2114ae8c1e92bd33ef1347f36f567c74
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2

software/monitor/instance.cfg

@@ -6,10 +6,11 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
 [switch_softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${instance-base-monitor:rendered}
-edgetest = $${instance-base-edgetest:rendered}
-edgebot = $${instance-base-edgebot:rendered}
+recipe = slapos.cookbook:switch-softwaretype
+default = instance-base-monitor:rendered
+edgetest = instance-base-edgetest:rendered
+edgebot = instance-base-edgebot:rendered
+RootSoftwareInstance = $${:default}
 
 [instance-base-monitor]
 recipe = slapos.recipe.template:jinja2

software/neoppod/software-common.cfg

@@ -132,11 +132,7 @@ ecdsa = 0.13
 mysqlclient = 1.3.12
 pycrypto = 2.6.1
 pycurl = 7.43.0
-setproctitle = 1.1.10
 cython-zstd = 0.2
-
-# Required by:
-# mock = 3.0.5
 funcsigs = 1.0.2
 
 

software/nginx-push-stream/README.md

+# Nginx Push Stream
+
+This software uses the [HTTP Push Stream](https://www.nginx.com/resources/wiki/modules/push_stream/)
+module of nginx to make a [Server Sent Event](https://html.spec.whatwg.org/multipage/server-sent-events.html)
+server.
+
+Two endpoints are available, published as the following connection parameters:
+
+ - `publisher-url`, which uses`/pub{?id}` format, with `id` being the ID of the channel.
+  Clients can subscribe to this URL and be notified of new messages.
+ - `subscriber-url`, which uses `/sub{/id}` format, with `id` being the ID of the channel.
+  `POST` requests to this URL send new messages to this channel.
+
+

software/nginx-push-stream/buildout.hash.cfg

+[template]
+filename = instance.cfg.in
+md5sum = f9b6d01e29f2edddd9d6f99591976c33
+
+[template-nginx-configuration]
+filename = template-nginx.cfg.in
+md5sum = 022e4b53e1b2db16c4e518fe76f638fa

software/nginx-push-stream/instance-nginx.cfg.in

-[buildout]
-parts =
-  nginx-service
-  htpasswd
-  htpasswd-runner
-  publish-connection-information
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-bin = $${buildout:directory}/bin
-srv = $${buildout:directory}/srv
-var = $${buildout:directory}/var
-run = $${:var}/run
-log = $${:var}/log
-varnginx = $${:var}/nginx
-services = $${:etc}/service
-cron-entries = $${:etc}/cron.d
-www = $${:srv}/www
-ssl = $${:etc}/ssl
-
-#################################
-# Nginx service
-#################################
-[nginx-service]
-recipe = slapos.recipe.template
-url = ${template-nginx-service:output}
-output = $${directory:services}/nginx
-mode = 0700
-virtual-depends =
-  $${nginx-configuration:ip}
-
-[nginx-configuration]
-recipe = slapos.recipe.template
-url = ${template-nginx-configuration:output}
-output = $${directory:etc}/nginx.cfg
-mode = 0600
-access_log = $${directory:log}/nginx-access.log
-error_log = $${directory:log}/nginx-error.log
-ip = $${slap-network-information:global-ipv6}
-local_ip = $${slap-network-information:local-ipv4}
-port = 9443
-publisher_location_prefix = /pub
-publisher_push_stream_store_messages = off
-publisher_client_max_body_size = 16k
-publisher_client_body_buffer_size = 16k
-
-subscriber_allow_origin = '*'
-subscriber_location_prefix = /sub
-# Prevent to use credential if origin is star
-subscriber_allow_credential = 'false'
-subscriber_allow_methods = 'GET, HEAD, OPTIONS'
-subscriber_allow_headers = 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since'
-
-[htpasswd]
-recipe = slapos.cookbook:generate.password
-storage-path = $${directory:etc}/.pwd
-bytes = 8
-
-[htpasswd-runner]
-recipe = plone.recipe.command
-stop-on-error = true
-htpasswd-path = $${directory:etc}/.htpasswd
-command = if [ ! -f "$${:htpasswd-path}" ]; then ${buildout:bin-directory}/htpasswd -cb $${:htpasswd-path} $${:user} $${:password}; fi
-update-command = $${:command}
-user = admin
-password = $${htpasswd:passwd}
-
-[publish-connection-information]
-recipe = slapos.cookbook:publish
-init-password = $${htpasswd:passwd}
-init-user = $${htpasswd-runner:user}
-publisher-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:publisher_location_prefix}
-subscriber-url = http://$${htpasswd-runner:user}:$${htpasswd:passwd}@[$${nginx-configuration:ip}]:$${nginx-configuration:port}$${nginx-configuration:subscriber_location_prefix}

software/nginx-push-stream/instance.cfg.in

 [buildout]
 parts =
-  switch-softwaretype
-
+  nginx-service
+  promises
+  publish-connection-information
+extends = ${monitor-template:rendered}
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[switch-softwaretype]
-recipe = slapos.cookbook:softwaretype
-default = $${:nginx}
-nginx = $${dynamic-template-nginx:rendered}
-
-[dynamic-template-nginx]
-recipe = slapos.recipe.template:jinja2
-template = ${template-nginx:output}
-rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
-filename = instance-nginx.cfg
-
-[slap-connection]
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
-
-[instance-parameter]
-# Fetches parameters defined in SlapOS Master for this instance.
-# Always the same.
-recipe = slapos.cookbook:slapconfiguration.serialised
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+run = $${:var}/run
+log = $${:var}/log
+varnginx = $${:var}/nginx
+services = $${:etc}/service
+cron-entries = $${:etc}/cron.d
+www = $${:srv}/www
+ssl = $${:etc}/ssl
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+#################################
+# Nginx service
+#################################
+[nginx-service]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/nginx
+command-line =
+  ${nginx-push-stream-output:nginx} -c $${nginx-configuration:output}
+
+[nginx-configuration]
+recipe = slapos.recipe.template
+url = ${template-nginx-configuration:output}
+output = $${directory:etc}/nginx.cfg
+mode = 0600
+access-log = $${directory:log}/nginx-access.log
+error-log = $${directory:log}/nginx-error.log
+ip = $${slap-configuration:ipv6-random}
+local-ip = $${slap-configuration:ipv4-random}
+port = 9443
+base-url = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}
+
+# Generate a self-signed TLS certificate.
+[nginx-certificate]
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${:key-file} ]
+  then
+    ${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${nginx-configuration:ip}" \
+      -newkey rsa:1024 -keyout $${:key-file} \
+      -out $${:cert-file}
+  fi
+update-command = $${:command}
+key-file = $${directory:ssl}/${:_buildout_section_name_}.key
+cert-file = $${directory:ssl}/${:_buildout_section_name_}.cert
+common-name = $${nginx-configuration:ip}
+stop-on-error = true
+
+[promises]
+recipe =
+promises =
+  $${nginx-available-promise:recipe}
+
+[nginx-available-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url = $${nginx-configuration:base-url}/status
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+# publisher-url and subscriber-url are URITemplates, with an id
+# parameter which is the ID of the channel.
+publisher-url = $${nginx-configuration:base-url}/pub{?id}
+subscriber-url = $${nginx-configuration:base-url}/sub{/id}

software/nginx-push-stream/software.cfg

 [buildout]
 extends =
   ../../stack/slapos.cfg
-  ../../component/dash/buildout.cfg
   ../../component/nginx/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ./buildout.hash.cfg
 
 parts =
   slapos-cookbook
@@ -10,36 +11,18 @@ parts =
   nginx-push-stream-module
   nginx-push-stream
   template
-  template-nginx-service
-  template-nginx
+
+[python]
+part = python3
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = eb4c69df9a8dbb94fb76d0a6c11e360f
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
-[template-nginx-service]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/template-nginx-service.sh.in
-md5sum = 90ba5a247c489261d3304528cba56e06
-output = ${buildout:directory}/template-nginx-service.sh.in
-mode = 0644
-
 [template-nginx-configuration]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/template-nginx.cfg.in
-md5sum = f5658154b82282bc1871f18ddf4529d8
+url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template-nginx.cfg.in
 mode = 0644
-
-[template-nginx]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-nginx.cfg.in
-md5sum = 936fea88f5548c4f14e287f1b27dc127
-output = ${buildout:directory}/instance-nginx.cfg.in
-mode = 0644
-
-[versions]
-inotifyx = 0.2.2

software/nginx-push-stream/template-nginx-service.sh.in

-#!${dash-output:dash}
-# BEWARE: This file is operated by slapos node
-# BEWARE: It will be overwritten automatically
-
-exec ${nginx-push-stream-output:nginx} \
-  -c $${nginx-configuration:output}

software/nginx-push-stream/template-nginx.cfg.in

@@ -8,7 +8,7 @@ events {
   # multi_accept on;
 }
 
-error_log $${nginx-configuration:error_log};
+error_log $${nginx-configuration:error-log};
 
 http {
 
@@ -33,8 +33,8 @@ http {
   # Logging Settings
   ##
 
-  access_log $${nginx-configuration:access_log};
-  error_log $${nginx-configuration:error_log};
+  access_log $${nginx-configuration:access-log};
+  error_log $${nginx-configuration:error-log};
 
   ##
   # Gzip Settings
@@ -54,9 +54,21 @@ http {
   ##
   push_stream_shared_memory_size                32m;
 
+
 server {
-  listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
-  listen $${nginx-configuration:local_ip}:$${nginx-configuration:port};
+  listen [$${nginx-configuration:ip}]:$${nginx-configuration:port} ssl http2;
+  listen $${nginx-configuration:local-ip}:$${nginx-configuration:port} ssl http2;
+
+  # generated 2021-08-02, Mozilla Guideline v5.6, nginx 1.19.2, OpenSSL 1.1.1k, modern configuration, no HSTS, no OCSP
+  # https://ssl-config.mozilla.org/#server=nginx&version=1.19.2&config=modern&openssl=1.1.1k&hsts=false&ocsp=false&guideline=5.6
+  ssl_certificate $${nginx-certificate:cert-file};
+  ssl_certificate_key $${nginx-certificate:key-file};
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:MozSSL:10m;
+  ssl_session_tickets off;
+  ssl_protocols TLSv1.3;
+  ssl_prefer_server_ciphers off;
+
 
   fastcgi_temp_path  $${directory:varnginx} 1 2;
   uwsgi_temp_path  $${directory:varnginx} 1 2;
@@ -65,42 +77,42 @@ server {
   client_body_temp_path $${directory:varnginx} 1 2;
   proxy_temp_path $${directory:varnginx} 1 2;
 
-  auth_basic           "Nginx Access";
-  auth_basic_user_file $${htpasswd-runner:htpasswd-path};
-
   ## Serve an error 204 (No Content) for favicon.ico
   location = /favicon.ico {
     return 204;
   }
 
-  location $${nginx-configuration:publisher_location_prefix} {
-            push_stream_publisher;
+  location = /status {
+    default_type "text/plain";
+    return 200 'OK';
+  }
 
-            push_stream_channels_path               $arg_id;
-            # store messages in memory
-            push_stream_store_messages              $${nginx-configuration:publisher_push_stream_store_messages};
+  location /pub {
+    push_stream_publisher;
 
-            # Message size limit
-            # client_max_body_size MUST be equal to client_body_buffer_size or
-            # you will be sorry.
-            client_max_body_size                    $${nginx-configuration:publisher_client_max_body_size};
-            client_body_buffer_size                 $${nginx-configuration:publisher_client_body_buffer_size};
+    push_stream_channels_path               $arg_id;
+    # store messages in memory
+    push_stream_store_messages              off;
 
-   }
+    # Message size limit
+    client_max_body_size                    16k;
+    client_body_buffer_size                 16k;
+    
+  }
 
-   location ~ $${nginx-configuration:subscriber_location_prefix}/(.*) {
-            # activate subscriber mode for this location
-            add_header "Access-Control-Allow-Origin" $${nginx-configuration:subscriber_allow_origin};
-            add_header 'Access-Control-Allow-Credentials' $${nginx-configuration:subscriber_allow_credential};
-            add_header 'Access-Control-Allow-Methods' $${nginx-configuration:subscriber_allow_methods};
-            add_header 'Access-Control-Allow-Headers' $${nginx-configuration:subscriber_allow_headers};
+  location ~ /sub/(.*) {
+    # activate subscriber mode for this location
+    add_header "Access-Control-Allow-Origin" '*';
+    add_header 'Access-Control-Allow-Credentials' 'false';
+    add_header 'Access-Control-Allow-Methods' 'GET, HEAD, OPTIONS';
+    add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,Keep-Alive,X-Requested-With,If-Modified-Since';
 
-            push_stream_subscriber eventsource;
-            # positional channel path
-            push_stream_channels_path                   $1;
+    push_stream_subscriber eventsource;
+    # positional channel path
+    push_stream_channels_path                   $1;
 
-            # content-type
-            default_type                                "text/event-stream; charset=utf-8";
+    # content-type
+    default_type                                "text/event-stream; charset=utf-8";
   }
 
   location / {

software/nginx-push-stream/test/README.md

+Tests for Nginx Push Stream software release

software/nginx-push-stream/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.nginx_push_stream'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Nginx Push Stream",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'requests',
+        'uritemplate',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/nginx-push-stream/test/test.py

+##############################################################################
+# coding: utf-8
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import multiprocessing
+
+import uritemplate
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestNginxPushStream(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = \
+        self.computer_partition.getConnectionParameterDict()
+
+  def test_push_stream_scenario(self):
+    def process_messages(q):
+      # type:(multiprocessing.Queue[bytes]) -> None
+      req = requests.get(
+          uritemplate.URITemplate(
+              self.connection_parameters['subscriber-url']).expand(
+                  id='channel_id'),
+          verify=False,
+          stream=True,
+      )
+      if not req.ok:
+        q.put(('error: wrong status code %s' % req.status_code).encode())
+      q.put(b'ready')
+      for _, line in zip(range(2), req.iter_lines()):
+        q.put(line)
+
+    q = multiprocessing.Queue()  # type: multiprocessing.Queue[bytes]
+    subscriber = multiprocessing.Process(target=process_messages, args=(q, ))
+    subscriber.start()
+    self.assertEqual(q.get(timeout=30), b'ready')
+
+    resp = requests.post(
+        uritemplate.URITemplate(
+            self.connection_parameters['publisher-url']).expand(
+                id='channel_id'),
+        verify=False,
+        data='Hello',
+        timeout=2,
+    )
+    resp.raise_for_status()
+
+    try:
+      subscriber.join(timeout=30)
+    except multiprocessing.TimeoutError:
+      subscriber.terminate()
+      subscriber.join(timeout=30)
+      self.fail('Process did not terminate')
+
+    self.assertEqual(q.get_nowait(), b': ')
+    self.assertEqual(q.get_nowait(), b'data: Hello')

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 733643122fb75dfd7374b973a95fa2ea
+md5sum = 2c8b202b0c366e27dd6e6f456fb18256

software/slapos-sr-testing/instance.cfg

@@ -44,7 +44,7 @@ tests = {{ tests | indent(2) }}
 template =
   inline:{% raw %}
   {%- set only_sr = slapparameter_dict.get('only-sr') %}
-  {%- if not isinstance(only_sr, list) %}
+  {%- if only_sr and not isinstance(only_sr, list) %}
   {%-   set only_sr = [only_sr] %}
   {%- endif %}
   {%- set unittest_args = slapparameter_dict.get('unittest-args', ['discover', '-v']) %}

software/slapos-sr-testing/software-py3.cfg

@@ -22,3 +22,4 @@ extra =
   proftpd ${slapos.test.proftpd-setup:setup}
   repman ${slapos.test.repman-setup:setup}
   restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+  headless-chromium ${slapos.test.headless-chromium-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -149,6 +149,11 @@ setup = ${slapos-repository:location}/software/jupyter/test/
 egg = slapos.test.nextcloud
 setup = ${slapos-repository:location}/software/nextcloud/test/
 
+[slapos.test.nginx-push-stream-setup]
+<= setup-develop-egg
+egg = slapos.test.nginx_push_stream
+setup = ${slapos-repository:location}/software/nginx-push-stream/test/
+
 [slapos.test.turnserver-setup]
 <= setup-develop-egg
 egg = slapos.test.turnserver
@@ -194,16 +199,26 @@ setup = ${slapos-repository:location}/software/jscrawler/test/
 egg = slapos.test.galene
 setup = ${slapos-repository:location}/software/galene/test/
 
-[slapos.core-repository]
-<= git-clone-repository
-repository = https://lab.nexedi.com/nexedi/slapos.core.git
-branch = master
+[slapos.test.headless-chromium-setup]
+<= setup-develop-egg
+egg = slapos.test.headless-chromium
+setup = ${slapos-repository:location}/software/headless-chromium/test/
 
 [slapos.test.caucase-setup]
 <= setup-develop-egg
 egg = slapos.test.caucase
 setup = ${slapos-repository:location}/software/caucase/test/
 
+[slapos.test.erp5testnode-setup]
+<= setup-develop-egg
+egg = slapos.test.erp5testnode
+setup = ${slapos-repository:location}/software/erp5testnode/test/
+
+[slapos.core-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.core.git
+branch = master
+
 [slapos.core-setup]
 <= setup-develop-egg
 egg = slapos.core
@@ -242,6 +257,7 @@ extra-eggs =
   ${slapos.test.slaprunner-setup:egg}
   ${slapos.test.jupyter-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
+  ${slapos.test.nginx-push-stream-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.theia-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
@@ -254,6 +270,8 @@ extra-eggs =
   ${slapos.test.html5as-setup:egg}
   ${slapos.test.html5as-base-setup:egg}
   ${slapos.test.fluentd-setup:egg}
+  ${slapos.test.headless-chromium-setup:egg}
+  ${slapos.test.erp5testnode-setup:egg}
 
 # We don't name this interpreter `python`, so that when we run slapos node
 # software, installation scripts running `python` use a python without any
@@ -312,6 +330,7 @@ tests =
   slaprunner ${slapos.test.slaprunner-setup:setup}
   theia ${slapos.test.theia-setup:setup}
   metabase ${slapos.test.metabase-setup:setup}
+  nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
 ###
   ${:extra}
 
@@ -337,6 +356,7 @@ extra =
   jscrawler ${slapos.test.jscrawler-setup:setup}
   html5as ${slapos.test.html5as-setup:setup}
   html5as-base ${slapos.test.html5as-base-setup:setup}
+  erp5testnode ${slapos.test.erp5testnode-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

software/theia/buildout.hash.cfg

@@ -15,15 +15,39 @@
 
 [instance-theia]
 _update_hash_filename_ = instance-theia.cfg.jinja.in
-md5sum = 11d347dd2bf762902341746a388673a0
+md5sum = 562acf69f344fa9f6d03992d696462d0
 
 [instance]
 _update_hash_filename_ = instance.cfg.in
-md5sum = 063d3e19da9d3d4bfb77e8e638aa3a77
+md5sum = a7d78b4002266c69ece05a476df82791
+
+[instance-import]
+_update_hash_filename_ = instance-import.cfg.jinja.in
+md5sum = 861ef130f27175c2978a9b946b138dd5
+
+[instance-export]
+_update_hash_filename_ = instance-export.cfg.jinja.in
+md5sum = b3cedaa1603ca8ed83fdd94ef4b35cc8
+
+[instance-resilient]
+_update_hash_filename_ = instance-resilient.cfg.jinja
+md5sum = d78a9f885bdebf6720197209e0c21aa0
+
+[theia-common]
+_update_hash_filename_ = theia_common.py
+md5sum = e57396473b4b6a17d26a747f0030293c
+
+[theia-export]
+_update_hash_filename_ = theia_export.py
+md5sum = b5f5ac1924b27d3f2be2e5ea291c119e
+
+[theia-import]
+_update_hash_filename_ = theia_import.py
+md5sum = 9e8c17a4b2d802695caf0c2c052f0d11
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock
-md5sum = 80e7ad91deea54cebcccef5a83fdb380
+md5sum = 18e8302b2acff3721cad23d829e3df55
 
 [python-language-server-requirements.txt]
 _update_hash_filename_ = python-language-server-requirements.txt

software/theia/download-plugins.cfg

@@ -13,6 +13,7 @@ urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.5
 	vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433
 	vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29
 	vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e
+	vscode-git https://open-vsx.org/api/vscode/git/1.54.3/file/vscode.git-1.54.3.vsix da2f2c0626b4a254a660ae20b0c97611
 	vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256
 	vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310
 	vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02

software/theia/generate_download_plugins_cfg.py

@@ -17,6 +17,7 @@ for plugin_and_version in '''\
     vscode/docker/latest
     vscode/emmet/latest
     vscode/fsharp/latest
+    vscode/git/latest
     vscode/go/latest
     vscode/groovy/latest
     vscode/grunt/latest

software/theia/instance-export.cfg.jinja.in

+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
+
+[buildout]
+extends = {{ theia_instance_cfg }}
+          {{ pbsready_export_cfg }}
+
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+  resilient-publish-connection-parameter
+
+
+# The resilient stack makes the 'resilient' instance
+# request the 'export' instance with a 'namebase' parameter.
+# The export template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = {{ parameter_dict['namebase'] }}
+
+
+# The resilient export stack periodically calls exporter:wrapper
+# and then notifies the pull-backup instance that data is ready
+# to be pulled from the export instance.
+# All it expects is that a script be available in exporter:wrapper.
+[exporter]
+wrapper = $${theia-export-script:rendered}
+
+[theia-export-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/theia-export-script
+mode = 0700
+exitcode-file = $${directory:srv}/export-exitcode-file
+error-file = $${directory:srv}/export-errormessage-file
+context =
+  raw python ${software-info:python-with-eggs}
+  raw theia_export ${software-info:theia-export}
+  raw bash ${software-info:bash}
+  raw rsync ${software-info:rsync}
+  raw sqlite3 ${software-info:sqlite3}
+  raw root_path $${buildout:directory}
+  raw backup_path $${directory:backup}
+  raw slapos_cfg $${directory:runner}/etc/slapos.cfg
+  raw project_path $${directory:project}
+  raw public_path $${directory:frontend-static-public}
+  key exitfile :exitcode-file
+  key errorfile :error-file
+{%- raw %}
+template =
+  inline:#!{{ bash }}
+  {{ python }} {{ theia_export }} \
+  --rsync {{ rsync }} \
+  --sqlite3 {{ sqlite3 }} \
+  --root {{ root_path }} \
+  --backup {{ backup_path }} \
+  --cfg {{ slapos_cfg }} \
+  --dirs {{ project_path }} \
+  --dirs {{ public_path }} \
+  --exitfile {{ exitfile }} \
+  --errorfile {{ errorfile }}
+{%- endraw %}
+
+
+# Add a promise to check that the export script has run
+# successfully and recently (at most 2 days ago).
+[promises]
+export-promises =
+  $${export-promise:name}
+
+[export-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = resiliency-export-promise.py
+config-command = $${export-promise-script:rendered}
+
+[initial-export-exitcode-file]
+recipe = slapos.recipe.template:jinja2
+rendered = $${theia-export-script:exitcode-file}
+template = inline:0
+once = $${:rendered}
+
+[export-promise-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/export-promise-script
+exitcode-file = $${initial-export-exitcode-file:rendered}
+context =
+  key exitcodefile :exitcode-file
+  key errorfile theia-export-script:error-file
+{%- raw %}
+template =
+  inline:#!/bin/sh
+  if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
+  then
+    echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 1
+  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  then
+    echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
+    cat {{ repr(errorfile) }}
+    exit 1
+  else
+    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  fi
+{%- endraw %}
+
+
+# Extend resilient parameters with normal theia connection parameters
+[resilient-publish-connection-parameter]
+<= publish-connection-parameter

software/theia/instance-import.cfg.jinja.in

+{%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) -%}
+{%- set additional_frontend = parameter_dict['additional-frontend-guid'] -%}
+
+[buildout]
+extends = {{ theia_instance_cfg }}
+          {{ pbsready_import_cfg }}
+
+parts +=
+  monitor-base
+  $${:theia-parts}
+  $${:theia-environment-parts}
+
+
+# The resilient stack makes the 'resilient' instance
+# request the 'import' instance with a 'namebase' parameter.
+# The import template then expects to receive it in
+# slap-parameter:namebase
+[slap-parameter]
+namebase = {{ parameter_dict['namebase'] }}
+
+
+# Change frontend name to avoid conflicts
+[remote-frontend]
+name = Import {{ parameter_dict['frontend-name'] }}
+
+{% if additional_frontend -%}
+[remote-additional-frontend]
+name = Import {{ parameter_dict['additional-frontend-name'] }}
+{%- endif %}
+
+
+# Change port ranges to avoid race conditions on port allocation
+[frontend-instance-port]
+minimum = 3200
+maximum = 3300
+
+[theia-service-port]
+minimum = 3700
+maximum = 3800
+
+[slapos-standalone-port]
+minimum = 4200
+maximum = 4300
+
+
+# Always disable autoprocessing in the import instance
+[slapos-autorun]
+autorun = stopped
+
+
+# Change the gravatar favicon seed
+[favicon.ico]
+seed = Import {{ root_title }}
+
+
+# The resilient stack calls post-notification-run:output followed by
+# importer:wrapper when the instance is notified that the backup files
+# have just been pushed to it. All it expects is the path of a script
+# in post-notification-run:output and in importer:wrapper.
+
+[post-notification-run]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/post-notification-run-script
+output = $${:rendered}
+mode = 0700
+template =
+  inline:#!${software-info:bash}
+  # Do nothing because the backup signature will
+  # be verified by the import script itself
+  exit 0
+
+[importer]
+wrapper = $${theia-import-script:rendered}
+
+[theia-import-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/theia-import-script
+mode = 0700
+exitcode-file = $${directory:srv}/import-exitcode-file
+error-file = $${directory:srv}/import-errormessage-file
+context =
+  raw python ${software-info:python-with-eggs}
+  raw theia_import ${software-info:theia-import}
+  raw bash ${software-info:bash}
+  raw rsync ${software-info:rsync}
+  raw sqlite3 ${software-info:sqlite3}
+  raw slapos ${software-info:slapos}
+  raw slapos_node_software_log $${directory:runner}/var/log/slapos-node-software.log
+  raw slapos_node_instance_log $${directory:runner}/var/log/slapos-node-instance.log
+  raw supervisorctl ${software-info:supervisorctl}
+  raw supervisord_conf $${directory:runner}/etc/supervisord.conf
+  raw root_path $${buildout:directory}
+  raw backup_path $${directory:backup}
+  raw slapos_cfg $${directory:runner}/etc/slapos.cfg
+  raw project_path $${directory:project}
+  raw public_path $${directory:frontend-static-public}
+  key exitfile :exitcode-file
+  key errorfile :error-file
+{%- raw %}
+template =
+  inline:#!{{ bash }}
+  . $${common-environment:rendered}
+  . $${slapos-standalone-activate:rendered}
+  {{ python }} {{ theia_import }} \
+  --rsync {{ rsync }} \
+  --sqlite3 {{ sqlite3 }} \
+  --slapos {{ slapos }} \
+  --srlog {{ slapos_node_software_log }} \
+  --cplog {{ slapos_node_instance_log }} \
+  --supervisorctl {{ supervisorctl }} \
+  --supervisordconf {{ supervisord_conf }} \
+  --root {{ root_path }} \
+  --backup {{ backup_path }} \
+  --cfg {{ slapos_cfg }} \
+  --dirs {{ project_path }} \
+  --dirs {{ public_path }} \
+  --exitfile {{ exitfile }} \
+  --errorfile {{ errorfile }}
+{%- endraw %}
+
+
+# Add a promise to check that the import script has run
+# successfully and recently (at most 2 days ago).
+[promises]
+import-promises =
+  $${import-promise:name}
+
+[import-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = resiliency-import-promise.py
+config-command = $${import-promise-script:rendered}
+
+[initial-import-exitcode-file]
+recipe = slapos.recipe.template:jinja2
+rendered = $${theia-import-script:exitcode-file}
+template = inline:0
+once = $${:rendered}
+
+[import-promise-script]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/import-promise-script
+exitcode-file = $${initial-import-exitcode-file:rendered}
+context =
+  key exitcodefile :exitcode-file
+  key errorfile theia-import-script:error-file
+{%- raw %}
+template =
+  inline:#!/bin/sh
+  if [ -z $(find {{ repr(exitcodefile) }} -mtime -2) ]
+  then
+    echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 1
+  elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
+  then
+    echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
+    cat {{ repr(errorfile) }}
+    exit 1
+  else
+    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
+    exit 0
+  fi
+{%- endraw %}
+
+
+# Resilient connection parameters of import instance are published
+# through the resilient stack.
+# Extend resilient parameters with normal theia connection parameters
+[resilient-publish-connection-parameter]
+<= publish-connection-parameter

software/theia/instance-resilient-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "type": "object",
+  "description": "Parameters to instantiate resilient Theia",
+  "allOf": [
+    {
+      "$ref": "instance-input-schema.json#/"
+    },
+    {
+      "properties": {
+        "resilient-clone-number": {
+          "title": "Amount of backup(s) to create",
+          "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
+          "type": "integer",
+          "default": 1,
+          "minimum": 0,
+          "maximum": 2,
+          "optional": true
+        },
+        "-sla-theia0-computer_guid": {
+          "title": "Target computer for main instance",
+          "description": "Target computer GUID for main instance.",
+          "type": "string",
+          "optional": true
+        },
+        "-sla-theia1-computer_guid": {
+          "title": "Target computer for first clone",
+          "description": "Target computer for first clone instance.",
+          "type": "string",
+          "optional": true
+        },
+        "-sla-pbs1-computer_guid": {
+          "title": "Target computer for first PBS",
+          "description": "Target computer for first PBS instance.",
+          "type": "string",
+          "optional": true
+        },
+        "-sla-theia2-computer_guid": {
+          "title": "Target computer for second clone",
+          "description": "Target computer for second clone instance.",
+          "type": "string",
+          "optional": true
+        },
+        "-sla-pbs2-computer_guid": {
+          "title": "Target computer for second PBS",
+          "description": "Target computer for second PBS instance.",
+          "type": "string",
+          "optional": true
+        },
+        "resiliency-backup-periodicity": {
+          "title": "Periodicity of backup",
+          "description": "Periodicity of backup, in cron format.",
+          "type": "string",
+          "optional": true
+        },
+        "remove-backup-older-than": {
+          "title": "Remove backups older than...",
+          "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+          "type": "string",
+          "default": "2W",
+          "optional": true
+        },
+        "ignore-known-hosts-file": {
+          "title": "Ignore known_hosts file",
+          "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, theia).",
+          "type": "boolean",
+          "default": false,
+          "optional": true
+        }
+      }
+    }
+  ]
+}

software/theia/instance-resilient.cfg.jinja

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+{% set number_of_instances = slapparameter_dict.get('resilient-clone-number', 1)|int %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+extends =
+  {{ monitor_template }}
+
+parts +=
+# Generate the parts to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-parts.cfg.in and stack/resilient/template-replicated.cfg.in
+# See below for the generation of the sections corresponding to the parts generated here
+  {{ parts.replicate("theia", number_of_instances + 1) }}
+# Also publish some connection parameters
+  publish-connection-parameter
+
+[ArgLeader]
+
+[ArgBackup]
+
+# Generate sections to request theia-export, pull-backup and theia-import
+# See stack/resilient/template-replicated.cfg.in
+# In particular:
+#
+# [request-theia]
+# <= ArgLeader
+# software-type = export
+# ...
+#
+# [request-theia-pseudo-replicating-1]
+# <= ArgBackup
+# software-type = import
+# ...
+#
+# [request-pbs-theia-1]
+# software-type = pull-backup
+# ...
+#
+{{ replicated.replicate("theia", number_of_instances + 1,
+                        "export", "import",
+                        "ArgLeader", "ArgBackup",
+                        slapparameter_dict=slapparameter_dict) }}
+
+# Extend the list of return parameters for the export request
+# The monitor parameters are only there to assert they are
+# actually published by the export instance
+[request-theia]
+return += url username password backend-url monitor-base-url monitor-setup-url
+
+# Extend the list of return parameters for the import request
+# with the monitor parameters to assert they are actually published
+[request-theia-pseudo-replicating-1]
+return += monitor-base-url monitor-setup-url
+
+# Publish some parameters from the export instance
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = ${request-theia:connection-url}
+username = ${request-theia:connection-username}
+password = ${request-theia:connection-password}
+backend-url = ${request-theia:connection-backend-url}
+
+# Publish resiliency parameters fetched by the resilient stack
+[publish-connection-parameter]
+<= publish-connection-information

software/theia/instance-theia.cfg.jinja.in

@@ -41,23 +41,26 @@ backend-url = $${frontend-instance:url}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-var = $${buildout:directory}/var
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
-tmp = $${buildout:directory}/tmp
-dot-theia = $${buildout:directory}/.theia/
+home = $${buildout:directory}
+etc = $${:home}/etc
+var = $${:home}/var
+srv = $${:home}/srv
+bin = $${:home}/bin
+tmp = $${:home}/tmp
+dot-theia = $${:home}/.theia/
 pidfiles = $${:var}/run
 
 services = $${:etc}/service
 runner = $${:srv}/runner
+backup = $${:srv}/backup/theia
 project = $${:srv}/project
+
 frontend-static = $${:srv}/frontend-static
 frontend-static-public = $${:frontend-static}/public
 frontend-static-css = $${:frontend-static}/css
 
-bash-completions = $${buildout:directory}/.local/share/bash-completion/completions/
-fish-completions = $${buildout:directory}/.config/fish/completions/
+bash-completions = $${:home}/.local/share/bash-completion/completions/
+fish-completions = $${:home}/.config/fish/completions/
 
 
 # Promises
@@ -68,11 +71,13 @@ recipe =
 instance-promises =
   $${theia-listen-promise:name}
   $${frontend-listen-promise:name}
+  $${frontend-authentification-promise:name}
   $${remote-frontend-url-available-promise:name}
   {% if additional_frontend %}
   $${remote-additional-frontend-url-available-promise:name}
   {% endif %}
   $${slapos-standalone-listen-promise:name}
+  $${slapos-standalone-ready-promise:name}
   $${slapos-autorun-promise:name}
 
 [theia-listen-promise]
@@ -89,6 +94,16 @@ name = $${:_buildout_section_name_}.py
 config-host = $${frontend-instance:ip}
 config-port = $${frontend-instance:port}
 
+[frontend-authentification-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+username = $${frontend-instance-password:username}
+password = $${frontend-instance-password:passwd}
+ip = $${frontend-instance:ip}
+port = $${frontend-instance:port}
+config-url = https://$${:username}:$${:password}@[$${:ip}]:$${:port}
+
 [remote-frontend-url-available-promise]
 <= monitor-promise-base
 module = check_url_available
@@ -113,13 +128,19 @@ name = standalone-listen-promise.py
 config-host = $${slapos-standalone-instance:hostname}
 config-port = $${slapos-standalone-instance:port}
 
+[slapos-standalone-ready-promise]
+<= monitor-promise-base
+module = check_socket_listening
+name = standalone-ready-promise.py
+config-abstract = $${directory:runner}/standalone_ready
+
 [slapos-autorun-promise]
 <= monitor-promise-base
 module = check_service_state
-# XXX promise plugins can not contain "slapos" in their names
 name = autorun-state-promise.py
 config-service = $${slapos-autorun:service-name}
 config-expect = $${slapos-autorun:autorun}
+config-run-directory = $${directory:runner}/var/run
 
 
 # Remote Caddy Frontend
@@ -279,13 +300,14 @@ wait-for-files = $${frontend-instance:pidfile}
 [favicon.ico]
 # generate a pseudo random favicon, different for each instance name.
 recipe = slapos.recipe.build
+seed = {{ root_title }}
 install =
   import hashlib, shutil
   buildout_offline = self.buildout['buildout']['offline']
   self.buildout['buildout']['offline'] = 'false'
   try:
     gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
-      b'''{{ root_title }}'''
+      b'''$${:seed}'''
     ).hexdigest() + "?s=256&d=retro"
     shutil.copy(self.download(gravatar_url), '''$${:location}''')
   except Exception:
@@ -310,7 +332,7 @@ mode = 0700
 template =
   inline:
   #!/bin/sh
-  export HOME=$${buildout:directory}
+  export HOME=$${directory:home}
   export PATH=${python-language-server:location}/bin:${java-jdk:location}/bin:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
 
 
@@ -375,9 +397,6 @@ template =
     . {{ activate }}
     unset GIT_EXEC_PATH
     set -- --rcfile {{ bashrc }}
-  # otherwise, assume this shell is running task and add an artificial delay to workaround https://github.com/eclipse-theia/theia/issues/2961
-  else
-    sleep 1
   fi
   exec "$SHELL" "$@"
   {% endraw %}
@@ -419,6 +438,7 @@ ip = {{ ipv4_random }}
 ipv4 = {{ ipv4_random }}
 ipv6 = {{ ipv6_random }}
 port = $${slapos-standalone-port:port}
+local-software-release-root = $${directory:home}
 slapos-configuration = $${directory:runner}/etc/slapos.cfg
 computer-id = slaprunner
 
@@ -447,6 +467,7 @@ template =
       $${slapos-standalone-config:ipv4} \
       $${slapos-standalone-config:ipv6} \
       $${slapos-standalone-config:port} \
+      $${slapos-standalone-config:local-software-release-root} \
       $${slapos-standalone-config:computer-id} \
       {%- if parameter_dict.get('embedded-sr') %}
       --sr='{{ parameter_dict['embedded-sr'] }}' \

software/theia/instance.cfg.in

@@ -2,6 +2,9 @@
 parts =
   switch-softwaretype
 
+extends =
+  ${template-resilient-templates:output}
+
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
@@ -18,6 +21,11 @@ recipe = slapos.cookbook:switch-softwaretype
 RootSoftwareInstance = $${:default}
 default = $${:theia}
 theia = theia:rendered
+export = export:rendered
+import = import:rendered
+resilient = resilient:rendered
+frozen = instance-frozen:rendered
+pull-backup = template-pull-backup:rendered
 
 [theia]
 recipe = slapos.recipe.template:jinja2
@@ -46,3 +54,43 @@ default-parameters =
     "additional-frontend-guid": null
   }
 frontend-sr = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+
+[import]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-import:output}
+rendered = $${buildout:directory}/instance-import.cfg
+mode = 0644
+context =
+  jsonkey default_parameter_dict theia:default-parameters
+  key parameter_dict slap-configuration:configuration
+  key theia_instance_cfg theia:rendered
+  key pbsready_import_cfg template-pbsready-import:rendered
+  key root_title slap-configuration:root-instance-title
+
+[export]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-export:output}
+rendered = $${buildout:directory}/instance-export.cfg
+mode = 0644
+context =
+  jsonkey default_parameter_dict theia:default-parameters
+  key parameter_dict slap-configuration:configuration
+  key theia_instance_cfg theia:rendered
+  key pbsready_export_cfg template-pbsready-export:rendered
+
+[resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-resilient:output}
+rendered = $${buildout:directory}/instance-resilient.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  key buildout buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key slapparameter_dict slap-configuration:configuration
+  raw monitor_template ${monitor-template:rendered}
+template-parts-destination = ${template-parts:target}
+template-replicated-destination = ${template-replicated:target}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination

software/theia/software.cfg

@@ -12,9 +12,12 @@ extends =
    ../../component/coreutils/buildout.cfg
    ../../component/java-jdk/buildout.cfg
    ../../component/fonts/buildout.cfg
+   ../../component/libsecret/buildout.cfg
+   ../../component/pkgconfig/buildout.cfg
    ../../stack/nodejs.cfg
    ../../stack/slapos.cfg
    ../../stack/monitor/buildout.cfg
+   ../../stack/resilient/buildout.cfg
    ../../component/defaults.cfg
    ./download-plugins.cfg
    ./buildout.hash.cfg
@@ -22,8 +25,14 @@ extends =
 parts =
   theia-wrapper
   slapos-cookbook
+  python-with-eggs
   instance-theia
   instance
+  instance-import
+  instance-export
+  instance-resilient
+  theia-common
+  theia-export
 
 # default for slapos-standalone
 shared-part-list =
@@ -54,9 +63,10 @@ initialization =
   import argparse
   import glob
   import json
-  import os.path
-  import sys
+  import os
   import signal
+  import socket
+  import sys
   import time
 
   import slapos.slap.standalone
@@ -66,6 +76,7 @@ initialization =
   parser.add_argument('ipv4')
   parser.add_argument('ipv6')
   parser.add_argument('server_port', type=int)
+  parser.add_argument('local_software_release_root')
   parser.add_argument('computer_id')
   parser.add_argument('--sr')
   parser.add_argument('--srtype')
@@ -101,58 +112,64 @@ initialization =
       instance_root="%s/instance" % args.base_directory,
       partition_forward_configuration=partition_forward_configuration,
       slapos_bin="${buildout:bin-directory}/slapos",
+      local_software_release_root=args.local_software_release_root,
   )
-  standalone.start()
-  partition_count = 20
-  if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
-    print("Standalone SlapOS: Formatting {partition_count} partitions".format(
-        partition_count=partition_count))
-    standalone.format(
-        partition_count,
-        args.ipv4,
-        args.ipv6
-    )
-  print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
-  # Run instance at least once, to start the supervisor managing instances.
-  try:
-    standalone.waitForInstance(max_retry=0)
-  except slapos.slap.standalone.SlapOSNodeCommandError as e:
-    print("Error instanciating: {}".format(e))
 
-  if args.sr:
-    try:
-      with open(args.srparams) as f:
-        params = json.load(f)
-    except Exception:
-      params = None
-    if not isinstance(params, dict):
-      params = None
-    print("Supplying and Requesting Embedded Software {sr} with type {srtype}".format(
-        sr=args.sr, srtype=args.srtype))
-    print("With parameters {param_dict} parsed from '{srparams}'".format(
-        param_dict=params, srparams=args.srparams))
-    standalone.supply(args.sr)
-    standalone.request(
-        args.sr,
-        "Embedded Instance",
-        args.srtype,
-        partition_parameter_kw=params,
-    )
-
-  quit_requested = []
   def signal_handler(signum, frame):
     print("Signal {signum} received".format(signum=signum))
-    quit_requested.append(True)
+    sys.exit()
   signal.signal(signal.SIGTERM, signal_handler)
 
-  print("Standalone SlapOS ready")
-  while not quit_requested:
-    time.sleep(.1)
+  standalone.start()
+  try:
+    partition_count = 20
+    if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
+      print("Standalone SlapOS: Formatting {partition_count} partitions".format(
+          partition_count=partition_count))
+      standalone.format(
+          partition_count,
+          args.ipv4,
+          args.ipv6,
+      )
+    print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
+    # Run instance at least once, to start the supervisor managing instances.
+    try:
+      standalone.waitForInstance(max_retry=0)
+    except slapos.slap.standalone.SlapOSNodeCommandError as e:
+      print("Error instanciating: {}".format(e))
+
+    if args.sr:
+      try:
+        with open(args.srparams) as f:
+          params = json.load(f)
+      except Exception:
+        params = None
+      if not isinstance(params, dict):
+        params = None
+      print("Supplying and Requesting Embedded Software {sr} with type {srtype}".format(
+          sr=args.sr, srtype=args.srtype))
+      print("With parameters {param_dict} parsed from '{srparams}'".format(
+          param_dict=params, srparams=args.srparams))
+      standalone.supply(args.sr)
+      standalone.request(
+          args.sr,
+          "Embedded Instance",
+          args.srtype,
+          partition_parameter_kw=params,
+      )
+
+    s = socket.socket(socket.AF_UNIX)
+    s.bind('\0' + os.path.join(args.base_directory, 'standalone_ready'))
+    s.listen(5)
+    print("Standalone SlapOS ready")
+    while True:
+      s.accept()[0].close()
+
+  finally:
+    print("Stopping standalone subsystem")
+    standalone.stop()
+    print("Exiting")
 
-  print("Stopping standalone subsystem")
-  standalone.stop()
-  print("Exiting")
-  sys.exit(0)
 
 needs-these-eggs-scripts-in-path =
   ${supervisor:recipe}
@@ -191,7 +208,11 @@ stop-on-error = true
 [theia]
 recipe = plone.recipe.command
 command = ${bash:location}/bin/bash -c "
-  export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin:$PATH &&
+  export \
+    TMPDIR=${:location}/tmp \
+    PATH=${nodejs:location}/bin:${pkgconfig:location}/bin:$PATH \
+    PKG_CONFIG_PATH=${libsecret:pkg-config-path} \
+    LDFLAGS='-Wl,-rpath=${libsecret:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${glib:location}/lib' && \
   mkdir -p ${:location} && \
   mkdir -p \$TMPDIR && \
   cd ${:location} && \
@@ -233,6 +254,11 @@ template =
   inline:{
       "private": true,
       "theia": {
+          "backend": {
+              "config": {
+                  "warnOnPotentiallyInsecureHostPattern": false
+              }
+          },
           "frontend": {
               "config": {
                   "applicationName": "Theia SlapOS",
@@ -262,7 +288,8 @@ template =
                       "plantuml.render": "PlantUMLServer",
                       "gitlens.remotes": [{ "domain": "lab.nexedi.com", "type": "GitLab" }],
                       "java.home": "${java-jdk:location}"
-                  }
+                  },
+                  "warnOnPotentiallyInsecureHostPattern": false
               }
           },
           "generator": {
@@ -272,17 +299,16 @@ template =
           }
       },
       "dependencies": {
+          "@theia/bulk-edit": "latest",
           "@theia/callhierarchy": "latest",
           "@theia/console": "latest",
           "@theia/core": "latest",
-          "@theia/cpp-debug": "latest",
           "@theia/debug": "latest",
           "@theia/editor": "latest",
           "@theia/editor-preview": "latest",
           "@theia/file-search": "latest",
           "@theia/filesystem": "latest",
           "@theia/getting-started": "latest",
-          "@theia/git": "latest",
           "@theia/keymaps": "latest",
           "@theia/markers": "latest",
           "@theia/messages": "latest",
@@ -292,16 +318,19 @@ template =
           "@theia/navigator": "latest",
           "@theia/outline-view": "latest",
           "@theia/output": "latest",
-          "@theia/plugin": "latest",
+          "@theia/plugin-dev": "latest",
           "@theia/plugin-ext": "latest",
           "@theia/plugin-ext-vscode": "latest",
           "@theia/preferences": "latest",
           "@theia/preview": "latest",
           "@theia/process": "latest",
+          "@theia/property-view": "latest",
           "@theia/scm": "latest",
+          "@theia/scm-extra": "latest",
           "@theia/search-in-workspace": "latest",
           "@theia/task": "latest",
           "@theia/terminal": "latest",
+          "@theia/timeline": "latest",
           "@theia/typehierarchy": "latest",
           "@theia/userstorage": "latest",
           "@theia/variable-resolver": "latest",
@@ -343,6 +372,18 @@ template =
   #!/bin/sh
   exec ${nodejs:location}/bin/node ${theia:location}/node_modules/.bin/theia-open "$@"
 
+[python-with-eggs]
+recipe = zc.recipe.egg
+interpreter = ${:_buildout_section_name_}
+eggs =
+  ${slapos-toolbox:eggs}
+  six
+  zc.buildout
+# Only generate the interpreter script to avoid conflicts with scripts
+# for eggs that are also generated by another section, like slapos.toolbox
+scripts = ${:interpreter}
+
+
 [instance-theia]
 <= template-base
 output = ${buildout:directory}/instance-theia.cfg.jinja
@@ -350,3 +391,37 @@ output = ${buildout:directory}/instance-theia.cfg.jinja
 [instance]
 <= template-base
 output = ${buildout:directory}/instance.cfg
+
+[instance-import]
+<= template-base
+output = ${buildout:directory}/instance-import.cfg.jinja
+
+[instance-export]
+<= template-base
+output = ${buildout:directory}/instance-export.cfg.jinja
+
+[instance-resilient]
+<= download-base
+
+[theia-common]
+<= download-base
+destination = ${buildout:directory}/theia_common.py
+
+[theia-export]
+<= download-base
+destination = ${buildout:directory}/theia_export.py
+
+[theia-import]
+<= download-base
+destination = ${buildout:directory}/theia_import.py
+
+[software-info]
+slapos = ${buildout:bin-directory}/slapos
+python-with-eggs = ${buildout:bin-directory}/${python-with-eggs:interpreter}
+python = ${python:location}/bin/python
+rsync = ${rsync:location}/bin/rsync
+sqlite3 = ${sqlite3:location}/bin/sqlite3
+bash = ${bash:location}/bin/bash
+supervisorctl = ${buildout:bin-directory}/supervisorctl
+theia-export = ${theia-export:output}
+theia-import = ${theia-import:output}

software/theia/software.cfg.json

@@ -9,6 +9,13 @@
       "description": "Default",
       "request": "instance-input-schema.json",
       "response": "instance-output-schema.json",
+      "index": 0
+    },
+    "resilient": {
+      "title": "Resilient",
+      "description": "Resilient Theia",
+      "request": "instance-resilient-input-schema.json",
+      "response": "instance-output-schema.json",
       "index": 1
     }
   }

software/theia/test/project_tests.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import gzip
+import json
+import os
+import re
+import subprocess
+import time
+import unittest
+
+import requests
+
+from datetime import datetime, timedelta
+from six.moves.urllib.parse import urljoin
+
+from slapos.testing.testcase import installSoftwareUrlList
+
+import test_resiliency
+from test import SlapOSInstanceTestCase, theia_software_release_url
+
+
+erp5_software_release_url = os.path.abspath(
+  os.path.join(
+    os.path.dirname(__file__), '..', '..', 'erp5', 'software.cfg'))
+
+
+def setUpModule():
+  installSoftwareUrlList(
+    SlapOSInstanceTestCase,
+    [theia_software_release_url, erp5_software_release_url],
+    debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
+  )
+
+
+class ERP5Mixin(object):
+  _test_software_url = erp5_software_release_url
+  _connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}")
+
+  def _getERP5ConnexionParameters(self, software_type='export'):
+    slapos = self._getSlapos(software_type)
+    out = subprocess.check_output(
+      (slapos, 'request', 'test_instance', self._test_software_url),
+      stderr=subprocess.STDOUT,
+    )
+    print(out)
+    return json.loads(self._connexion_parameters_regex.search(out).group(1))
+
+  def _getERP5Url(self, connexion_parameters, path=''):
+    return urljoin(connexion_parameters['family-default-v6'], path)
+
+  def _getERP5User(self, connexion_parameters):
+    return connexion_parameters['inituser-login']
+
+  def _getERP5Password(self, connexion_parameters):
+    return connexion_parameters['inituser-password']
+
+  def _waitERP5connected(self, url, user, password):
+    for _ in range(5):
+      try:
+        resp = requests.get('%s/getId' % url, auth=(user, password), verify=False, allow_redirects=False)
+      except Exception:
+        time.sleep(20)
+        continue
+      if resp.status_code != 200:
+        time.sleep(20)
+        continue
+      break
+    else:
+      self.fail('Failed to connect to ERP5')
+    self.assertEqual(resp.text, 'erp5')
+
+  def _getERP5Partition(self, servicename):
+    p = subprocess.Popen(
+      (self._getSlapos(), 'node', 'status'),
+      stdout=subprocess.PIPE, universal_newlines=True)
+    out, _ = p.communicate()
+    found = set()
+    for line in out.splitlines():
+      if servicename in line:
+        found.add(line.split(':')[0])
+    if not found:
+      raise Exception("ERP5 %s partition not found" % servicename)
+    elif len(found) > 1:
+      raise Exception("Found several partitions for ERP5 %s" % servicename)
+    return found.pop()
+
+  def _getERP5PartitionPath(self, software_type, servicename, *paths):
+    partition = self._getERP5Partition(servicename)
+    return self._getPartitionPath(
+      software_type, 'srv', 'runner', 'instance', partition, *paths)
+
+
+class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
+  test_instance_max_retries = 12
+  backup_max_tries = 480
+  backup_wait_interval = 60
+
+  def _prepareExport(self):
+    super(TestTheiaResilienceERP5, self)._prepareExport()
+
+    # Connect to erp5
+    info = self._getERP5ConnexionParameters()
+    user = self._getERP5User(info)
+    password = self._getERP5Password(info)
+    url = self._getERP5Url(info, 'erp5')
+    self._waitERP5connected(url, user, password)
+
+    # Change title
+    new_title = time.strftime("HelloTitle %a %d %b %Y %H:%M:%S", time.localtime(time.time()))
+    requests.get('%s/portal_types/setTitle?value=%s' % (url, new_title), auth=(user, password), verify=False)
+    resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
+    self.assertEqual(resp.text, new_title)
+    self._erp5_new_title = new_title
+
+    # Wait until changes have been catalogued
+    mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
+    mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
+    wait_activities_script = os.path.join(
+      mariadb_partition, 'software_release', 'parts', 'erp5',
+      'Products', 'CMFActivity', 'bin', 'wait_activities')
+    subprocess.check_call((wait_activities_script, 'erp5'), env={'MYSQL': mysql_bin})
+
+    # Check that changes have been catalogued
+    output = subprocess.check_output((mysql_bin, 'erp5', '-e', 'SELECT title FROM catalog WHERE id="portal_types"'))
+    self.assertIn(new_title, output)
+
+    # Compute backup date in the near future
+    soon = (datetime.now() + timedelta(minutes=4)).replace(second=0)
+    date = '*:%d:00' % soon.minute
+    params = '_={"zodb-zeo": {"backup-periodicity": "%s"}, "mariadb": {"backup-periodicity": "%s"} }' % (date, date)
+
+    # Update ERP5 parameters
+    print('Requesting ERP5 with parameters %s' % params)
+    slapos = self._getSlapos()
+    subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
+
+    # Process twice to propagate parameter changes
+    for _ in range(2):
+      subprocess.check_call((slapos, 'node', 'instance'))
+
+    # Restart cron (actually all) services to let them take the new date into account
+    # XXX this should not be required, updating ERP5 parameters should be enough
+    subprocess.call((slapos, 'node', 'restart', 'all'))
+
+    # Wait until after the programmed backup date, and a bit more
+    t = (soon - datetime.now()).total_seconds()
+    self.assertLess(0, t)
+    time.sleep(t + 120)
+
+    # Check that mariadb backup has started
+    mariadb_backup = os.path.join(mariadb_partition, 'srv', 'backup', 'mariadb-full')
+    mariadb_backup_dump, = os.listdir(mariadb_backup)
+
+    # Check that zodb backup has started
+    zodb_backup = self._getERP5PartitionPath('export', 'zeo', 'srv', 'backup', 'zodb', 'root')
+    self.assertEqual(len(os.listdir(zodb_backup)), 3)
+
+    # Check that mariadb catalog backup contains expected changes
+    with gzip.open(os.path.join(mariadb_backup, mariadb_backup_dump)) as f:
+      self.assertIn(new_title, f.read(), "Mariadb catalog backup %s is not up to date" % mariadb_backup_dump)
+
+  def _checkTakeover(self):
+    super(TestTheiaResilienceERP5, self)._checkTakeover()
+
+    # Connect to erp5
+    info = self._getERP5ConnexionParameters()
+    user = self._getERP5User(info)
+    password = self._getERP5Password(info)
+    url = self._getERP5Url(info, 'erp5')
+    self._waitERP5connected(url, user, password)
+
+    resp = requests.get('%s/portal_types/getTitle' % url, auth=(user, password), verify=False, allow_redirects=False)
+    self.assertEqual(resp.text, self._erp5_new_title)
+
+    # Check that the mariadb catalog is not yet restored
+    mariadb_partition = self._getERP5PartitionPath('export', 'mariadb')
+    mysql_bin = os.path.join(mariadb_partition, 'bin', 'mysql')
+    query = 'SELECT title FROM catalog WHERE id="portal_types"'
+    try:
+      out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
+    except subprocess.CalledProcessError:
+      out = ''
+    self.assertNotIn(self._erp5_new_title, out)
+
+    # Stop all services
+    slapos = self._getSlapos()
+    print("Stop all services")
+    subprocess.call((slapos, 'node', 'stop', 'all'))
+
+    # Manually restore mariadb from backup
+    mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
+    print("Restore mariadb from backup")
+    subprocess.check_call(mariadb_restore_script)
+
+    # Check that the test instance is properly redeployed after restoring mariadb
+    # This restarts the services and checks the promises of the test instance
+    # Process twice to propagate state change
+    for _ in range(2):
+      self._processEmbeddedInstance(self.test_instance_max_retries)
+
+    # Check that the mariadb catalog was properly restored
+    out = subprocess.check_output((mysql_bin, 'erp5', '-e', query))
+    self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored')

software/theia/test/resilience_dummy/README.md

+# Resilience Dummy Software
+
+A very simple SR to test resiliency:
+- fast installation and deployment
+- self-contained - no dependency outside this folder
+- has a simple `exporter.exclude` and `runner-import-restore`

software/theia/test/resilience_dummy/instance.cfg.in

+[buildout]
+parts =
+  log-writer
+  exporter.exclude
+  runner-import-restore
+  backup-identity-script
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = plone.recipe.command
+home = $${buildout:directory}
+srv = $${:home}/srv
+etc = $${:home}/etc
+run = $${:etc}/run
+command = mkdir -p $${:run} $${:srv}
+
+[log-writer]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  echo "Hello : $(date)" >> $${directory:home}/log.log
+rendered = $${directory:run}/log-writer
+
+[exporter.exclude]
+recipe = slapos.recipe.template:jinja2
+template = inline:$${directory:home}/exclude
+rendered = $${directory:srv}/exporter.exclude
+
+[runner-import-restore]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  echo "Hello : $(date)" >> $${directory:home}/runner-import-restore.log
+  exit $TEST_RESTORE_STATUS
+rendered = $${directory:srv}/runner-import-restore
+
+[backup-identity-script]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  echo "Custom script"
+  for i in "$@"
+  do
+    echo $(sha256sum $i)
+  done
+  exit $TEST_BACKUP_STATUS
+rendered = $${directory:srv}/.backup_identity_script

software/theia/test/resilience_dummy/software.cfg

+[buildout]
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+
+parts =
+  instance-template
+  plone-recipe-command
+
+versions = versions
+
+[plone-recipe-command]
+recipe = zc.recipe.egg
+eggs = plone.recipe.command
+
+[instance-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+mode = 0644
+
+[versions]
+setuptools = 44.0.0
+zc.buildout = 2.7.1+slapos010
+zc.recipe.egg = 2.0.3+slapos003
+Jinja2 = 2.11.2
+MarkupSafe = 1.0

software/theia/test/test.py

@@ -26,31 +26,28 @@
 ##############################################################################
 from __future__ import unicode_literals
 
-import os
-import textwrap
+import json
 import logging
+import os
+import re
 import subprocess
-import tempfile
 import time
-import re
-import json
-from six.moves.urllib.parse import urlparse, urljoin
 
 import pexpect
 import psutil
 import requests
-import sqlite3
 import six
 
+from six.moves.urllib.parse import urlparse, urljoin
+
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-from slapos.grid.svcbackend import getSupervisorRPC
-from slapos.grid.svcbackend import _getSupervisordSocketPath
+from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
 
 
 software_cfg = 'software%s.cfg' % ('-py3' if six.PY3 else '')
-setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..', software_cfg)))
+theia_software_release_url = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', software_cfg))
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_software_release_url)
 
 
 class TheiaTestCase(SlapOSInstanceTestCase):
@@ -67,12 +64,21 @@ class TestTheia(TheiaTestCase):
   def setUp(self):
     self.connection_parameters = self.computer_partition.getConnectionParameterDict()
 
+  def get(self, url, expect_code=requests.codes.ok):
+    resp = requests.get(url, verify=False)
+    self.assertEqual(
+      expect_code,
+      resp.status_code,
+      '%s returned %d instead of %d' % (url, resp.status_code, expect_code),
+    )
+    return resp
+
   def test_backend_http_get(self):
-    resp = requests.get(self.connection_parameters['backend-url'], verify=False)
-    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    backend_url = self.connection_parameters['backend-url']
+    self.get(backend_url, requests.codes.unauthorized)
 
     # with login/password, this is allowed
-    parsed_url = urlparse(self.connection_parameters['backend-url'])
+    parsed_url = urlparse(backend_url)
     authenticated_url = parsed_url._replace(
         netloc='{}:{}@[{}]:{}'.format(
             self.connection_parameters['username'],
@@ -80,12 +86,11 @@ class TestTheia(TheiaTestCase):
             parsed_url.hostname,
             parsed_url.port,
         )).geturl()
-    resp = requests.get(authenticated_url, verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.get(authenticated_url)
 
   def test_http_get(self):
-    resp = requests.get(self.connection_parameters['url'], verify=False)
-    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+    url = self.connection_parameters['url']
+    self.get(url, requests.codes.unauthorized)
 
     # with login/password, this is allowed
     parsed_url = urlparse(self.connection_parameters['url'])
@@ -96,33 +101,28 @@ class TestTheia(TheiaTestCase):
             parsed_url.hostname,
             parsed_url.port,
         )).geturl()
-    resp = requests.get(authenticated_url, verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.get(authenticated_url)
 
     # there's a public folder to serve file
     with open('{}/srv/frontend-static/public/test_file'.format(
         self.computer_partition_root_path), 'w') as f:
       f.write("hello")
-    resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
+    resp = self.get(urljoin(authenticated_url, '/public/'))
     self.assertIn('test_file', resp.text)
-    resp = requests.get(
-        urljoin(authenticated_url, '/public/test_file'), verify=False)
+    resp = self.get(urljoin(authenticated_url, '/public/test_file'))
     self.assertEqual('hello', resp.text)
 
     # there's a (not empty) favicon
-    resp = requests.get(
-        urljoin(authenticated_url, '/favicon.ico'), verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
+    resp = self.get(urljoin(authenticated_url, '/favicon.ico'))
     self.assertTrue(resp.raw)
 
     # there is a CSS referencing fonts
-    css_text = requests.get(urljoin(authenticated_url, '/css/slapos.css'), verify=False).text
+    css_text = self.get(urljoin(authenticated_url, '/css/slapos.css')).text
     css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text)
     self.assertTrue(css_urls)
     # and fonts are served
     for url in css_urls:
-      resp = requests.get(urljoin(authenticated_url, url), verify=False)
-      self.assertEqual(requests.codes.ok, resp.status_code)
+      resp = self.get(urljoin(authenticated_url, url))
       self.assertTrue(resp.raw)
 
   def test_theia_slapos(self):
@@ -281,7 +281,7 @@ class TestTheiaEnv(TheiaTestCase):
     }
 
   def test_theia_env(self):
-    """Make sure environment variables are the same wether we use shell or supervisor services.
+    """Make sure environment variables are the same whether we use shell or supervisor services.
     """
     # The path of the env.json file expected to be generated by building the dummy software release
     env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
@@ -303,47 +303,104 @@ class TestTheiaEnv(TheiaTestCase):
     # Start a theia shell that inherits the environment of the theia process
     # This simulates the environment of a shell launched from the browser application
     theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
-    theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
-
-    # Launch slapos node software from theia shell
-    theia_shell_process.sendline('slapos node software')
-    theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
-    theia_shell_process.expect('Finished software releases.')
-
-    # Get the theia shell environment
-    with open(env_json_path) as f:
-      theia_shell_env = json.load(f)
-
-    # Remove the env.json file to later be sure that a new one has been generated
-    os.remove(env_json_path)
-
-    # Launch slapos node software service from the embedded supervisord.
-    # Note that we have two services, slapos-not-software and slapos-not-software-all
-    # The later uses --all which is what we want to use here, because the software
-    # is already installed and we want to install it again, this time from supervisor
-    embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
-    embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
-    with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
-      previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
-      embedded_supervisor.startProcess('slapos-node-software-all')
-      for _retries in range(20):
-        time.sleep(1)
-        if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
-          break
-      else:
-        self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
-
-    # Get the supervisord environment
-    with open(env_json_path) as f:
-      supervisord_env = json.load(f)
-
-    # Compare relevant variables from both environments
-    self.maxDiff = None
-    self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
-    self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
-    self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
-    self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
-
-    # Cleanup the theia shell process
-    theia_shell_process.terminate()
-    theia_shell_process.wait()
+    try:
+      theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
+
+      # Launch slapos node software from theia shell
+      theia_shell_process.sendline('slapos node software')
+      theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
+      theia_shell_process.expect('Finished software releases.')
+
+      # Get the theia shell environment
+      with open(env_json_path) as f:
+        theia_shell_env = json.load(f)
+
+      # Remove the env.json file to later be sure that a new one has been generated
+      os.remove(env_json_path)
+
+      # Launch slapos node software service from the embedded supervisord.
+      # Note that we have two services, slapos-node-software and slapos-node-software-all
+      # The later uses --all which is what we want to use here, because the software
+      # is already installed and we want to install it again, this time from supervisor
+      embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
+      embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
+      with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
+        previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
+        embedded_supervisor.startProcess('slapos-node-software-all')
+        for _retries in range(20):
+          time.sleep(1)
+          if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
+            break
+        else:
+          self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
+
+      # Get the supervisord environment
+      with open(env_json_path) as f:
+        supervisord_env = json.load(f)
+
+      # Compare relevant variables from both environments
+      self.maxDiff = None
+      self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
+      self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
+      self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
+      self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
+
+    finally:
+      # Cleanup the theia shell process
+      theia_shell_process.terminate()
+      theia_shell_process.wait()
+
+
+class ResilientTheiaMixin(object):
+  @classmethod
+  def setUpClass(cls):
+    super(ResilientTheiaMixin, cls).setUpClass()
+    # Add resiliency files to snapshot patterns
+    cls._save_instance_file_pattern_list += (
+      '*/srv/export-exitcode-file',
+      '*/srv/export-errormessage-file',
+      '*/srv/import-exitcode-file',
+      '*/srv/import-errormessage-file',
+    )
+
+  @classmethod
+  def _getPartition(cls, software_type):
+    software_url = cls.getSoftwareURL()
+    for computer_partition in cls.slap.computer.getComputerPartitionList():
+      partition_url = computer_partition.getSoftwareRelease()._software_release
+      partition_type = computer_partition.getType()
+      if partition_url == software_url and partition_type == software_type:
+        return computer_partition
+    raise Exception("Theia %s partition not found" % software_type)
+
+  @classmethod
+  def _getPartitionId(cls, software_type):
+    return cls._getPartition(software_type).getId()
+
+  @classmethod
+  def _getPartitionPath(cls, software_type, *paths):
+    return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
+
+  @classmethod
+  def _getSlapos(cls, software_type='export'):
+    return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'resilient'
+
+
+class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
+  @classmethod
+  def setUpClass(cls):
+    super(TestTheiaResilientInterface, cls).setUpClass()
+    # Patch the computer root path to that of the export theia instance
+    cls.computer_partition_root_path = cls._getPartitionPath('export')
+
+
+class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
+  @classmethod
+  def setUpClass(cls):
+    super(TestTheiaResilientWithSR, cls).setUpClass()
+    # Patch the computer root path to that of the export theia instance
+    cls.computer_partition_root_path = cls._getPartitionPath('export')

software/theia/test/test_resiliency.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import errno
+import os
+import re
+import shutil
+import six
+import subprocess
+import time
+import unittest
+
+import requests
+
+from slapos.testing.testcase import SlapOSNodeCommandError, installSoftwareUrlList
+
+from test import TheiaTestCase, ResilientTheiaMixin, theia_software_release_url
+
+
+dummy_software_url = os.path.abspath(
+  os.path.join('resilience_dummy', 'software.cfg'))
+
+
+class WorkaroundSnapshotConflict(TheiaTestCase):
+  @classmethod
+  def _copySnapshot(cls, source_file_name, name):
+    # Workaround setUpModule snapshots name conflicts
+    if not name.startswith(cls.__module__):
+      name = '%s.%s' % (cls.__module__, name)
+    super(WorkaroundSnapshotConflict, cls)._copySnapshot(source_file_name, name)
+
+
+def setUpModule():
+  installSoftwareUrlList(
+    WorkaroundSnapshotConflict,
+    [theia_software_release_url],
+    debug=bool(int(os.environ.get('SLAPOS_TEST_DEBUG', 0))),
+  )
+
+
+class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
+  @classmethod
+  def _processEmbeddedInstance(cls, retries=0, software_type='export'):
+    slapos = cls._getSlapos(software_type)
+    for _ in range(retries):
+      try:
+        output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
+      except subprocess.CalledProcessError:
+        continue
+      print(output)
+      break
+    else:
+      if retries:
+        # Sleep a bit as an attempt to workaround monitoring boostrap not being ready
+        print("Wait before running slapos node instance one last time")
+        time.sleep(120)
+      subprocess.check_call((slapos, 'node', 'instance'))
+
+  @classmethod
+  def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
+    slapos = cls._getSlapos(software_type)
+    subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
+    try:
+      subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+      print(e.output)
+      raise
+    subprocess.check_call((slapos, 'request', instance_name, software_url))
+    cls._processEmbeddedInstance(retries, software_type)
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'autorun': 'stopped'}
+
+
+class ResilienceMixin(object):
+  def _prepareExport(self):
+    pass
+
+  def _doSync(self):
+    raise NotImplementedError
+
+  def _checkSync(self):
+    pass
+
+  def _doTakeover(self):
+    raise NotImplementedError
+
+  def _checkTakeover(self):
+    pass
+
+  def test(self):
+    # Do stuff on the main instance
+    # e.g. deploy an embedded software instance
+    self._prepareExport()
+
+    # Backup the main instance to a clone
+    # i.e. call export and import scripts
+    self._doSync()
+
+    # Check that the export-backup-import process went well
+    # e.g. look at logs and compare data
+    self._checkSync()
+
+    # Let the clone become a main instance
+    # i.e. start embedded services
+    self._doTakeover()
+
+    # Check that the takeover went well
+    # e.g. check services
+    self._checkTakeover()
+
+
+class ExportAndImportMixin(object):
+  def getExportExitfile(self):
+    return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
+
+  def getExportErrorfile(self):
+    return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
+
+  def getImportExitfile(self):
+    return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
+
+  def getImportErrorfile(self):
+    return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
+
+  def makedirs(self, path):
+    try:
+      os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
+    except OSError as e:
+      if e.errno != errno.EEXIST:
+        raise
+
+  def writeFile(self, path, content, mode='w'):
+    self.makedirs(path)
+    executable = mode == 'exec'
+    mode = 'w' if executable else mode
+    with open(path, mode) as f:
+      if executable:
+        f.write('#!/bin/sh\n')
+      f.write(content)
+    if executable:
+      os.chmod(path, 0o700)
+
+  def assertPromiseSucess(self):
+    # Force promises to recompute regardless of periodicity
+    self.slap._force_slapos_node_instance_all = True
+    try:
+      self.slap.waitForInstance(error_lines=0)
+    except SlapOSNodeCommandError as e:
+      s = str(e)
+      self.assertNotIn("Promise 'resiliency-export-promise.py' failed", s)
+      self.assertNotIn('ERROR export script', s)
+      self.assertNotIn("Promise 'resiliency-import-promise.py' failed", s)
+      self.assertNotIn('ERROR import script', s)
+    else:
+      pass
+
+  def _doExport(self):
+    # Compute last modification of the export exitcode file
+    exitfile = self.getExportExitfile()
+    initial_exitdate = os.path.getmtime(exitfile)
+
+    # Call export script manually
+    theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
+    subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
+
+    # Check that the export exitcode file was modified
+    self.assertGreater(os.path.getmtime(exitfile), initial_exitdate)
+    with open(exitfile) as f:
+      self.assertEqual('0', f.read())
+
+    # Check promises
+    self.assertPromiseSucess()
+
+  def _doTransfer(self):
+    # Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
+    export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
+    import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
+    shutil.rmtree(import_backup_path)
+    shutil.copytree(export_backup_path, import_backup_path)
+
+  def _doImport(self):
+    # Compute last modification of the import exitcode file
+    exitfile = self.getImportExitfile()
+    initial_exitdate = os.path.getmtime(exitfile)
+
+    # Call the import script manually
+    theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
+    subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
+
+    # Check that the import exitcode file was updated
+    self.assertGreater(os.path.getmtime(exitfile), initial_exitdate)
+    with open(exitfile) as f:
+      self.assertEqual('0', f.read())
+
+    # Check promises
+    self.assertPromiseSucess()
+
+
+class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestCase):
+  script_relpath = os.path.join(
+    'srv', 'runner', 'instance', 'slappart0',
+    'srv', '.backup_identity_script')
+
+  def assertPromiseFailure(self, *msg):
+    # Force promises to recompute regardless of periodicity
+    self.slap._force_slapos_node_instance_all = True
+    try:
+      self.slap.waitForInstance(error_lines=0)
+    except SlapOSNodeCommandError as e:
+      s = str(e).replace('\\n', '\n')
+      for m in msg:
+        self.assertIn(m, s)
+    else:
+      self.fail('No promise failed')
+
+  def assertScriptFailure(self, func, errorfile, exitfile, *msg):
+    self.assertRaises(
+      subprocess.CalledProcessError,
+      func,
+    )
+    if msg:
+      with open(errorfile) as f:
+        error = f.read()
+      for m in msg:
+        self.assertIn(m, error)
+    with open(exitfile) as f:
+      self.assertNotEqual('0', f.read())
+
+  def assertExportFailure(self, *msg):
+    self.assertScriptFailure(
+      self._doExport,
+      self.getExportErrorfile(),
+      self.getExportExitfile(),
+      *msg)
+    self.assertPromiseFailure('ERROR export script failed', *msg)
+
+  def assertImportFailure(self, *msg):
+    self.assertScriptFailure(
+      self._doImport,
+      self.getImportErrorfile(),
+      self.getImportExitfile(),
+      *msg)
+    self.assertPromiseFailure('ERROR import script failed', *msg)
+
+  def customScript(self, path, content=None):
+    if content:
+      self.writeFile(path, content, mode='exec')
+    else:
+      if os.path.exists(path):
+        os.remove(path)
+
+  def customSignatureScript(self, content=None):
+    custom_script = self._getPartitionPath('export', self.script_relpath)
+    self.customScript(custom_script, content)
+
+  def customRestoreScript(self, content=None):
+    restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
+    self.customScript(restore_script, content)
+    return restore_script
+
+  def cleanupExitfiles(self):
+    self.writeFile(self.getExportExitfile(), '0')
+    self.writeFile(self.getImportExitfile(), '0')
+
+  def setUp(self):
+    self.customSignatureScript(content=None)
+    self.customRestoreScript(content=None)
+    self.cleanupExitfiles()
+
+  def test_export_promise(self):
+    self.writeFile(self.getExportExitfile(), '1')
+    self.assertPromiseFailure('ERROR export script failed')
+
+  def test_import_promise(self):
+    self.writeFile(self.getImportExitfile(), '1')
+    self.assertPromiseFailure('ERROR import script failed')
+
+  def test_custom_hash_script(self):
+    errmsg = 'Bye bye'
+    self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
+    backup_script = self._getPartitionPath(
+      'export', 'srv', 'backup', 'theia', self.script_relpath)
+    self.assertExportFailure('Compute backup signature\n ... ERROR !',
+      'Custom signature script %s failed' % os.path.abspath(backup_script),
+      'and stderr:\n%s' % errmsg)
+
+  def test_signature_mismatch(self):
+    signature_file = self._getPartitionPath('import', 'srv', 'backup', 'theia', 'backup.signature')
+    moved_file = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.moved')
+    self.writeFile(moved_file, 'Bogus Hash\n', mode='a')
+    os.rename(moved_file, signature_file)
+    self.assertImportFailure('ERROR the backup signatures do not match')
+
+  def test_restore_script_error(self):
+    self._doExport()
+    self._doTransfer()
+    restore_script = self.customRestoreScript('exit 1')
+    self.assertImportFailure('Run custom restore script %s\n ... ERROR !' % restore_script)
+
+
+class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientTheiaTestCase):
+  def test_twice(self):
+    # Run two synchronisations on the same instances
+    # to make sure everything still works the second time
+    self._doSync()
+
+  def checkLog(self, log_path, initial=[], newline="Hello"):
+    with open(log_path) as f:
+      log = f.readlines()
+    self.assertEqual(len(log), len(initial) + int(bool(newline)))
+    for line, initial_line in zip(log, initial):
+      self.assertEqual(line, initial_line)
+    if newline:
+      self.assertTrue(log[-1].startswith(newline), log[-1])
+    return log
+
+  def _prepareExport(self):
+    # Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
+    dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
+    shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
+    self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
+
+    # Deploy dummy instance in export partition
+    self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
+
+    relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
+    self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
+    self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
+
+    # Check that dummy instance was properly deployed
+    self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
+
+    # Create ~/include and ~/include/included
+    self.writeFile(os.path.join(dummy_root, 'include', 'included'),
+      'This file should be included in resilient backup')
+
+    # Create ~/exclude and ~/exclude/excluded
+    self.writeFile(os.path.join(dummy_root, 'exclude', 'excluded'),
+      'This file should be excluded from resilient backup')
+
+    # Check that ~/srv/exporter.exclude and ~/srv/runner-import-restore
+    # As well as ~/srv/.backup_identity_script
+    self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'exporter.exclude')))
+    self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', 'runner-import-restore')))
+    self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
+
+  def _doSync(self):
+    self._doExport()
+    self._doTransfer()
+    self._doImport()
+
+  def _checkSync(self):
+    dummy_root = self.import_dummy_root
+
+    # Check that the software url is correct
+    adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
+    proxy_content = subprocess.check_output(
+      (self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
+    self.assertIn(adapted_test_url, proxy_content)
+    self.assertNotIn(self._test_software_url, proxy_content)
+
+    # Check that ~/srv/project was exported
+    self.assertTrue(os.path.exists(adapted_test_url))
+
+    # Check that the dummy instance is not yet started
+    self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
+
+    # Check that ~/srv/.backup_identity_script was called
+    signature = self._getPartitionPath('import', 'srv', 'backup', 'backup.signature.proof')
+    with open(signature) as f:
+      self.assertIn('Custom script', f.read())
+
+    # Check that ~/include and ~/include/included were included
+    self.assertTrue(os.path.exists(os.path.join(dummy_root, 'include', 'included')))
+
+    # Check that ~/exclude was excluded
+    self.assertFalse(os.path.exists(os.path.join(dummy_root, 'exclude')))
+
+    # Check that ~/srv/runner-import-restore was called
+    self.checkLog(os.path.join(dummy_root, 'runner-import-restore.log'))
+
+  def _doTakeover(self):
+    # Start the dummy instance as a sort of fake takeover
+    subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
+
+  def _checkTakeover(self):
+    # Check that dummy instance was properly re-deployed
+    log_path = os.path.join(self.import_dummy_root, 'log.log')
+    self.checkLog(log_path, self.initial_log)
+
+
+class TakeoverMixin(ExportAndImportMixin):
+  def _getTakeoverUrlAndPassword(self, scope="theia-1"):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    takeover_url = parameter_dict["takeover-%s-url" % scope]
+    takeover_password = parameter_dict["takeover-%s-password" % scope]
+    return takeover_url, takeover_password
+
+  def _getTakeoverPage(self, takeover_url):
+    resp = requests.get(takeover_url, verify=True)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    return resp.text
+
+  def _waitScriptDone(self, name, start, exitfile, errorfile, maxtries, interval):
+    print('Wait until %s script has run' % name.lower())
+    for t in range(maxtries):
+      if os.path.getmtime(exitfile) < start:
+        time.sleep(interval)
+        continue
+      with open(exitfile) as f:
+        if f.read() == '0':
+          print(name + ' script ran successfully')
+          return maxtries - t
+      print(name + ' script failed:\n')
+      with open(errorfile) as f:
+        print(f.read())
+      self.fail(name + ' script failed')
+    self.fail(name + ' script did not finish before timeout')
+
+  def _waitTakeoverReady(self, takeover_url, start, maxtries, interval):
+    export_exitfile = self.getExportExitfile()
+    export_errorfile =  self.getExportErrorfile()
+    tries = self._waitScriptDone(
+      'Export', start, export_exitfile, export_errorfile, maxtries, interval)
+    import_exitfile = self.getImportExitfile()
+    import_errorfile =  self.getImportErrorfile()
+    tries = self._waitScriptDone(
+      'Import', start, import_exitfile, import_errorfile, tries, interval)
+    for _ in range(tries):
+      info = self._getTakeoverPage(takeover_url)
+      if "No backup downloaded yet, takeover should not happen now." in info:
+        print('Takeover page still reports export script in progress')
+      elif "<b>Importer script(s) of backup in progress:</b> True" in info:
+        print('Takeover page still reports import script in progress')
+      else:
+        return
+      time.sleep(interval)
+    self.fail('Takeover page failed to report readiness')
+
+  def _requestTakeover(self, takeover_url, takeover_password):
+    resp = requests.get("%s?password=%s" % (takeover_url, takeover_password), verify=True)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertNotIn("Error", resp.text, "An Error occured: %s" % resp.text)
+    self.assertIn("Success", resp.text, "An Error occured: %s" % resp.text)
+    return resp.text
+
+
+@unittest.skipIf(six.PY3, "resilient stack is not python3-compatible")
+class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase):
+  test_instance_max_retries = 0
+  backup_max_tries = 70
+  backup_wait_interval = 10
+
+  _test_software_url = dummy_software_url
+
+  def _prepareExport(self):
+    # Deploy test instance
+    self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
+
+    # Check that there is an export and import instance and get their partition IDs
+    self.export_id = self._getPartitionId('export')
+    self.import_id = self._getPartitionId('import')
+
+  def _doSync(self):
+    start = time.time()
+
+    # Call exporter script instead of waiting for cron job
+    # XXX Accelerate cron frequency instead ?
+    exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
+    transaction_id = str(int(time.time()))
+    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
+
+    takeover_url, _ = self._getTakeoverUrlAndPassword()
+
+    # Wait for takoever to be ready
+    self._waitTakeoverReady(takeover_url, start, self.backup_max_tries, self.backup_wait_interval)
+
+  def _doTakeover(self):
+    # Takeover
+    takeover_url, takeover_password = self._getTakeoverUrlAndPassword()
+    self._requestTakeover(takeover_url, takeover_password)
+
+    # Wait for import instance to become export instance and new import to be allocated
+    # This also checks that all promises of theia instances succeed
+    self.slap.waitForInstance(self.instance_max_retry)
+    self.computer_partition = self.requestDefaultInstance()
+
+  def _checkTakeover(self):
+    # Check that there is an export, import and frozen instance and get their new partition IDs
+    import_id = self.import_id
+    export_id = self.export_id
+    new_export_id = self._getPartitionId('export')
+    new_import_id = self._getPartitionId('import')
+    new_frozen_id = self._getPartitionId('frozen')
+
+    # Check that old export instance is now frozen
+    self.assertEqual(export_id, new_frozen_id)
+
+    # Check that old import instance is now the new export instance
+    self.assertEqual(import_id, new_export_id)
+
+    # Check that there is a new import instance
+    self.assertNotIn(new_import_id, (export_id, new_export_id))
+
+    # Check that the test instance is properly redeployed
+    # This checks the promises of the test instance
+    self._processEmbeddedInstance(self.test_instance_max_retries)

software/theia/theia_common.py

+import contextlib
+import errno
+import glob
+import hashlib
+import os
+import re
+import subprocess as sp
+import sqlite3
+
+import six
+import zc.buildout.configparser
+
+from slapos.util import bytes2str, str2bytes
+
+
+RSYNC_FLAGS = ('-rlptgo', '--safe-links', '--stats', '--ignore-missing-args', '--delete', '--delete-excluded')
+RSYNC_REGEX = '^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+EXCLUDE_PATTERNS = ('*.sock', '*.socket', '*.pid', '.installed*.cfg')
+EXCLUDE_FLAGS = ['--exclude={}'.format(x) for x in sorted(EXCLUDE_PATTERNS)]
+
+
+def makedirs(path):
+  try:
+    os.makedirs(path if os.path.isdir(path) else os.path.dirname(path))
+  except OSError as e:
+    if e.errno != errno.EEXIST:
+      raise
+
+
+def copytree(rsyncbin, src, dst, exclude=[], extrargs=[], verbosity='-v'):
+  # Ensure there is a trailing slash in the source directory
+  # to avoid creating an additional directory level at the destination
+  src = os.path.join(src, '')
+
+  # Compute absolute path of destination
+  dst = os.path.abspath(dst)
+
+  # Create destination dir if it doesn't exist
+  makedirs(dst)
+
+  command = [rsyncbin]
+  command.extend(RSYNC_FLAGS)
+
+  # Exclude destination file from sources
+  command.append('--filter=-/ {}'.format(dst))
+
+  command.extend(EXCLUDE_FLAGS)
+  command.extend(('--filter=-/ {}'.format(x) for x in sorted(exclude)))
+  command.extend(extrargs)
+  command.append(verbosity)
+  command.append(src)
+  command.append(dst)
+  try:
+    return sp.check_output(command, universal_newlines=True)
+  except sp.CalledProcessError as e:
+    # Not all rsync errors are to be considered as errors
+    if e.returncode != 24 or re.search(RSYNC_REGEX, e.output, re.M) is None:
+      raise
+    return e.output
+
+
+def copydb(sqlite3bin, src_db, dst_db):
+  makedirs(dst_db)
+  sp.check_output((sqlite3bin, src_db, '.backup ' + dst_db))
+
+
+def remove(path):
+  try:
+    os.remove(path)
+  except OSError:
+    if os.path.exists(path):
+      raise
+
+
+def parse_installed(partition):
+  paths = []
+  custom_script = os.path.join(partition, 'srv', '.backup_identity_script')
+  for cfg in glob.glob(os.path.join(partition, '.installed*.cfg')):
+    try:
+      with open(cfg) as f:
+        installed_cfg = zc.buildout.configparser.parse(f, cfg)
+    except IOError as e:
+      if e.errno != errno.ENOENT:
+        raise
+    else:
+      for section in six.itervalues(installed_cfg):
+        for p in section.get('__buildout_installed__', '').splitlines():
+          p = p.strip()
+          if p and p != custom_script:
+            paths.append(p)
+  return paths
+
+
+def sha256sum(file_path, chunk_size=1024 * 1024):
+  sha256 = hashlib.sha256()
+  with open(file_path, 'rb') as f:
+    chunk = f.read(chunk_size)
+    while chunk:
+      sha256.update(chunk)
+      chunk = f.read(chunk_size)
+  return sha256.hexdigest()
+
+
+def hashwalk(backup_dir, mirror_partitions):
+  scripts = {}
+  for p in mirror_partitions:
+    script_path = os.path.join(p, 'srv', '.backup_identity_script')
+    if os.path.exists(script_path):
+      scripts[os.path.abspath(p)] = script_path
+  for dirpath, dirnames, filenames in os.walk(backup_dir):
+    filenames.sort()
+    for f in filenames:
+      filepath = os.path.join(dirpath, f)
+      if os.path.isfile(filepath):
+        displaypath = os.path.relpath(filepath, start=backup_dir)
+        yield '%s %s' % (sha256sum(filepath), displaypath)
+    remaining_dirnames = []
+    for subdir in dirnames:
+      subdirpath = os.path.abspath(os.path.join(dirpath, subdir))
+      custom_hashscript = scripts.get(subdirpath)
+      if custom_hashscript:
+        print('Using custom signature script %s' % custom_hashscript)
+        for s in hashcustom(subdirpath, backup_dir, custom_hashscript):
+          yield s
+      else:
+        remaining_dirnames.append(subdir)
+    remaining_dirnames.sort()
+    dirnames[:] = remaining_dirnames
+
+
+@contextlib.contextmanager
+def cwd(path):
+  old_path = os.getcwd()
+  try:
+    os.chdir(path)
+    yield
+  finally:
+    os.chdir(old_path)
+
+
+def hashcustom(mirrordir, backup_dir, custom_hashscript):
+  workingdir = os.path.join(mirrordir, os.pardir, os.pardir, os.pardir)
+  with cwd(os.path.abspath(workingdir)):
+    for dirpath, _, filenames in os.walk(mirrordir):
+      filepaths = []
+      for f in filenames:
+        path = os.path.join(dirpath, f)
+        if os.path.isfile(path):
+          filepaths.append('./' + os.path.relpath(path, start=workingdir))
+      if not filepaths:
+        continue
+      hashprocess = sp.Popen(
+        custom_hashscript, stdin=sp.PIPE, stdout=sp.PIPE, stderr=sp.PIPE)
+      out, err = hashprocess.communicate(str2bytes('\0'.join(filepaths)))
+      if hashprocess.returncode != 0:
+        template = "Custom signature script %s failed on inputs:\n%s"
+        msg = template % (custom_hashscript, '\n'.join(filepaths))
+        msg += "\nwith stdout:\n%s" % bytes2str(out)
+        msg += "\nand stderr:\n%s" % bytes2str(err)
+        raise Exception(msg)
+      signatures = bytes2str(out).strip('\n').split('\n')
+      signatures.sort()
+      displaypath = os.path.relpath(dirpath, start=backup_dir)
+      for s in signatures:
+        yield '%s %s/ (custom)' % (s, displaypath)

software/theia/theia_export.py

+import argparse
+import glob
+import itertools
+import os
+import sys
+import time
+import traceback
+
+import six
+from six.moves import configparser
+
+sys.path.append(os.path.dirname(__file__))
+from theia_common import copytree, copydb, hashwalk, parse_installed, remove
+
+
+os.environ['LC_ALL'] = 'C'
+os.umask(0o77)
+
+
+BACKUP_WAIT = 10
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--rsync', required=True)
+  parser.add_argument('--sqlite3', required=True)
+  parser.add_argument('--root', required=True)
+  parser.add_argument('--backup', required=True)
+  parser.add_argument('--cfg', required=True)
+  parser.add_argument('--dirs', action='append')
+  parser.add_argument('--exitfile', required=True)
+  parser.add_argument('--errorfile', required=True)
+  args = parser.parse_args()
+
+  TheiaExport(args)()
+
+
+class TheiaExport(object):
+
+  def __init__(self, args):
+    self.rsync_bin = args.rsync
+    self.sqlite3_bin = args.sqlite3
+    self.root_dir = args.root
+    self.backup_dir = args.backup
+    self.slapos_cfg = cfg = args.cfg
+    self.dirs = args.dirs
+    self.exit_file = args.exitfile
+    self.error_file = args.errorfile
+    configp = configparser.SafeConfigParser()
+    configp.read(cfg)
+    self.proxy_db = configp.get('slapproxy', 'database_uri')
+    self.instance_dir = configp.get('slapos', 'instance_root')
+    partitions = glob.glob(os.path.join(self.instance_dir, 'slappart*'))
+    self.partition_dirs = [p for p in partitions if os.path.isdir(p)]
+    self.copytree_partitions_args = {}
+    self.logs = []
+
+  def mirrorpath(self, src):
+    return os.path.abspath(os.path.join(
+      self.backup_dir, os.path.relpath(src, start=self.root_dir)))
+
+  def backuptree(self, src, exclude=[], extrargs=[], verbosity='-v'):
+    dst = self.mirrorpath(src)
+    return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
+
+  def backupdb(self):
+    copydb(self.sqlite3_bin, self.proxy_db, self.mirrorpath(self.proxy_db))
+
+  def backuppartition(self, partition):
+    installed = parse_installed(partition)
+    rules = os.path.join(partition, 'srv', 'exporter.exclude')
+    extrargs = ('--filter=.-/ ' + rules,) if os.path.exists(rules) else ()
+    self.backuptree(partition, exclude=installed, extrargs=extrargs)
+    self.copytree_partitions_args[partition] = (installed, extrargs)
+
+  def sign(self, signaturefile):
+    remove(signaturefile)
+    pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
+    tmpfile = os.path.join(pardir, 'backup.signature.tmp')
+    mirror_partitions = [self.mirrorpath(p) for p in self.partition_dirs]
+    with open(tmpfile, 'w') as f:
+      for s in hashwalk(self.backup_dir, mirror_partitions):
+        f.write(s + '\n')
+    os.rename(tmpfile, signaturefile)
+
+  def checkpartition(self, partition, pattern='/srv/backup/'):
+    installed, extrargs = self.copytree_partitions_args[partition]
+    output = self.backuptree(
+      partition,
+      exclude=installed,
+      extrargs=extrargs + ('--dry-run', '--update'),
+      verbosity='--out-format=%n',
+    )
+    return [path for path in output.splitlines() if pattern in path]
+
+  def loginfo(self, msg):
+    print(msg)
+    self.logs.append(msg)
+
+  def __call__(self):
+    remove(self.error_file)
+    exitcode = 0
+    try:
+      self.export()
+    except Exception:
+      exitcode = 1
+      exc = traceback.format_exc()
+      with open(self.error_file, 'w') as f:
+        f.write('\n ... OK\n\n'.join(self.logs))
+        f.write('\n ... ERROR !\n\n')
+        f.write(exc)
+      print('\n\nERROR\n\n' + exc)
+    finally:
+      with open(self.exit_file, 'w') as f:
+        f.write(str(exitcode))
+      sys.exit(exitcode)
+
+  def export(self):
+    export_start_date = int(time.time())
+
+    etc_dir = os.path.join(self.root_dir, 'etc')
+    with open(os.path.join(etc_dir, '.resilient_timestamp'), 'w') as f:
+      f.write(str(export_start_date))
+
+    self.loginfo('Backup directory ' + etc_dir)
+    self.backuptree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
+
+    for d in self.dirs:
+      self.loginfo('Backup directory ' + d)
+      self.backuptree(d)
+
+    self.loginfo('Backup slapproxy database')
+    self.backupdb()
+
+    self.loginfo('Backup partitions')
+    for p in self.partition_dirs:
+      self.backuppartition(p)
+
+    self.loginfo('Compute backup signature')
+    self.sign(os.path.join(self.backup_dir, 'backup.signature'))
+
+    time.sleep(10)
+    self.loginfo('Check partitions')
+    modified = list(itertools.chain.from_iterable(
+      self.checkpartition(p) for p in self.partition_dirs))
+    if modified:
+      msg = 'Some files have been modified since the backup started'
+      self.loginfo(msg + ':')
+      self.loginfo('\n'.join(modified))
+      self.loginfo("Let's wait %d minutes and try again" % BACKUP_WAIT)
+      time.sleep(BACKUP_WAIT * 60)
+      raise Exception(msg)
+
+    self.loginfo('Done')
+
+
+if __name__ == '__main__':
+  main()

software/theia/theia_import.py

+import argparse
+import glob
+import itertools
+import os
+import sys
+import subprocess as sp
+import traceback
+
+import six
+from six.moves import configparser
+
+sys.path.append(os.path.dirname(__file__))
+from theia_common import copytree, copydb, hashwalk, parse_installed, remove
+
+
+os.environ['LC_ALL'] = 'C'
+os.umask(0o77)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--rsync', required=True)
+  parser.add_argument('--sqlite3', required=True)
+  parser.add_argument('--slapos', required=True)
+  parser.add_argument('--srlog', required=True)
+  parser.add_argument('--cplog', required=True)
+  parser.add_argument('--supervisorctl', required=True)
+  parser.add_argument('--supervisordconf', required=True)
+  parser.add_argument('--root', required=True)
+  parser.add_argument('--backup', required=True)
+  parser.add_argument('--cfg', required=True)
+  parser.add_argument('--dirs', action='append')
+  parser.add_argument('--exitfile', required=True)
+  parser.add_argument('--errorfile', required=True)
+  args = parser.parse_args()
+
+  TheiaImport(args)()
+
+
+class TheiaImport(object):
+
+  def __init__(self, args):
+    self.rsync_bin = args.rsync
+    self.sqlite3_bin = args.sqlite3
+    self.slapos_bin = args.slapos
+    self.sr_log = args.srlog
+    self.cp_log = args.cplog
+    self.supervisorctl_bin = args.supervisorctl
+    self.supervisord_conf = args.supervisordconf
+    self.root_dir = args.root
+    self.backup_dir = args.backup
+    self.slapos_cfg = cfg = args.cfg
+    self.dirs = args.dirs
+    self.exit_file = args.exitfile
+    self.error_file = args.errorfile
+    configp = configparser.SafeConfigParser()
+    configp.read(cfg)
+    self.proxy_db = configp.get('slapproxy', 'database_uri')
+    self.instance_dir = configp.get('slapos', 'instance_root')
+    mirror_dir = self.mirrorpath(self.instance_dir)
+    partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
+    self.mirror_partition_dirs = [p for p in partitions if os.path.isdir(p)]
+    self.logs = []
+
+  def mirrorpath(self, dst):
+    return os.path.abspath(os.path.join(
+      self.backup_dir, os.path.relpath(dst, start=self.root_dir)))
+
+  def dstpath(self, src):
+    return os.path.abspath(os.path.join(
+      self.root_dir, os.path.relpath(src, start=self.backup_dir)))
+
+  def restoretree(self, dst, exclude=[], extrargs=[], verbosity='-v'):
+    src = self.mirrorpath(dst)
+    return copytree(self.rsync_bin, src, dst, exclude, extrargs, verbosity)
+
+  def restoredb(self):
+    copydb(self.sqlite3_bin, self.mirrorpath(self.proxy_db), self.proxy_db)
+
+  def restorepartition(self, mirror_partition):
+    p = self.dstpath(mirror_partition)
+    installed = parse_installed(p) if os.path.exists(p) else []
+    copytree(self.rsync_bin, mirror_partition, p, exclude=installed)
+
+  def supervisorctl(self, *args):
+    supervisor_command = (self.supervisorctl_bin, '-c', self.supervisord_conf)
+    command = supervisor_command + args
+    print(' '.join(command))
+    sp.check_call(command)
+
+  def slapos(self, *args):
+    command = (self.slapos_bin,) + args + ('--cfg', self.slapos_cfg)
+    print(' '.join(command))
+    sp.check_call(command)
+
+  def verify(self, signaturefile):
+    pardir = os.path.abspath(os.path.join(self.backup_dir, os.pardir))
+    moved = os.path.join(pardir, 'backup.signature.moved')
+    proof = os.path.join(pardir, 'backup.signature.proof')
+    if os.path.exists(signaturefile):
+      os.rename(signaturefile, moved)
+    if not os.path.exists(moved):
+      msg = 'ERROR the backup signature file is missing'
+      print(msg)
+      raise Exception(msg)
+    with open(proof, 'w') as f:
+      for s in hashwalk(self.backup_dir, self.mirror_partition_dirs):
+        f.write(s + '\n')
+    diffcommand = ('diff', moved, proof)
+    print(' '.join(diffcommand))
+    try:
+      sp.check_output(
+        diffcommand, stderr=sp.STDOUT, universal_newlines=True)
+    except sp.CalledProcessError as e:
+      template = 'ERROR the backup signatures do not match\n\n%s'
+      msg = template % e.output
+      print(msg)
+      raise Exception(msg)
+
+  def loginfo(self, msg):
+    print(msg)
+    self.logs.append(msg)
+
+  def __call__(self):
+    remove(self.error_file)
+    exitcode = 0
+    try:
+      self.restore()
+    except Exception:
+      exitcode = 1
+      exc = traceback.format_exc()
+      with open(self.error_file, 'w') as f:
+        f.write('\n ... OK\n\n'.join(self.logs))
+        f.write('\n ... ERROR !\n\n')
+        f.write(exc)
+      print('\n\nERROR\n\n' + exc)
+    finally:
+      with open(self.exit_file, 'w') as f:
+        f.write(str(exitcode))
+      sys.exit(exitcode)
+
+  def restore(self):
+    self.loginfo('Verify backup signature')
+    self.verify(os.path.join(self.backup_dir, 'backup.signature'))
+
+    self.loginfo('Stop slapproxy')
+    self.supervisorctl('stop', 'slapos-proxy')
+
+    self.loginfo('Restore partitions')
+    for m in self.mirror_partition_dirs:
+      self.restorepartition(m)
+
+    for d in self.dirs:
+      self.loginfo('Restore directory ' + d)
+      self.restoretree(d)
+
+    self.loginfo('Restore slapproxy database')
+    self.restoredb()
+
+    etc_dir = os.path.join(self.root_dir, 'etc')
+    self.loginfo('Restore directory ' + etc_dir)
+    self.restoretree(etc_dir, extrargs=('--filter=- */', '--filter=-! .*'))
+
+    custom_script = os.path.join(self.root_dir, 'srv', 'runner-import-restore')
+    if os.path.exists(custom_script):
+      self.loginfo('Run custom restore script %s' % custom_script)
+      sp.check_call(custom_script)
+
+    self.loginfo('Start slapproxy again')
+    self.supervisorctl('start', 'slapos-proxy')
+
+    self.loginfo('Reformat partitions')
+    self.slapos('node', 'format', '--now')
+
+    self.loginfo('Remove old supervisord configuration files')
+    conf_dir = os.path.join(self.instance_dir, 'etc', 'supervisor.conf.d')
+    for f in glob.glob(os.path.join(conf_dir, '*')):
+      os.remove(f)
+
+    self.loginfo('Build Software Releases')
+    for i in range(3):
+      try:
+        self.slapos('node', 'software', '--all', '--logfile', self.sr_log)
+      except sp.CalledProcessError:
+        if i == 2:
+          raise
+      else:
+        break
+
+    self.loginfo('Remove old custom instance scripts')
+    partitions_glob = os.path.join(self.instance_dir, 'slappart*')
+    scripts = os.path.join(partitions_glob, 'srv', 'runner-import-restore')
+    for f in glob.glob(scripts):
+      remove(f)
+
+    self.loginfo('Remove partition timestamps')
+    timestamps = os.path.join(partitions_glob, '.timestamp')
+    for f in glob.glob(timestamps):
+      remove(f)
+
+    self.loginfo('Build Instances')
+    cp_log = self.cp_log
+    for i in range(3):
+      try:
+        self.slapos('node', 'instance', '--force-stop', '--logfile', cp_log)
+      except sp.CalledProcessError:
+        if i == 2:
+          raise
+      else:
+        break
+
+    for custom_script in glob.glob(scripts):
+      self.loginfo('Running custom instance script %s' % custom_script)
+      sp.check_call(custom_script)
+
+    self.loginfo('Done')
+
+
+if __name__ == '__main__':
+  main()

software/unstable/condor/software.cfg

@@ -62,31 +62,10 @@ slapos.recipe.cmmi = 0.1.1
 slapos.recipe.template = 2.4.2
 xml-marshaller = 0.9.7
 slapos.cookbook=0.78.1
-
-# Required by:
-# slapos.core==0.35.1
 Flask = 0.9
-
-# Required by:
-# slapos.cookbook==0.78.1
 lock-file = 2.0
-
-# Required by:
-# slapos.core==0.35.1
 netifaces = 0.10.4
-
-# Required by:
-# slapos.core==0.35.1
 pyflakes = 0.7.2
-
-# Required by:
-# slapos.core==0.35.1
 supervisor = 3.0b1
-
-# Required by:
-# slapos.core==0.35.1
 unittest2 = 0.5.1
-
-# Required by:
-# slapos.core==0.35.1
 zope.interface = 4.0.5

stack/cloudooo.cfg

@@ -75,8 +75,4 @@ PasteScript = 3.2.0:whl
 WSGIUtils = 0.7.2
 WSGIserver = 1.3
 python-magic = 0.4.18
-
-# Required by:
-# PasteScript== 3.2.0
-# cloudooo==1.2.6.dev0
 PasteDeploy = 2.1.0

stack/erp5/buildout.cfg

@@ -720,88 +720,39 @@ cloudpickle = 0.5.3
 dask = 0.18.1
 toolz = 0.9.0
 zope.globalrequest = 1.5
-waitress = 1.3.0
+waitress = 1.4.4
 xlrd = 1.1.0
 
 # Re-add for as it is required to be there for uninstallation
 erp5.recipe.w3validator = 1.0.2
-
-# Required by:
-# Products.CMFCore==2.2.10
 Products.ZSQLMethods = 2.13.5
-
-# Required by:
-# SOAPpy===0.12.0nxd001
 fpconst = 0.7.2
-
-# Required by:
-# objgraph==3.1.0
 graphviz = 0.5.2
-
-# Required by:
-# Pillow==6.2.2
 olefile = 0.44
-
-# Required by:
-# munnel==0.3
 python-libmilter = 1.0.3
-
-# Required by:
-# zope.app.testing==3.8.1
 zope.app.debug = 3.4.1
-
-# Required by:
-# zope.app.testing==3.8.1
 zope.app.dependable = 3.5.1
-
-# Required by:
-# Products.CMFCalendar==2.2.3
-# five.formlib==1.0.4
 zope.app.form = 4.0.2
-
 et-xmlfile = 1.0.1
 h5py = 2.7.1
 mpmath = 0.19
 openpyxl = 2.4.8
 sympy = 1.1.1
-
-# Required by:
-# openpyxl==2.4.8
 jdcal = 1.3
-
 xmltodict = 0.11.0
 deepdiff = 3.3.0
 unidiff = 0.5.5
-
-# Required by:
-# deepdiff = 3.3.0
 jsonpickle = 0.9.6
-
 responses = 0.10.6
-
-# Required by:
-# responses = 0.10.6
 cookies = 2.2.1
-
 jedi = 0.15.1
 parso = 0.5.1
-
 yapf = 0.28.0
-
-# Required by:
-# erp5.util==0.4.65
 z3c.etestbrowser = 3.0.1
 zope.testbrowser = 5.5.1
-
-# Required by:
-# zope.testbrowser==5.5.1
 WSGIProxy2 = 0.4.6
 WebTest = 2.0.33
 beautifulsoup4 = 4.8.2
-
-# Required by:
-# WSGIProxy2==0.4.6
 WebOb = 1.8.5
 soupsieve = 1.9.5
-
 eggtestinfo = 0.3

stack/resilient/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [pbsready]
 filename = pbsready.cfg.in
-md5sum = 9ceceeee21fa90837c887d2d6866859e
+md5sum = 005125621d157b3ae04c428ea6060e37
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
@@ -26,7 +26,7 @@ md5sum = 2b0c71b085cfe8017f28098c160b1f49
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = e64e13854332bcc2595df187fcae1203
+md5sum = b240dc76a663190304d8bcb9cabcda8f
 
 [template-replicated]
 filename = template-replicated.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -137,12 +137,17 @@ command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --
 feed-path = $${directory:monitor-resilient}/pbs-status-rss
 wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
 
+[pbs-status-feed-first-run]
+recipe = plone.recipe.command
+command = $${pbs-resilient-status-feed:wrapper-path}
+stop-on-error = true
+
 [cron-pbs-status-feed]
 <= cron
 recipe = slapos.cookbook:cron.d
 name = resilient-pbs-status-feed
 frequency = */5 * * * *
-command = $${pbs-resilient-status-feed:wrapper-path}
+command = $${pbs-status-feed-first-run:command}
 
 [logrotate-entry-notifier]
 <= logrotate-entry-base

stack/resilient/pbsready.cfg.in

@@ -145,12 +145,17 @@ command-line = ${buildout:bin-directory}/generatefeed --output $${:feed-path} --
 feed-path = $${directory:monitor-resilient}/notifier-status-rss
 wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py
 
+[notifier-status-feed-first-run]
+recipe = plone.recipe.command
+command = $${notifier-resilient-status-feed:wrapper-path}
+stop-on-error = true
+
 [cron-entry-notifier-status-feed]
 <= cron
 recipe = slapos.cookbook:cron.d
 name = resilient-notifier-status-feed
 frequency = */5 * * * *
-command = $${notifier-resilient-status-feed:wrapper-path}
+command = $${notifier-status-feed-first-run:command}
 
 [notifier-stalled-promise-bin]
 recipe = slapos.cookbook:wrapper

stack/slapos.cfg

@@ -186,11 +186,12 @@ pytz = 2016.10
 regex = 2020.9.27
 requests = 2.24.0
 scandir = 1.10.0
+setproctitle = 1.1.10
 setuptools-dso = 1.7
-rubygemsrecipe  = 0.3.0
+rubygemsrecipe  = 0.4.2
 six = 1.12.0
 slapos.cookbook = 1.0.197
-slapos.core = 1.6.17
+slapos.core = 1.6.18
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
@@ -208,141 +209,46 @@ xml-marshaller = 1.0.2
 zc.lockfile = 1.0.2
 zdaemon = 4.2.0
 zipp = 1.2.0:whl
-zodburi = 2.4.0
+zodburi = 2.5.0
 zope.event = 3.5.2
 paramiko = 2.1.3
 CacheControl = 0.12.6:whl
 msgpack = 0.6.2
-
-# Required by:
-# slapos.core==1.5.0
 Flask = 1.1.2
-
-# Required by:
-# slapos.toolbox==0.94
 GitPython = 2.1.11
-
-# Required by:
-# GitPython==2.1.11
 gitdb2 = 2.0.5
-
-# Required by:
-# gitdb==2.0.5
 smmap2 = 2.0.5
-
-# Required by:
-# slapos.toolbox==0.94
 PyRSS2Gen = 1.1
-
-# Required by:
-# slapos.toolbox==0.94
 apache-libcloud = 2.4.0
-
-# Required by:
-# slapos.toolbox==0.94
 atomize = 0.2.0
-
-# Required by:
-# slapos.toolbox==0.94
 croniter = 0.3.25
-
-# Required by:
-# slapos.toolbox==0.94
 dnspython = 1.16.0
-
 enum34 = 1.1.10
-
-# Required by:
-# slapos.toolbox==0.94
-erp5.util = 0.4.69
-
-# Required by:
-# slapos.toolbox==0.94
+erp5.util = 0.4.70
 feedparser = 5.2.1
-
-# Required by:
-# jsonschema==3.0.2
 functools32 = 3.2.3.post2
-
-# Required by:
-# jsonschema==3.0.2
 attrs = 18.2.0
-
-# Required by:
-# jsonschema==3.0.2
 pyrsistent = 0.16.1
-
 pytest-runner = 5.2.0:whl
-
 ipaddress = 1.0.23
-
-# Required by:
-# slapos.cookbook==1.0.143
 jsonschema = 3.0.2:whl
-
-# Required by:
-# slapos.toolbox==0.94
 lockfile = 0.12.2:whl
-
-# Required by:
-# slapos.core==1.5.0
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.7
-
 packaging = 16.8
-
-# Required by:
-# slapos.toolbox==0.94
 passlib = 1.7.1
-
-# Required by:
-# slapos.toolbox==0.94
 pyasn1 = 0.4.5
-
-# Required by:
-# cffi==1.9.1
 pycparser = 2.20
-
-# Required by:
-# slapos.toolbox==0.94
 pycurl = 7.43.0
-
-# Required by:
-# slapos.toolbox==0.94
 python-dateutil = 2.7.3:whl
-
-# Required by:
-# slapos.toolbox==0.94
 rpdb = 0.1.5
-
-# Required by:
-# slapos.core==1.5.0
 supervisor = 4.1.0
-
-# Required by:
-# slapos.toolbox==0.94
 tzlocal = 1.5.1
-
-# Required by:
-# slapos.core==1.5.0
 uritemplate = 3.0.0
-
-# Required by:
-# slapos.core==1.5.0
 zope.interface = 4.3.3
-
-# Required by:
-# requests==2.24.0
 certifi = 2020.6.20
-
-# Required by:
-# requests==2.24.0
 chardet = 3.0.4
-
-# Required by:
-# requests==2.24.0
 urllib3 = 1.25.9
-
 pkgconfig = 1.5.1
 
 [networkcache]