Merge branch 'master' into master+ZODB4-wc2

* master: (56 commits)
  golang: v↑ go1.17  (1.17.3 -> 1.17.6)
  golang: v↑ go1.16  (1.16.10 -> 1.16.13)
  stack/erp5: Shorten haproxy socket path
  software/erp5/test: fix flaky test_activity_processing
  component/apache: version up 2.4.52, including security fixes.
  component/trafficserver: version up 9.1.1.
  stack/slapos: version up lxml 4.6.5, including security fixes.
  ninja: simplify configure step
  consul, packer, phantomjs: enable shared
  version up: zc.buildout 2.7.1+slapos015, slapos.recipe.build 0.53, slapos.recipe.cmmi 0.19
  version up: CMake 3.22.1
  component/userhosts: use github.com/figiel/hosts for compatibility with recent libc
  software/erp5/upgrade_test: backport new fixes in old software
  software/erp5/upgrade_test: skip software checks
  component/coturn: build with slapos libnsl
  component/apache-php: build with slapos libnsl
  repman: improve parameters used for cluster
  component/theia: version up 1.21.0
  software/theia: update python packages in python-language-server-requirements
  component/theia: update plugins with current versions
  ...

component/apache-php/buildout.cfg

@@ -23,6 +23,7 @@ extends =
   ../imagemagick/buildout.cfg
   ../icu/buildout.cfg
   ../openssl/buildout.cfg
+  ../libnsl/buildout.cfg
 
 [php-redis]
 recipe = slapos.recipe.cmmi
@@ -112,7 +113,7 @@ environment =
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${libzip:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
   CPPFLAGS=-I${libzip:location}/include
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -Wl,-rpath -Wl,${curl:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid -L${libzip:location}/lib -Wl,-rpath -Wl,${libzip:location}/lib -L${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${argon2:location}/lib/x86_64-linux-gnu -Wl,-rpath -Wl,${zstd:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib 
   TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
   HOME=${apache:location}
 

component/apache/buildout.cfg

@@ -39,9 +39,9 @@ configure-options =
 [apache]
 recipe = slapos.recipe.cmmi
 shared = true
-version = 2.4.51
+version = 2.4.52
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = d2793fc1c8cb8ba355cee877d1f2d46d
+md5sum = a94ae42b84309d5ef6e613ae825b92fa
 configure-options = --disable-static
                     --enable-authn-alias
                     --enable-bucketeer

component/chromedriver/buildout.cfg

@@ -15,37 +15,29 @@ parts =
 
 [chromedriver-wrapper-91]
 <= chromedriver-wrapper
-wrapper-name = chromedriver-91
-part = ${chromedriver-91:location}
+part = chromedriver-91
 
 [chromedriver-wrapper-2.41]
 <= chromedriver-wrapper
-wrapper-name = chromedriver-2.41
-part = ${chromedriver-2.41:location}
+part = chromedriver-2.41
 
 [chromedriver-wrapper]
 # generate a wrapper named ${:wrapper-name} setting $LD_LIBRARY_PATH
 # to run chromedriver installed from ${:part}
-wrapper-name = chromedriver
-part = ${chromedriver:location}
+part = chromedriver
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  chromedriver = options['part']
-  part = self.buildout[os.path.basename(chromedriver)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
-  exec ./chromedriver "$@"
-  """.format(
-    chromedriver,
-    ':'.join(part['library'].split()),
-    ':'.join(part['path'].split()),
-  ))
+  export LD_LIBRARY_PATH=%s
+  exec %s/chromedriver "$@"
+  """ % (':'.join(part['library'].split()), part['location']))
     os.fchmod(f.fileno(), 0o755)
 
 
@@ -67,18 +59,13 @@ md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
 [chromedriver-download]
 # Installs chromedriver ${version}.
 # This chromedriver is not usable directly, it needs a wrapper.
-version =
-md5sum-x86_64 =
-
-recipe = slapos.recipe.build
-x86_64-linux-gnu = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_linux64.zip ${:md5sum-x86_64}
+recipe = slapos.recipe.build:download-unpacked
+url = https://chromedriver.storage.googleapis.com/${:version}/chromedriver_${:_url}.zip
 library =
   ${nss:location}/lib
   ${nspr:location}/lib
   ${libX11:location}/lib
-path =
-install =
-  import os, shutil
-  extract_dir = self.extract(self.download(*options[multiarch()].split()))
-  os.mkdir(location)
-  shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
+
+[chromedriver-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux64
+md5sum = ${:md5sum-x86_64}

component/chromium/buildout.cfg

@@ -11,7 +11,6 @@
 # in the system.
 extends =
   ../alsa/buildout.cfg
-  ../coreutils/buildout.cfg
   ../cups/buildout.cfg
   ../dbus/buildout.cfg
   ../findutils/buildout.cfg
@@ -37,30 +36,30 @@ parts =
 
 
 [chromium-wrapper]
-# Install a chromium wrapper named ${:wrapper-name} in buildout's bin-directory,
-# wrapping chromium installed in ${:part}
+# Install a chromium wrapper named ${:part} in buildout's bin-directory,
+# wrapping chromium installed in [${:part}]
 # This [chromium-wrapper] installs the default chromium with name `chromium` and
 # can also be used as a macro to install under a different name.
-wrapper-name = chromium
-part = ${chromium:location}
+part = chromium
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  chromium = options['part']
-  part = self.buildout[os.path.basename(chromium)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
+  d=%s
   # this also needs a $FONTCONFIG_FILE set, otherwise system fonts will be used and if
   # no system fonts are available, chrome will refuse to start with this error:
   # FATAL:platform_font_linux.cc(83)] Check failed: InitDefaultFont(). Could not find the default font
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
-  exec ./chrome --disable-setuid-sandbox --no-sandbox --disable-gpu $@
-  """.format(
-    chromium,
+  export LD_LIBRARY_PATH=$d:%s
+  export PATH=%s:$PATH
+  exec $d/chrome --disable-setuid-sandbox --no-sandbox --disable-gpu "$@"
+  """ % (
+    part['location'],
     ':'.join(part['library'].split()),
     ':'.join(part['path'].split()),
   ))
@@ -68,13 +67,11 @@ install =
 
 [chromium-wrapper-91]
 <= chromium-wrapper
-wrapper-name = chromium-91
-part = ${chromium-91:location}
+part = chromium-91
 
 [chromium-wrapper-69]
 <= chromium-wrapper
-wrapper-name = chromium-69
-part = ${chromium-69:location}
+part = chromium-69
 
 
 [chromium]
@@ -101,24 +98,7 @@ generation-x86_64 = 1532051976706023
 [chromium-download]
 # macro to download a binary build of chromium and generate a
 # wrapper as chrome-slapos in the part directory
-
-# the full version tag
-version =
-
-# needs the following URL parts:
-# revision is from the "base position"
-revision_x86-64 =
-# generation is in the final download URL
-generation-x86_64 =
-# this is the md5sum of the downloaded archive
-md5sum-x86_64 =
-
-
-recipe = slapos.recipe.build
-
-x86_64-linux-gnu = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media ${:md5sum-x86_64}
-
-
+recipe = slapos.recipe.build:download-unpacked
 library =
   ${atk:location}/lib
   ${at-spi2-atk:location}/lib
@@ -166,12 +146,7 @@ library =
   ${zlib:location}/lib
 path =
   ${fontconfig:location}/bin
-install =
-  url, md5sum = self.options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), location)
-  # XXX adjust some permissions
-  import os
-  os.system("'${findutils:location}/bin/find' '%s' -type d -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
-  os.system("'${findutils:location}/bin/find' '%s' -type f -executable -exec '${coreutils:location}/bin/chmod' a+rx {} \\;" % location)
-  os.system("'${findutils:location}/bin/find' '%s' -type f -exec '${coreutils:location}/bin/chmod' a+r {} \\;" % location)
+
+[chromium-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+url = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${:revision_x86-64}%2Fchrome-linux.zip?generation=${:generation-x86_64}&alt=media
+md5sum = ${:md5sum-x86_64}

component/cmake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 [cmake]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://cmake.org/files/v3.21/cmake-3.21.3.tar.gz
-md5sum = c0feb5855604f68b09bdb3acb623619e
+url = https://cmake.org/files/v3.22/cmake-3.22.1.tar.gz
+md5sum = 83802be08c114bb34729300206488321
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/consul/buildout.cfg

@@ -4,7 +4,14 @@ parts =
 
 [consul]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 version = 0.8.3
-base = https://releases.hashicorp.com/consul/${:version}/consul_${:version}
-i386-linux-gnu   = ${:base}_linux_386.zip    dfdc0eedd79baab7e6bc56c1582fd02e
-x86_64-linux-gnu = ${:base}_linux_amd64.zip  d6bc0898ea37ae2198370a9e1978d1bb
+url = https://releases.hashicorp.com/consul/${:version}/consul_${:version}_${:_url}.zip
+
+[consul:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux_386
+md5sum = dfdc0eedd79baab7e6bc56c1582fd02e
+
+[consul:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux_amd64
+md5sum = d6bc0898ea37ae2198370a9e1978d1bb

component/coturn/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../sqlite3/buildout.cfg
   ../openssl/buildout.cfg
   ../libevent/buildout.cfg
+  ../libnsl/buildout.cfg
 parts = coturn
 
 [coturn]
@@ -13,4 +14,4 @@ environment =
   PATH=${sqlite3:location}/bin:%(PATH)s
   CPPFLAGS=-I${openssl:location}/include -I${sqlite3:location}/include -I${libevent2:location}/include
   CFLAGS=-I${libevent2:location}/include
-  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath -Wl,${sqlite3:location}/lib -L${libevent2:location}/lib -Wl,-rpath -Wl,${libevent2:location}/lib -L${libnsl:location}/lib -Wl,-rpath -Wl,${libnsl:location}/lib

component/cpio/963304-remove-superfluous-declaration-of-program_name

+author	Sergey Poznyakoff <gray@gnu.org>
+
+https://git.savannah.gnu.org/cgit/cpio.git/commit/?id=641d3f489cf6238bb916368d4ba0d9325a235afb
+
+* src/global.c: Remove superfluous declaration of program_name
+
+diff --git a/src/global.c b/src/global.c
+index fb3abe9..acf92bc 100644
+--- a/src/global.c
++++ b/src/global.c
+@@ -184,9 +184,6 @@ unsigned int warn_option = 0;
+ /* Extract to standard output? */
+ bool to_stdout_option = false;
+ 
+-/* The name this program was run with.  */
+-char *program_name;
+-
+ /* A pointer to either lstat or stat, depending on whether
+    dereferencing of symlinks is done for input files.  */
+ int (*xstat) ();

component/cpio/buildout.cfg

 # GNU cpio copies files into or out of a cpio or tar archive. The archive can be another file on the disk, a magnetic tape, or a pipe.
 [buildout]
+extends =
+  ../patch/buildout.cfg
 parts = cpio
 
 [cpio]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/cpio/cpio-2.12.tar.bz2
-md5sum = 93eea9f07c0058c097891c73e4955456
+url = https://ftp.gnu.org/gnu/cpio/cpio-2.13.tar.bz2
+md5sum = f3438e672e3fa273a7dc26339dd1eed6
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/963304-remove-superfluous-declaration-of-program_name#1aadb39448eedaf49d03cf45f9d78ea4
+environment =
+  PATH=${patch:location}/bin:%(PATH)s

component/firefox/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../alsa/buildout.cfg
   ../dbus/buildout.cfg
   ../fontconfig/buildout.cfg
+  ../fonts/buildout.cfg
   ../gtk-2/buildout.cfg
   ../gtk-3/buildout.cfg
   ../libffi/buildout.cfg
@@ -22,59 +23,52 @@ parts =
 
 
 [firefox-wrapper]
-# Install a firefox wrapper names ${:wrapper-name} in buildout's bin-directory,
-# wrapping firefox installed in ${:part}
+# Install a firefox wrapper named ${:part} in buildout's bin-directory,
+# wrapping firefox installed in [${:part}]
 # This [firefox-wrapper] installs the default firefox with name `firefox` and
 # can also be used as a macro to install under a different name.
-wrapper-name = firefox
-part = ${firefox:location}
+part = firefox
 
 recipe = slapos.recipe.build
-location = ${buildout:bin-directory}/${:wrapper-name}
+location = ${buildout:bin-directory}/${:part}
+fonts-conf = ${firefox-default-fonts-conf:rendered}
+init =
+  self.buildout[options['part']]
 install =
   import os
-  firefox = options['part']
-  part = self.buildout[os.path.basename(firefox)]
+  part = self.buildout[options['part']]
   with open(location, 'w') as f:
     f.write("""#!/bin/sh -e
-  cd {}
-  export LD_LIBRARY_PATH=$PWD:{}
-  export PATH={}:$PATH
+  d=%s
+  export LD_LIBRARY_PATH=$d:%s
+  export PATH=%s:$PATH
   # BBB use a default fonts.conf for compatibility, but it's software instance
   # responsability to build a fonts.conf with the fonts they want.
-  [ -z $FONTCONFIG_FILE ] && export FONTCONFIG_FILE=${firefox-default-fonts-conf:rendered}
-  exec ./firefox "$@"
-  """.format(
-    firefox,
+  [ "$FONTCONFIG_FILE" ] || export FONTCONFIG_FILE=%s
+  exec $d/firefox "$@"
+  """ % (
+    part['location'],
     ':'.join(part['library'].split()),
     ':'.join(part['path'].split()),
+    options['fonts-conf'],
   ))
     os.fchmod(f.fileno(), 0o755)
 
 [firefox-wrapper-78]
 <= firefox-wrapper
-wrapper-name = firefox-78
-part = ${firefox-78:location}
+part = firefox-78
 
 [firefox-wrapper-68]
 <= firefox-wrapper
-wrapper-name = firefox-68
-part = ${firefox-68:location}
+part = firefox-68
 
 [firefox-wrapper-60]
 <= firefox-wrapper
-wrapper-name = firefox-60
-part = ${firefox-60:location}
+part = firefox-60
 
 [firefox-wrapper-52]
 <= firefox-wrapper
-wrapper-name = firefox-52
-part = ${firefox-52:location}
-
-[firefox-wrapper-51]
-<= firefox-wrapper
-wrapper-name = firefox-51
-part = ${firefox-51:location}
+part = firefox-52
 
 [firefox-default-fonts-conf]
 recipe = slapos.recipe.template:jinja2
@@ -124,20 +118,12 @@ version = 52.9.0esr
 i686-md5sum = 9aa18888b7812670208490609d75c9bc
 x86_64-md5sum = 9336d70f45070c743d08e5473b783a7a
 
-[firefox-51]
-<= firefox-download
-version = 51.0.1
-i686-md5sum = 9a5b67e9d759a1e4df004294a24b2b43
-x86_64-md5sum = bd93f2652d1d90d59ae462439a93c85f
-
-
 [firefox-download]
 recipe = slapos.recipe.build
 slapos_promise =
   file:firefox
 
-i386-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 ${:i686-md5sum}
-x86_64-linux-gnu = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ${:x86_64-md5sum}
+url = https://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/${:_url}/en-US/firefox-${:version}.tar.bz2
 
 library =
   ${alsa:location}/lib
@@ -185,12 +171,19 @@ path =
   ${fontconfig:location}/bin
 
 install =
-  url, md5sum = options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  self.copyTree(guessworkdir(extract_dir), location)
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   ${:post-install}
 post-install =
 
+[firefox-download:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-i686
+md5sum = ${:i686-md5sum}
+
+[firefox-download:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-x86_64
+md5sum = ${:x86_64-md5sum}
+
+
 [geckodriver]
 # Current geckodriver installed as ${buildout:bin-directory}/geckodriver
 <= geckodriver-0.24.0
@@ -207,47 +200,26 @@ version = 0.22.0
 i686-md5sum = 6de7544753fda56fbaa8382dcac99aaa
 x86_64-md5sum = 81746200ce5841e00cabf3b8ea7db542
 
-[geckodriver-0.21.0]
-<= geckodriver-base
-version = 0.21.0
-i686-md5sum = 9fc1657dd1b94272d0cdb3b29ca80f79
-x86_64-md5sum = 6bc36d4fd4975e296cdb3fa3c5e26a41
-
-[geckodriver-0.19.0]
-<= geckodriver-base
-version = 0.19.0
-i686-md5sum = 07cd383c8aef8ea5ef194a506141afd6
-x86_64-md5sum = ca6935a72fd0527d15a78a17a35e56e8
-
-[geckodriver-0.17.0]
-<= geckodriver-base
-version = 0.17.0
-i686-md5sum = 79b1a158f96d29942a111c0905f1c807
-x86_64-md5sum = be18faeea6e7db9db6990d8667e2298f
-
 [geckodriver-0.16.1]
 <= geckodriver-base
 version = 0.16.1
 i686-md5sum = not not on github
 x86_64-md5sum = 57dfd55d4759d9878eb75b4c0123d00c
 
-[geckodriver-0.14.0]
-<= geckodriver-base
-version = 0.14.0
-i686-md5sum = b5836f5a944fe9f3ed1a67c7b342c6a7
-x86_64-md5sum = 4a185d3179862a35104603b9274452e7
-
-
 [geckodriver-base]
 # Installs geckodriver ${version}
 recipe = slapos.recipe.build
+url = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-${:_url}.tar.gz
 location = ${buildout:bin-directory}/${:_buildout_section_name_}
+install =
+  import os, shutil
+  extract_dir = self.extract(self.download(options['url'], options['md5sum']))
+  shutil.copy(os.path.join(extract_dir, 'geckodriver'), location)
 
-i386-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux32.tar.gz ${:i686-md5sum}
-x86_64-linux-gnu = https://github.com/mozilla/geckodriver/releases/download/v${:version}/geckodriver-v${:version}-linux64.tar.gz ${:x86_64-md5sum}
+[geckodriver-base:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux32
+md5sum = ${:i686-md5sum}
 
-install =
-  import shutil
-  url, md5sum = options[multiarch()].split()
-  extract_dir = self.extract(self.download(url, md5sum))
-  shutil.copy(extract_dir + '/geckodriver', location)
+[geckodriver-base:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux64
+md5sum = ${:x86_64-md5sum}

component/golang/buildout.cfg

@@ -59,8 +59,8 @@ environment-extra =
 
 [golang1.16]
 <= golang-common
-url = https://golang.org/dl/go1.16.10.src.tar.gz
-md5sum = 49f0a54f0bdcba297bac194d8dafe431
+url = https://golang.org/dl/go1.16.13.src.tar.gz
+md5sum = 1c076f952d9af57590a36fa7d36f695a
 
 # go1.16 needs go1.4 to bootstrap
 environment-extra =
@@ -68,8 +68,8 @@ environment-extra =
 
 [golang1.17]
 <= golang-common
-url = https://golang.org/dl/go1.17.3.src.tar.gz
-md5sum = 3ea82e5966275f405f0db4f52511bb6e
+url = https://golang.org/dl/go1.17.6.src.tar.gz
+md5sum = dc57f93f323e9f8189e5ffc1f223e346
 
 # go1.17 needs go1.4 to bootstrap
 environment-extra =

component/headless-chromium/buildout.cfg

@@ -45,9 +45,9 @@ gclient-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 # called "src".
 name = src
 
-# There's nothing special about version 92.0.4515.107. It just happened
-# to be the current Chromium stable version at the time of writing.
-version = 92.0.4515.107
+# 96.0.4664.129 version is the latest stable version in December 2021.
+# Note that we need a version compiling without python2
+version = 96.0.4664.129
 
 
 [headless-chromium]

component/libnsl/buildout.cfg

+[buildout]
+extends =
+  ../xz-utils/buildout.cfg
+  ../libtirpc/buildout.cfg
+
+[libnsl]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/thkukuk/libnsl/releases/download/v1.3.0/libnsl-1.3.0.tar.xz
+md5sum = 9214f674bd0c2bcfdd6c1da0cadb061f
+make-options = 
+  CFLAGS=-I${libtirpc:location}/include/tirpc
+  LDFLAGS='-L${libtirpc:location}/lib -Wl,-rpath=${libtirpc:location}/lib -ltirpc'
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/libreoffice-bin/buildout.cfg

@@ -8,26 +8,15 @@ parts =
 
 [libreoffice-bin]
 recipe = slapos.recipe.build
-# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
 version = 5.2.4.2
-url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
-# supported architectures md5sums
-md5sum_x86 = 7a0b33a2d18f06143258c428c32de213
-md5sum_x86-64 = cbea6cd17063b5bcbe0cb32f7819f0cf
+url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/${:_url1}/LibreOffice_${:version}_${:_url2}_rpm.tar.gz
 
 # where office code can be found?
 officedir = libreoffice5.2
 install =
   import os
   import sys
-  arch = {
-    'i386-linux-gnu': ('x86', 'x86'),
-    'x86_64-linux-gnu': ('x86_64', 'x86-64'),
-  }[multiarch()]
-  url = options['url'] % arch
-  md5sum = options['md5sum_' + arch[1]]
-  extract_dir = self.extract(self.download(url, md5sum))
-  workdir = guessworkdir(extract_dir)
+  workdir = guessworkdir(self.extract(self.download()))
   storagedir = os.path.join(workdir, 'storage')
   os.mkdir(storagedir)
   rpmsdir = os.path.join(workdir, 'RPMS')
@@ -48,3 +37,13 @@ install =
 # helper binaries
 cpio = ${cpio:location}/bin/cpio
 rpm2cpio = ${rpm2cpio:target}
+
+[libreoffice-bin:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url1 = x86
+_url2 = Linux_x86
+md5sum = 7a0b33a2d18f06143258c428c32de213
+
+[libreoffice-bin:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url1 = x86_64
+_url2 = Linux_x86-64
+md5sum = cbea6cd17063b5bcbe0cb32f7819f0cf

component/libtirpc/buildout.cfg

+[buildout]
+parts = libtirpc
+
+[libtirpc]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://downloads.sourceforge.net/libtirpc/libtirpc-1.3.2.tar.bz2
+md5sum = cf4ca51f3fc401bea61c702c69171ab0
+configure-options = --disable-gssapi

component/nginx/buildout.cfg

@@ -61,8 +61,8 @@ mime = ${nginx:location}/conf/mime.types
 [nginx-push-stream-module]
 recipe = slapos.recipe.build:download-unpacked
 shared = true
-url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.4.0.tar.gz
-md5sum = d9cba621b8739e13bdb5e02b9425f205
+url = https://github.com/wandenberg/nginx-push-stream-module/archive/0.5.5.tar.gz
+md5sum = 9bb5237a93c957130aaf7033ed3c0989
 
 [nginx-push-stream]
 <= nginx-common

component/ninja/buildout.cfg

 [buildout]
-extends =
-  ../defaults.cfg
 parts =
   ninja
 
@@ -10,7 +8,7 @@ shared = true
 version = 1.10.2
 url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
 md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
-configure-command = ${python:executable} configure.py
+configure-command = ${buildout:executable} configure.py
 configure-options = --bootstrap --verbose
 make-binary = true
 post-install =

component/p7zip/buildout.cfg

 [buildout]
+extends =
+  ../patch/buildout.cfg
 parts =
   p7zip
 
 [p7zip]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
-version = 9.38.1
-md5sum = 6cba8402ccab2370d3b70c5e28b3d651
+url = https://downloads.sourceforge.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
+version = 16.02
+md5sum = a0128d661cfe7cc8c121e73519c54fbf
+patch-binary = ${patch:location}/bin/patch
+patch-options = -p1
+patch-base = https://sources.debian.org/data/main/p/p7zip/16.02%2Bdfsg-8/debian/patches/
+patches =
+  ${:patch-base}/12-CVE-2016-9296.patch#f0f5dc36d97d9a71a6da3adba717fbf2
+  ${:patch-base}/13-CVE-2017-17969.patch#8399bdc913cf8cf8194ecdb5487f48d9
+  ${:patch-base}/14-Fix-g%2B%2B-warning.patch#9db8fbfbc8d624669c095240621460bd
+  ${:patch-base}/15-Fix-FTBFS-gcc10.patch#50ebfd79c715d8b5d649e57a1da2ce34
 configure-command = rm -r DOC
-make-binary =
-make-options =
-make-targets = make -j1 7z install
-  DEST_HOME=@@LOCATION@@
+make-targets = 7z
+post-install = make install DEST_HOME=%(location)s

component/packer/buildout.cfg

@@ -4,7 +4,14 @@ parts =
 
 [packer]
 recipe = slapos.recipe.build:download-unpacked
+shared = true
 version = 0.7.5
-base = https://releases.hashicorp.com/packer/${:version}/packer_${:version}
-i386-linux-gnu   = ${:base}_linux_386.zip    a545108a0ccfde7c1e74de6c4e6fdded
-x86_64-linux-gnu = ${:base}_linux_amd64.zip  f343d709b84db494e8d6ec38259aa4a6
+url = https://releases.hashicorp.com/packer/${:version}/packer_${:version}_${:_url}.zip
+
+[packer:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux_386
+md5sum = a545108a0ccfde7c1e74de6c4e6fdded
+
+[packer:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux_amd64
+md5sum = f343d709b84db494e8d6ec38259aa4a6

component/phantomjs/buildout.cfg

@@ -7,23 +7,36 @@ parts =
 
 [phantomjs]
 recipe = slapos.recipe.build
+shared = true
 slapos_promise =
   file:phantomjs-slapos
 
-base = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux
-i386-linux-gnu   = ${:base}-i686.tar.bz2   9c1426eef5b04679d65198b1bdd6ef88
-x86_64-linux-gnu = ${:base}-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
-
+url = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-${:_url}.tar.bz2
 install =
   import os
-  extract_dir = self.extract(self.download(*options[multiarch()].split()))
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(workdir, location)
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   with open(os.path.join(location, "phantomjs-slapos"), 'w') as wrapper:
     wrapper.write("""#!/bin/sh -e
-  cd %(location)s
-  export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
-  export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs "$@"
-  """ % options)
+  export LD_LIBRARY_PATH=%s
+  export PATH=%s:$PATH
+  exec %s/bin/phantomjs "$@"
+  """ % (
+    ':'.join(options['library'].split()),
+    ':'.join(options['path'].split()),
+    location,
+  ))
     os.fchmod(wrapper.fileno(), 0o755)
+library =
+  ${freetype:location}/lib
+  ${fontconfig:location}/lib
+  ${libexpat:location}/lib
+path =
+  ${fontconfig:location}/bin
+
+[phantomjs:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-i686
+md5sum = 9c1426eef5b04679d65198b1bdd6ef88
+
+[phantomjs:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-x86_64
+md5sum = f278996c3edd0e8d8ec4893807f27d71

component/postfix/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../patch/buildout.cfg
   ../pcre/buildout.cfg
   ../cyrus-sasl/buildout.cfg
+  ../libnsl/buildout.cfg
   ../m4/buildout.cfg
 
 [postfix]
@@ -20,7 +21,9 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/noroot.patch#05fc6333e05576ea8e5a49f27a6ef951
 configure-command = make
-configure-options = makefiles CCARGS='-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl' AUXLIBS='-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib'
+configure-options = makefiles CCARGS=${:configure-options-CCARGS} AUXLIBS=${:configure-options-AUXLIBS}
+configure-options-CCARGS = '-DUSE_SASL_AUTH -DUSE_CYRUS_SASL -DUSE_TLS -DHAS_PCRE -DHAS_DB -I${libdb:location}/include -I${pcre:location}/include -I${openssl:location}/include -I${cyrus-sasl:location}/include/sasl -I${libnsl:location}/include'
+configure-options-AUXLIBS = '-L${openssl:location}/lib -L${pcre:location}/lib -L${libdb:location}/lib -L${cyrus-sasl:location}/lib -L${libtirpc:location}/lib -L${libnsl:location}/lib -lssl -lpcre -ldb -lcrypto -lsasl2 -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${libdb:location}/lib -Wl,-rpath=${cyrus-sasl:location}/lib  -Wl,-rpath=${libnsl:location}/lib' 
 make-targets = non-interactive-package install_root=${:location}
 environment =
   PATH=${patch:location}/bin:${m4:location}/bin:%(PATH)s

component/python-2.7/buildout.cfg

@@ -42,6 +42,7 @@ patches =
   ${:_profile_base_location_}/pytracemalloc_pep445.patch#9f3145817afa2b7fad801fde8447e396
   ${:_profile_base_location_}/disabled_module_list.patch#e038a8016475574c810cbaaf0e42f4ac
   ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
+  ${:_profile_base_location_}/py27-subproc-closefds-fast.patch#e495e44491694a8972da11739206f2e6
 url =
   http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/python-2.7/py27-subproc-closefds-fast.patch

+From 9a79fa64c6298998d04a5ae6d699222a93cd0cb8 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Fri, 24 Dec 2021 13:55:08 +0300
+Subject: [PATCH] [py27] subprocess: Use /proc/self/fd/ to know opened file
+ descriptors
+
+... to avoid O(MAXFD) slowdown on closefds=True.
+
+This is semantic backport of what Python3 does.
+---
+ Lib/subprocess.py | 23 ++++++++++++++++++++++-
+ 1 file changed, 22 insertions(+), 1 deletion(-)
+
+diff --git a/Lib/subprocess.py b/Lib/subprocess.py
+index 1f2da0ffbe8..300557220ac 100644
+--- a/Lib/subprocess.py
++++ b/Lib/subprocess.py
+@@ -869,7 +869,28 @@ def pipe_cloexec(self):
+ 
+ 
+         def _close_fds(self, but):
+-            if hasattr(os, 'closerange'):
++            # try to retrieve list of opened file descriptors to avoid being slow on large MAXFD
++            # https://www.erp5.com/project_section/vifib/forum/python2--subprocess.Popen-is-slow-when-close_fds-True-and-nofile-ulimit-is-high-PO01sREVXn
++            # https://bugs.python.org/issue8052
++            # https://github.com/python/cpython/commit/8facece99a59
++            fdv = None
++            try:
++                fdv = os.listdir('/proc/self/fd')
++            except OSError as e:
++                if e.errno != errno.ENOENT:
++                    raise
++            if fdv is not None:
++                for fd_ in fdv:
++                    fd = int(fd_)
++                    if (0 <= fd <= 2) or fd == but:
++                        continue
++                    try:
++                        os.close(fd)
++                    except Exception: # e.g. not KeyboardInterrupt nor SystemExit
++                        pass
++
++            # fallback to closing all file descriptors in 3..MAXFD range
++            elif hasattr(os, 'closerange'):
+                 os.closerange(3, but)
+                 os.closerange(but + 1, MAXFD)
+             else:
+-- 
+2.30.2
+

component/replication-manager/buildout.cfg

@@ -28,5 +28,6 @@ command = set -e
 <= go-git-package
 go.importpath = github.com/signal18/replication-manager
 repository =  https://github.com/signal18/replication-manager
-branch = 2.1
-revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
+#branch = 2.1
+branch = develop
+revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file

component/theia/buildout.hash.cfg

@@ -19,4 +19,4 @@ md5sum = 8157c22134200bd862a07c6521ebf799
 
 [yarn.lock]
 _update_hash_filename_ = yarn.lock
-md5sum = 067d2db611b21f77885f3adfd7f81453
+md5sum = e0341b7a715cc757a671aef902e6767d

component/theia/download-plugins.cfg

 # This file is automatically generated from generate_download_plugins_cfg.py
 # Do not edit directly.
 [theia-download-plugins]
-urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.54.3/file/vscode.bat-1.54.3.vsix 77de5b930e0b9baa90059abb1711bd22
-	vscode-clojure https://open-vsx.org/api/vscode/clojure/1.54.3/file/vscode.clojure-1.54.3.vsix f89e90ad2e808e58bc1b4d24fbbf680f
-	vscode-coffeescript https://open-vsx.org/api/vscode/coffeescript/1.54.3/file/vscode.coffeescript-1.54.3.vsix 0800e4a04e9a0cfa4810c0461f26ab00
-	vscode-configuration-editing https://open-vsx.org/api/vscode/configuration-editing/1.54.3/file/vscode.configuration-editing-1.54.3.vsix 44c5f0439733f93813482138656001a0
-	vscode-cpp https://open-vsx.org/api/vscode/cpp/1.54.3/file/vscode.cpp-1.54.3.vsix 851461a32d8a69b817d85f82bf9b4035
-	vscode-csharp https://open-vsx.org/api/vscode/csharp/1.54.3/file/vscode.csharp-1.54.3.vsix e9f69d28870a3ba1fa8cf30d6ceaabcf
-	vscode-css https://open-vsx.org/api/vscode/css/1.54.3/file/vscode.css-1.54.3.vsix 8539c80c26897a95e8a55d1640c205dc
-	vscode-css-language-features https://open-vsx.org/api/vscode/css-language-features/1.54.3/file/vscode.css-language-features-1.54.3.vsix 530806dfef61b6936bce7dccd79a9383
-	vscode-debug-auto-launch https://open-vsx.org/api/vscode/debug-auto-launch/1.54.3/file/vscode.debug-auto-launch-1.54.3.vsix 5840891614f42fd22a7adcb812a2a26b
-	vscode-docker https://open-vsx.org/api/vscode/docker/1.54.3/file/vscode.docker-1.54.3.vsix a4e3cdd03833fe6bf3f5045fc5be4433
-	vscode-emmet https://open-vsx.org/api/vscode/emmet/1.54.3/file/vscode.emmet-1.54.3.vsix 5786c95921794ed3188083a7f804fa29
-	vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.54.3/file/vscode.fsharp-1.54.3.vsix 028f4ec9593533ce38603032b3f2821e
-	vscode-git https://open-vsx.org/api/vscode/git/1.54.3/file/vscode.git-1.54.3.vsix da2f2c0626b4a254a660ae20b0c97611
-	vscode-go https://open-vsx.org/api/vscode/go/1.54.3/file/vscode.go-1.54.3.vsix 9f095a75e4137079351150722d091256
-	vscode-groovy https://open-vsx.org/api/vscode/groovy/1.54.3/file/vscode.groovy-1.54.3.vsix 7ae14c40786311fda29059178f30c310
-	vscode-grunt https://open-vsx.org/api/vscode/grunt/1.54.3/file/vscode.grunt-1.54.3.vsix f7894259f1fa60e939ff2d9b49138a02
-	vscode-gulp https://open-vsx.org/api/vscode/gulp/1.54.3/file/vscode.gulp-1.54.3.vsix cc5c570dda3b69ba3e1e1ffbe6f3a0c8
-	vscode-handlebars https://open-vsx.org/api/vscode/handlebars/1.54.3/file/vscode.handlebars-1.54.3.vsix ea433237fb8940ba2ff075253f58b9a6
-	vscode-hlsl https://open-vsx.org/api/vscode/hlsl/1.54.3/file/vscode.hlsl-1.54.3.vsix d111abc12ad0e9f6d15e3b3c522e3072
-	vscode-html https://open-vsx.org/api/vscode/html/1.54.3/file/vscode.html-1.54.3.vsix f8fa8f2e54ca8f3d3d64c60cb617bfde
+urls = vscode-bat https://open-vsx.org/api/vscode/bat/1.62.3/file/vscode.bat-1.62.3.vsix 61e32be47fbd5a88907bf7b2e8498621
+	vscode-clojure https://open-vsx.org/api/vscode/clojure/1.62.3/file/vscode.clojure-1.62.3.vsix 9b44f96739dcac6d5117859fd72fbdba
+	vscode-coffeescript https://open-vsx.org/api/vscode/coffeescript/1.62.3/file/vscode.coffeescript-1.62.3.vsix 54fa3459253d7817535c4151f704c84c
+	vscode-configuration-editing https://open-vsx.org/api/vscode/configuration-editing/1.62.3/file/vscode.configuration-editing-1.62.3.vsix 941b13b5f15a4b4de122f68987b79e0c
+	vscode-cpp https://open-vsx.org/api/vscode/cpp/1.62.3/file/vscode.cpp-1.62.3.vsix aefd964c2bddc59c87f14e76aee959fa
+	vscode-csharp https://open-vsx.org/api/vscode/csharp/1.62.3/file/vscode.csharp-1.62.3.vsix 38cde5827cf2f88c8a2988f559a5a02f
+	vscode-css https://open-vsx.org/api/vscode/css/1.62.3/file/vscode.css-1.62.3.vsix 56ba10d5d7d63140b7b68c47be090c72
+	vscode-css-language-features https://open-vsx.org/api/vscode/css-language-features/1.62.3/file/vscode.css-language-features-1.62.3.vsix 9bc4e515a491a9e4749084aa1753ddac
+	vscode-debug-auto-launch https://open-vsx.org/api/vscode/debug-auto-launch/1.62.3/file/vscode.debug-auto-launch-1.62.3.vsix 9236132f864d689e839b51bb52d28def
+	vscode-docker https://open-vsx.org/api/vscode/docker/1.62.3/file/vscode.docker-1.62.3.vsix f27713e5ca6d1d08cb4b58668d59db66
+	vscode-emmet https://open-vsx.org/api/vscode/emmet/1.62.3/file/vscode.emmet-1.62.3.vsix 5ad6c366cd49cb8786eec28908fabf1c
+	vscode-fsharp https://open-vsx.org/api/vscode/fsharp/1.62.3/file/vscode.fsharp-1.62.3.vsix 9450d03e78d9dc671b8068c9a73e5570
+	vscode-git https://open-vsx.org/api/vscode/git/1.62.3/file/vscode.git-1.62.3.vsix 76390cc667080a62bdebc7302134be08
+	vscode-go https://open-vsx.org/api/vscode/go/1.62.3/file/vscode.go-1.62.3.vsix d7b620cd630dfd126100a1c7366a665b
+	vscode-groovy https://open-vsx.org/api/vscode/groovy/1.62.3/file/vscode.groovy-1.62.3.vsix ed8b4d51bc41d68cfbe2ac72f6884d2d
+	vscode-grunt https://open-vsx.org/api/vscode/grunt/1.62.3/file/vscode.grunt-1.62.3.vsix 96800d71f8a3842d0413684550bc5e0f
+	vscode-gulp https://open-vsx.org/api/vscode/gulp/1.62.3/file/vscode.gulp-1.62.3.vsix e50eb3f086fa2053249c0d817a208f57
+	vscode-handlebars https://open-vsx.org/api/vscode/handlebars/1.62.3/file/vscode.handlebars-1.62.3.vsix b762f43fff8bd5629dd9dc6454c3045b
+	vscode-hlsl https://open-vsx.org/api/vscode/hlsl/1.62.3/file/vscode.hlsl-1.62.3.vsix 899e646896216df4ed2d46e8f17812c9
+	vscode-html https://open-vsx.org/api/vscode/html/1.62.3/file/vscode.html-1.62.3.vsix a3448adf65e4158a250b37d470ce396c
 	vscode-html-language-features https://open-vsx.org/api/vscode/html-language-features/1.52.1/file/vscode.html-language-features-1.52.1.vsix a83619f2924ec58713691e76f17c7fb6
-	vscode-ini https://open-vsx.org/api/vscode/ini/1.54.3/file/vscode.ini-1.54.3.vsix 3f758e07aad99c4ba15fbc59c0807e2d
-	vscode-jake https://open-vsx.org/api/vscode/jake/1.54.3/file/vscode.jake-1.54.3.vsix 60b90d9620e0b85d8effe0a5265079f0
-	vscode-java https://open-vsx.org/api/vscode/java/1.54.3/file/vscode.java-1.54.3.vsix 9b933657f3e5eedfe0247006a268ac92
-	vscode-javascript https://open-vsx.org/api/vscode/javascript/1.54.3/file/vscode.javascript-1.54.3.vsix 32860b0b1bea5b2fc15b4d7eb5eaf630
-	ms-vscode-js-debug https://open-vsx.org/api/ms-vscode/js-debug/1.52.2/file/ms-vscode.js-debug-1.52.2.vsix f0f0c11c9c1855b6d3c87b230790c949
-	vscode-json https://open-vsx.org/api/vscode/json/1.54.3/file/vscode.json-1.54.3.vsix 7dda28a6cdcf703d0af938ffd2a60a56
+	vscode-ini https://open-vsx.org/api/vscode/ini/1.62.3/file/vscode.ini-1.62.3.vsix ee71130f95861b61d92d644c3cd8ded7
+	vscode-jake https://open-vsx.org/api/vscode/jake/1.62.3/file/vscode.jake-1.62.3.vsix fb7d81d41eaa04d53183bd6163e83ed3
+	vscode-java https://open-vsx.org/api/vscode/java/1.62.3/file/vscode.java-1.62.3.vsix d8239ae11719762e8b8152010bca50c9
+	vscode-javascript https://open-vsx.org/api/vscode/javascript/1.62.3/file/vscode.javascript-1.62.3.vsix 6f5babbbd2f437f2e34df2feb97ce595
+	ms-vscode-js-debug https://open-vsx.org/api/ms-vscode/js-debug/1.61.0/file/ms-vscode.js-debug-1.61.0.vsix 6457840c5c8d43e5b7a39fabf72b7367
+	vscode-json https://open-vsx.org/api/vscode/json/1.62.3/file/vscode.json-1.62.3.vsix 9e0e7b25a2d4c5df88ea3a75cf60372b
 	vscode-json-language-features https://open-vsx.org/api/vscode/json-language-features/1.45.1/file/vscode.json-language-features-1.45.1.vsix b7aa9d96d67792dedf9c2558880f38c0
-	vscode-less https://open-vsx.org/api/vscode/less/1.54.3/file/vscode.less-1.54.3.vsix 1edc8fca796c520d7e1aa89c069070a4
-	vscode-log https://open-vsx.org/api/vscode/log/1.54.3/file/vscode.log-1.54.3.vsix 9552f796a8f2ca38950d0f25df21f9cb
-	vscode-lua https://open-vsx.org/api/vscode/lua/1.54.3/file/vscode.lua-1.54.3.vsix ed6a4a09c8505e0f1437d2530c69bfe5
-	vscode-make https://open-vsx.org/api/vscode/make/1.54.3/file/vscode.make-1.54.3.vsix 996b8d1a2292318d177ae4951b67c990
-	vscode-markdown https://open-vsx.org/api/vscode/markdown/1.54.3/file/vscode.markdown-1.54.3.vsix 4188ca7c2bda0037e88f24cc443a24f8
-	vscode-markdown-language-features https://open-vsx.org/api/vscode/markdown-language-features/1.39.2/file/vscode.markdown-language-features-1.39.2.vsix 9ab6e372796b3bacede80fcbcfd35640
-	vscode-merge-conflict https://open-vsx.org/api/vscode/merge-conflict/1.54.3/file/vscode.merge-conflict-1.54.3.vsix 554eeb664788a7b5b80ce1da4795f0b1
-	vscode-npm https://open-vsx.org/api/vscode/npm/1.54.3/file/vscode.npm-1.54.3.vsix e6eeeedebf3b6986f1d7043c94807d01
-	ms-vscode-node-debug https://open-vsx.org/api/ms-vscode/node-debug/1.44.8/file/ms-vscode.node-debug-1.44.8.vsix 367bac016859084e63acf792ed4a1aeb
-	ms-vscode-node-debug2 https://open-vsx.org/api/ms-vscode/node-debug2/1.42.5/file/ms-vscode.node-debug2-1.42.5.vsix 7882d7d90d18820be17aa5f3e12e71e1
-	vscode-objective-c https://open-vsx.org/api/vscode/objective-c/1.54.3/file/vscode.objective-c-1.54.3.vsix 6b3e02fc4ef1ddefa7e3eabef5bf5906
-	vscode-perl https://open-vsx.org/api/vscode/perl/1.54.3/file/vscode.perl-1.54.3.vsix 6f3bc3e5b42e34e23fa287db2548b7c8
-	vscode-powershell https://open-vsx.org/api/vscode/powershell/1.54.3/file/vscode.powershell-1.54.3.vsix da887b8a031620f77676f529a7403580
-	vscode-pug https://open-vsx.org/api/vscode/pug/1.54.3/file/vscode.pug-1.54.3.vsix 8144700e4a33e99fa9b669d1254f8836
-	vscode-python https://open-vsx.org/api/vscode/python/1.54.3/file/vscode.python-1.54.3.vsix cea58111aad00d216f3643d9a180e1e5
-	vscode-r https://open-vsx.org/api/vscode/r/1.54.3/file/vscode.r-1.54.3.vsix fa4452b7777db74cb80a037b84a594a4
-	vscode-razor https://open-vsx.org/api/vscode/razor/1.54.3/file/vscode.razor-1.54.3.vsix da9e67b2f3530658778ac3180725e483
-	vscode-ruby https://open-vsx.org/api/vscode/ruby/1.54.3/file/vscode.ruby-1.54.3.vsix 9784e2d9ee737ea31192f60c97659832
-	vscode-rust https://open-vsx.org/api/vscode/rust/1.54.3/file/vscode.rust-1.54.3.vsix 8763f9e271841772e096a9fbbe4e342d
-	vscode-scss https://open-vsx.org/api/vscode/scss/1.54.3/file/vscode.scss-1.54.3.vsix 89d19984246b7c708738534648bb9fd0
-	vscode-shaderlab https://open-vsx.org/api/vscode/shaderlab/1.54.3/file/vscode.shaderlab-1.54.3.vsix 6cda4de8ff8c5d84b53dd8a234ae4d4f
-	vscode-shellscript https://open-vsx.org/api/vscode/shellscript/1.54.3/file/vscode.shellscript-1.54.3.vsix 32ec524b436a5c8a9d9be7c452112552
-	vscode-sql https://open-vsx.org/api/vscode/sql/1.54.3/file/vscode.sql-1.54.3.vsix bd448cac364ea09b4593db7efca625d3
-	vscode-swift https://open-vsx.org/api/vscode/swift/1.54.3/file/vscode.swift-1.54.3.vsix 32b12e0e1b284c273e5c120c394a75bf
-	vscode-theme-abyss https://open-vsx.org/api/vscode/theme-abyss/1.54.3/file/vscode.theme-abyss-1.54.3.vsix 9c580498861d19e4aff9ab2bf46020e0
-	vscode-theme-defaults https://open-vsx.org/api/vscode/theme-defaults/1.54.3/file/vscode.theme-defaults-1.54.3.vsix 80ff64379434c5773be739e89a8f2f34
-	vscode-theme-kimbie-dark https://open-vsx.org/api/vscode/theme-kimbie-dark/1.54.3/file/vscode.theme-kimbie-dark-1.54.3.vsix 52aa8d28f1f1299400d72d9023e65a01
-	vscode-theme-monokai https://open-vsx.org/api/vscode/theme-monokai/1.54.3/file/vscode.theme-monokai-1.54.3.vsix 6d65eb1998a7a7eb06ac56e555659999
-	vscode-theme-monokai-dimmed https://open-vsx.org/api/vscode/theme-monokai-dimmed/1.54.3/file/vscode.theme-monokai-dimmed-1.54.3.vsix f7ccf3d56fbb811db36e075b604cbe25
-	vscode-theme-quietlight https://open-vsx.org/api/vscode/theme-quietlight/1.54.3/file/vscode.theme-quietlight-1.54.3.vsix d66199d4c0104df14a0365933b3afe60
-	vscode-theme-red https://open-vsx.org/api/vscode/theme-red/1.54.3/file/vscode.theme-red-1.54.3.vsix 2d996393ab212a781682e0a8ee641713
-	vscode-theme-solarized-dark https://open-vsx.org/api/vscode/theme-solarized-dark/1.54.3/file/vscode.theme-solarized-dark-1.54.3.vsix e63d031121dd1c19bca3e88df5745851
-	vscode-theme-tomorrow-night-blue https://open-vsx.org/api/vscode/theme-tomorrow-night-blue/1.54.3/file/vscode.theme-tomorrow-night-blue-1.54.3.vsix 955cb9ac6f759e424182152552927d52
-	vscode-typescript https://open-vsx.org/api/vscode/typescript/1.54.3/file/vscode.typescript-1.54.3.vsix c2488834b57ba3413e7d43db2af0ed77
+	vscode-less https://open-vsx.org/api/vscode/less/1.62.3/file/vscode.less-1.62.3.vsix 9eb5e65d5d089abae8fc7cb2ebcdd53a
+	vscode-log https://open-vsx.org/api/vscode/log/1.62.3/file/vscode.log-1.62.3.vsix 454bea32383e2f93a035c8c8c9d680f1
+	vscode-lua https://open-vsx.org/api/vscode/lua/1.62.3/file/vscode.lua-1.62.3.vsix 89bf9895c9562b15ed5ae11c035df577
+	vscode-make https://open-vsx.org/api/vscode/make/1.62.3/file/vscode.make-1.62.3.vsix 19189fe2785ff3c0386a8877c9f9e132
+	vscode-markdown https://open-vsx.org/api/vscode/markdown/1.62.3/file/vscode.markdown-1.62.3.vsix b84496cb28d9032fe1dbf5738139c315
+	vscode-markdown-language-features https://open-vsx.org/api/vscode/markdown-language-features/1.62.3/file/vscode.markdown-language-features-1.62.3.vsix 4acc850aef4282c9adf773489eecedc5
+	vscode-merge-conflict https://open-vsx.org/api/vscode/merge-conflict/1.62.3/file/vscode.merge-conflict-1.62.3.vsix db6c52dafd6eeb65837262c7740c2b7a
+	vscode-npm https://open-vsx.org/api/vscode/npm/1.62.3/file/vscode.npm-1.62.3.vsix 16b3da4eac4e66ac3db90d3bb3e24479
+	ms-vscode-node-debug https://open-vsx.org/api/ms-vscode/node-debug/1.45.0/file/ms-vscode.node-debug-1.45.0.vsix 676769e9901f5f51ed6a21d7c6a831fb
+	ms-vscode-node-debug2 https://open-vsx.org/api/ms-vscode/node-debug2/1.43.0/file/ms-vscode.node-debug2-1.43.0.vsix 4c78ac1f3c4d753d005d72a68c13f7c7
+	vscode-objective-c https://open-vsx.org/api/vscode/objective-c/1.62.3/file/vscode.objective-c-1.62.3.vsix ea4419075d1a17cb57f984ec3ea783d0
+	vscode-perl https://open-vsx.org/api/vscode/perl/1.62.3/file/vscode.perl-1.62.3.vsix e1f9ebd43c5b3d4092418806bd066783
+	vscode-powershell https://open-vsx.org/api/vscode/powershell/1.62.3/file/vscode.powershell-1.62.3.vsix baf348ed743d3ba4a5b0b0702f6fa1c4
+	vscode-pug https://open-vsx.org/api/vscode/pug/1.62.3/file/vscode.pug-1.62.3.vsix 0d6fa06e57f2f9dbc5841e6996ba45d8
+	vscode-python https://open-vsx.org/api/vscode/python/1.62.3/file/vscode.python-1.62.3.vsix 98799c0d59f986378bc26d3a5809a76d
+	vscode-r https://open-vsx.org/api/vscode/r/1.62.3/file/vscode.r-1.62.3.vsix 9240efffdd93d5bcfa9e3833937df156
+	vscode-razor https://open-vsx.org/api/vscode/razor/1.62.3/file/vscode.razor-1.62.3.vsix 209d13c511ef5a6bdb8c23324a5df022
+	vscode-ruby https://open-vsx.org/api/vscode/ruby/1.62.3/file/vscode.ruby-1.62.3.vsix 52befb51090436c73273696b4e54baa4
+	vscode-rust https://open-vsx.org/api/vscode/rust/1.62.3/file/vscode.rust-1.62.3.vsix 0b6c07f593b7a06aa1acc21201b14b44
+	vscode-scss https://open-vsx.org/api/vscode/scss/1.62.3/file/vscode.scss-1.62.3.vsix e3ee9c810fb8c871f5df5dade2cc7479
+	vscode-shaderlab https://open-vsx.org/api/vscode/shaderlab/1.62.3/file/vscode.shaderlab-1.62.3.vsix d5f55aeedf313b3fe14d3c326ada2f2b
+	vscode-shellscript https://open-vsx.org/api/vscode/shellscript/1.62.3/file/vscode.shellscript-1.62.3.vsix 4abb4cad91156b25efd15cd029e02737
+	vscode-sql https://open-vsx.org/api/vscode/sql/1.62.3/file/vscode.sql-1.62.3.vsix 77392ebc2f3a1869823733099ca9ed87
+	vscode-swift https://open-vsx.org/api/vscode/swift/1.62.3/file/vscode.swift-1.62.3.vsix 29a1ea0de4f36baebee5f117336a92cc
+	vscode-theme-abyss https://open-vsx.org/api/vscode/theme-abyss/1.62.3/file/vscode.theme-abyss-1.62.3.vsix 04a5dcd017e04daa4269b7785d99def0
+	vscode-theme-defaults https://open-vsx.org/api/vscode/theme-defaults/1.62.3/file/vscode.theme-defaults-1.62.3.vsix 7fa0789b9ee1cb55a49698b018ac59ad
+	vscode-theme-kimbie-dark https://open-vsx.org/api/vscode/theme-kimbie-dark/1.62.3/file/vscode.theme-kimbie-dark-1.62.3.vsix ae60051044aab7708f33c13678f03f0b
+	vscode-theme-monokai https://open-vsx.org/api/vscode/theme-monokai/1.62.3/file/vscode.theme-monokai-1.62.3.vsix b2235d3b2fcb07858fbd2e9e3b9b9c77
+	vscode-theme-monokai-dimmed https://open-vsx.org/api/vscode/theme-monokai-dimmed/1.62.3/file/vscode.theme-monokai-dimmed-1.62.3.vsix 78aae18c5946da43bd756755a6df069a
+	vscode-theme-quietlight https://open-vsx.org/api/vscode/theme-quietlight/1.62.3/file/vscode.theme-quietlight-1.62.3.vsix 740e79e24fb7edeb277b6349a930c190
+	vscode-theme-red https://open-vsx.org/api/vscode/theme-red/1.62.3/file/vscode.theme-red-1.62.3.vsix d6ed6650920a1253a9c522300442f6f6
+	vscode-theme-solarized-dark https://open-vsx.org/api/vscode/theme-solarized-dark/1.62.3/file/vscode.theme-solarized-dark-1.62.3.vsix 6216c9d67d5d9443ca27e3afdb816e46
+	vscode-theme-tomorrow-night-blue https://open-vsx.org/api/vscode/theme-tomorrow-night-blue/1.62.3/file/vscode.theme-tomorrow-night-blue-1.62.3.vsix 04ca5f2984a9b5fed4102e040e4e4fda
+	vscode-typescript https://open-vsx.org/api/vscode/typescript/1.62.3/file/vscode.typescript-1.62.3.vsix 4f4b4b40506dacce128306facf3ed092
 	vscode-typescript-language-features https://open-vsx.org/api/vscode/typescript-language-features/1.54.3/file/vscode.typescript-language-features-1.54.3.vsix 968364ffa226c88d685f55cb21359954
-	vscode-vb https://open-vsx.org/api/vscode/vb/1.54.3/file/vscode.vb-1.54.3.vsix ac0b74fb681eaa2c51c6698623c3d43f
-	vscode-vscode-theme-seti https://open-vsx.org/api/vscode/vscode-theme-seti/1.54.3/file/vscode.vscode-theme-seti-1.54.3.vsix 4d0c2e66b8aef747221900367adba946
-	vscode-xml https://open-vsx.org/api/vscode/xml/1.54.3/file/vscode.xml-1.54.3.vsix f7aca865a3403a6e1b9e11e8019928b1
-	vscode-yaml https://open-vsx.org/api/vscode/yaml/1.54.3/file/vscode.yaml-1.54.3.vsix 0b2f74133505286388c043a52494168d
-	EditorConfig-EditorConfig https://open-vsx.org/api/EditorConfig/EditorConfig/0.16.4/file/EditorConfig.EditorConfig-0.16.4.vsix 0f8e4278d7a3638b9f9df84a07a56021
-	dbaeumer-vscode-eslint https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.8/file/dbaeumer.vscode-eslint-2.1.8.vsix d3668785b2744386867631d79537552a
-	ms-vscode-references-view https://open-vsx.org/api/ms-vscode/references-view/0.0.77/file/ms-vscode.references-view-0.0.77.vsix 0d0be6e6fd0ccfdd2a230b077ff8b4d7
+	vscode-vb https://open-vsx.org/api/vscode/vb/1.62.3/file/vscode.vb-1.62.3.vsix cd17e947324bb9cd05e799546d7d4638
+	vscode-vscode-theme-seti https://open-vsx.org/api/vscode/vscode-theme-seti/1.62.3/file/vscode.vscode-theme-seti-1.62.3.vsix b27e54ecb1c38f270f8c671a6574dcbc
+	vscode-xml https://open-vsx.org/api/vscode/xml/1.62.3/file/vscode.xml-1.62.3.vsix e3ba682c8ecb22516a1fd02abcdff841
+	vscode-yaml https://open-vsx.org/api/vscode/yaml/1.62.3/file/vscode.yaml-1.62.3.vsix f9719aaae4fd9671fd2fa15d41343e1d
+	EditorConfig-EditorConfig https://open-vsx.org/api/EditorConfig/EditorConfig/0.16.6/file/EditorConfig.EditorConfig-0.16.6.vsix e787245e6c68617178ae995ad97c3ccb
+	dbaeumer-vscode-eslint https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.20/file/dbaeumer.vscode-eslint-2.1.20.vsix 1cb024ac02ebeb5ce6b0dfed6e51cdd2
+	ms-vscode-references-view https://open-vsx.org/api/ms-vscode/references-view/0.0.82/file/ms-vscode.references-view-0.0.82.vsix d23827c6600ae821c829fca1629968c7
 	vscjava-vscode-java-debug https://open-vsx.org/api/vscjava/vscode-java-debug/0.29.0/file/vscjava.vscode-java-debug-0.29.0.vsix 1eb95110f84ff8dcabbe3c672066b86d
 	redhat-java https://open-vsx.org/api/redhat/java/0.61.0/file/redhat.java-0.61.0.vsix 72e548e2845e1ff655f28111558d6942
 	vscjava-vscode-java-test https://open-vsx.org/api/vscjava/vscode-java-test/0.26.0/file/vscjava.vscode-java-test-0.26.0.vsix fd63da5537a4bee1d3ceaae0fa6bf419
 	ms-python-python https://open-vsx.org/api/ms-python/python/2020.9.112786/file/ms-python.python-2020.9.112786.vsix c64b79fa822418e07b6d0f57b8838b44
-	perrinjerome-vscode-zc-buildout https://open-vsx.org/api/perrinjerome/vscode-zc-buildout/0.5.0/file/perrinjerome.vscode-zc-buildout-0.5.0.vsix 739889de7ccaf8232fb6c6fe8571a1fa
+	perrinjerome-vscode-zc-buildout https://open-vsx.org/api/perrinjerome/vscode-zc-buildout/0.6.2/file/perrinjerome.vscode-zc-buildout-0.6.2.vsix 76c1420f238a0754a505459563220973
 	jebbs-plantuml https://open-vsx.org/api/jebbs/plantuml/2.14.0/file/jebbs.plantuml-2.14.0.vsix 13fa7cbd14a30ecca166c41a307c7a73
 	rafaelmaiolla-diff https://open-vsx.org/api/rafaelmaiolla/diff/0.0.1/file/rafaelmaiolla.diff-0.0.1.vsix 1d8f868bc19b7d703c1be2bf99c4c7f9
 	perrinjerome-git-commit-syntax https://open-vsx.org/api/perrinjerome/git-commit-syntax/0.0.1/file/perrinjerome.git-commit-syntax-0.0.1.vsix 46625f2f05e244911c2cb9cc5032c0ef

component/theia/generate_download_plugins_cfg.py

@@ -41,8 +41,7 @@ for plugin_and_version in '''\
     vscode/lua/latest
     vscode/make/latest
     vscode/markdown/latest
-# https://github.com/eclipse-theia/theia/issues/7780
-    vscode/markdown-language-features/1.39.2
+    vscode/markdown-language-features/latest
     vscode/merge-conflict/latest
     vscode/npm/latest
     ms-vscode/node-debug/latest
@@ -71,13 +70,17 @@ for plugin_and_version in '''\
     vscode/theme-solarized-dark/latest
     vscode/theme-tomorrow-night-blue/latest
     vscode/typescript/latest
-    vscode/typescript-language-features/latest
+# latest (1.62.3) does not activate because it uses a proposed API not in theia:
+# Activating extension 'TypeScript and JavaScript Language Features (built-in)' failed: r.languages.createLanguageStatusItem is not a function
+    vscode/typescript-language-features/1.54.3
     vscode/vb/latest
     vscode/vscode-theme-seti/latest
     vscode/xml/latest
     vscode/yaml/latest
     EditorConfig/EditorConfig/latest
-    dbaeumer/vscode-eslint/latest
+# latest (2.2.2) does not activate:
+# Activating extension 'ESLint' failed: Class extends value undefined is not a constructor or null
+    dbaeumer/vscode-eslint/2.1.20
     ms-vscode/references-view/latest
 # golang.Go removed because it overwrites the PATH in theia shell
 # golang/Go/0.16.2

component/trafficserver/buildout.cfg

@@ -24,8 +24,8 @@ min_version = 8
 
 [trafficserver]
 recipe = slapos.recipe.cmmi
-url = http://apache.claz.org/trafficserver/trafficserver-9.1.0.tar.bz2
-md5sum = 994b0aa879cbd95054048f34bf8ed954
+url = https://dlcdn.apache.org/trafficserver/trafficserver-9.1.1.tar.bz2
+md5sum = 666945652942cce8c3ff0ae2757cc3c6
 shared = true
 patch-options = -p1
 configure-options =

component/userhosts/buildout.cfg

 [userhosts]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://lab.nexedi.com/nexedi/userhosts/repository/${:revision}/archive.tar.gz
-revision = a05fe5a3a5cb7005351ef4ec41460089f3ce4d0a
-md5sum = 5a80b4d962d975f290a60cf790c3334d
+url = https://github.com/figiel/hosts/archive/${:revision}.tar.gz
+revision = fdb45fe219593d63453f4be55cfc2a1199c18f59
+md5sum = 9ff5407132a2a67200ef1036a02f0ee2
 configure-command = true
 make-options = PREFIX=@@LOCATION@@
-make-targets = check install
+make-targets = test install

component/wkhtmltopdf/buildout.cfg

@@ -4,7 +4,6 @@ extends =
   ../freetype/buildout.cfg
   ../fontconfig/buildout.cfg
   ../libpng/buildout.cfg
-  ../tar/buildout.cfg
   ../xorg/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
@@ -14,27 +13,39 @@ parts =
 
 [wkhtmltopdf]
 recipe = slapos.recipe.build
-
-# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-base = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_linux-generic
-i386-linux-gnu   = ${:base}-i386.tar.xz  ce1a2c0b2cf786ccc5d5828c42c99ddd
-x86_64-linux-gnu = ${:base}-amd64.tar.xz 96b7306cebb9e65355f69f7ab63df68b
-
+url = http://download.gna.org/wkhtmltopdf/0.12/0.12.4/wkhtmltox-0.12.4_${:_url}.tar.xz
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s
 install =
-  import os,shutil, sys, tempfile
-  path = self.download(*options[multiarch()].split())
-  extract_dir = tempfile.mkdtemp(self.name)
-  self.cleanup_dir_list.append(extract_dir)
-  self.logger.debug('Created working directory %s', extract_dir)
-  env = os.environ.copy()
-  env["PATH"] = "${tar:location}/bin:${xz-utils:location}/bin" + (":" + env["PATH"] if env.get("PATH") else "")
-  env["LD_LIBRARY_PATH"] = "${xz-utils:location}/lib" + (":" + env["LD_LIBRARY_PATH"] if env.get("LD_LIBRARY_PATH") else "")
-  call(["tar", "xJf", path, "-C", extract_dir], env=env)
-  shutil.move(os.path.join(extract_dir, "wkhtmltox"), location)
+  import os
+  self.copyTree(guessworkdir(self.extract(self.download())), location)
   with open(os.path.join(location, "wkhtmltopdf"), 'w') as wrapper:
-    wrapper.write("""#!/bin/sh
-  export LD_LIBRARY_PATH=%(location)s/lib:${fontconfig:location}/lib:${freetype:location}/lib:${libX11:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libpng12:location}/lib:${zlib:location}/lib
-  export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/wkhtmltopdf "$@"
-  """ % options)
+    wrapper.write("""#!/bin/sh -e
+  d=%s
+  export LD_LIBRARY_PATH=$d/lib:%s
+  export PATH=%s:$PATH
+  exec $d/bin/wkhtmltopdf "$@"
+  """ % (
+    location,
+    ':'.join(options['library'].split()),
+    ':'.join(options['path'].split()),
+  ))
     os.fchmod(wrapper.fileno(), 0o755)
+library =
+  ${fontconfig:location}/lib
+  ${freetype:location}/lib
+  ${libX11:location}/lib
+  ${libXext:location}/lib
+  ${libXrender:location}/lib
+  ${libpng12:location}/lib
+  ${zlib:location}/lib
+path =
+  ${fontconfig:location}/bin
+
+[wkhtmltopdf:getattr(sys,'_multiarch',None)=='i386-linux-gnu']
+_url = linux-generic-i386
+md5sum = ce1a2c0b2cf786ccc5d5828c42c99ddd
+
+[wkhtmltopdf:getattr(sys,'_multiarch',None)=='x86_64-linux-gnu']
+_url = linux-generic-amd64
+md5sum = 96b7306cebb9e65355f69f7ab63df68b

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.220'
+version = '1.0.226'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/generic_mysql/mysql.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
   while True:
     while True:
       mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        universal_newlines=True)
       result = mysql_upgrade.communicate()[0]
       if mysql_upgrade.returncode:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
+        print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
         break
-      print "MySQL database upgraded with result:\n%s" % result
+      print("MySQL database upgraded with result:\n%s" % result)
       mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
-          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+          universal_newlines=True)
       result = mysql.communicate(mysql_script)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
       # import timezone database
       mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               universal_newlines=True)
       timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
       if mysql_tzinfo_to_sql.returncode != 0:
-        print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
         break
       mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                               universal_newlines=True)
       result = mysql.communicate(timezone_sql)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
-      print 'SlapOS initialisation script succesfully applied on database.'
+      print('SlapOS initialisation script succesfully applied on database.')
       return
     sleep = min(sleep+1, 30)
-    print 'Sleeping for %ss and retrying' % sleep
+    print('Sleeping for %ss and retrying' % sleep)
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

software/erp5/test/test/__init__.py

@@ -49,17 +49,20 @@ def setUpModule():
 class ERP5InstanceTestCase(SlapOSInstanceTestCase):
   """ERP5 base test case
   """
-  def getRootPartitionConnectionParameterDict(self):
+  @classmethod
+  def getRootPartitionConnectionParameterDict(cls):
     """Return the output paramters from the root partition"""
     return json.loads(
-        self.computer_partition.getConnectionParameterDict()['_'])
+        cls.computer_partition.getConnectionParameterDict()['_'])
 
-  def getComputerPartition(self, partition_reference):
-    for computer_partition in self.slap.computer.getComputerPartitionList():
+  @classmethod
+  def getComputerPartition(cls, partition_reference):
+    for computer_partition in cls.slap.computer.getComputerPartitionList():
       if partition_reference == computer_partition.getInstanceParameter(
           'instance_title'):
         return computer_partition
 
-  def getComputerPartitionPath(self, partition_reference):
-    partition_id = self.getComputerPartition(partition_reference).getId()
-    return os.path.join(self.slap._instance_root, partition_id)
+  @classmethod
+  def getComputerPartitionPath(cls, partition_reference):
+    partition_id = cls.getComputerPartition(partition_reference).getId()
+    return os.path.join(cls.slap._instance_root, partition_id)

software/erp5/test/test/test_erp5.py

@@ -26,21 +26,30 @@
 ##############################################################################
 
 from __future__ import absolute_import
-import os
-import json
+
+import contextlib
 import glob
-import six.moves.urllib.parse
+import json
+import os
+import shutil
 import socket
+import ssl
+import subprocess
+import sys
+import tempfile
 import time
+import unittest
 
 import psutil
 import requests
-
-from . import ERP5InstanceTestCase
-from . import setUpModule
 import six
-from six.moves import map
-from six.moves import range
+import six.moves.urllib.parse
+import six.moves.xmlrpc_client
+import urllib3
+from slapos.testing.utils import CrontabMixin
+
+from . import ERP5InstanceTestCase, setUpModule
+
 setUpModule # pyflakes
 
 
@@ -68,8 +77,8 @@ class TestPublishedURLIsReachableMixin(object):
       session.mount(
           base_url,
           requests.adapters.HTTPAdapter(
-              max_retries=requests.packages.urllib3.util.retry.Retry(
-                  total=60,
+              max_retries=urllib3.util.retry.Retry(
+                  total=20,
                   backoff_factor=.5,
                   status_forcelist=(404, 500, 503))))
 
@@ -389,3 +398,455 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
         }, {
           "cache-size": None,
         })
+
+
+class TestWatchActivities(ERP5InstanceTestCase):
+  """Tests for bin/watch_activities scripts in zope partitions.
+  """
+  __partition_reference__ = 'wa'
+
+  def test(self):
+    # "watch_activites" scripts use watch command. We'll fake a watch command
+    # that executes the actual command only once to check the output.
+    tmpdir = tempfile.mkdtemp()
+    self.addCleanup(shutil.rmtree, tmpdir)
+    with open(os.path.join(tmpdir, 'watch'), 'w') as f:
+      f.write("""#!/bin/sh
+
+      if [ "$1" != "-n" ] || [ "$2" != "5" ]
+      then
+        echo unexpected arguments: "$1" "$2"
+        exit 1
+      fi
+      shift
+      shift
+
+      exec bash -c "$@"
+      """)
+      os.fchmod(f.fileno(), 0o700)
+    try:
+      output = subprocess.check_output(
+          [
+              os.path.join(
+                  self.getComputerPartitionPath('zope-1'),
+                  'bin',
+                  'watch_activities',
+              )
+          ],
+          env=dict(os.environ,
+                   PATH=os.pathsep.join([tmpdir, os.environ['PATH']])),
+          stderr=subprocess.STDOUT,
+          universal_newlines=True,
+      )
+    except subprocess.CalledProcessError as e:
+      self.fail(e.output)
+    self.assertIn(' dict ', output)
+
+
+class ZopeTestMixin(CrontabMixin):
+  """Mixin class for zope features.
+  """
+  wsgi = NotImplemented # type: bool
+
+
+  __partition_reference__ = 'z'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+        '_':
+        json.dumps({
+            "zope-partition-dict": {
+                "default": {
+                    "longrequest-logger-interval": 1,
+                    "longrequest-logger-timeout": 1,
+                },
+            },
+            "wsgi": cls.wsgi,
+        })
+    }
+
+  @classmethod
+  def _setUpClass(cls):
+    super(ZopeTestMixin, cls)._setUpClass()
+
+    param_dict = cls.getRootPartitionConnectionParameterDict()
+    # rebuild an url with user and password
+    parsed = six.moves.urllib.parse.urlparse(param_dict['family-default'])
+    cls.zope_base_url = parsed._replace(
+        netloc='{}:{}@{}:{}'.format(
+            param_dict['inituser-login'],
+            param_dict['inituser-password'],
+            parsed.hostname,
+            parsed.port,
+        ),
+        path=param_dict['site-id'] + '/',
+    ).geturl()
+
+    cls.zope_deadlock_debugger_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        '/manage_debug_threads?{deadlock-debugger-password}'.format(
+            **param_dict),
+    )
+
+
+    @contextlib.contextmanager
+    def getXMLRPCClient():
+      # don't verify certificate
+      ssl_context = ssl.create_default_context()
+      ssl_context.check_hostname = False
+      ssl_context.verify_mode = ssl.CERT_NONE
+      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+          cls.zope_base_url,
+          context=ssl_context,
+      )
+      # BBB use as a context manager only on python3
+      if sys.version_info < (3, ):
+        yield erp5_xmlrpc_client
+      else:
+        with erp5_xmlrpc_client:
+          yield erp5_xmlrpc_client
+
+    with getXMLRPCClient() as erp5_xmlrpc_client:
+      # wait for ERP5 to be ready (TODO: this should probably be a promise)
+      for _ in range(120):
+        time.sleep(1)
+        try:
+          erp5_xmlrpc_client.getTitle()
+        except (six.moves.xmlrpc_client.ProtocolError,
+                six.moves.xmlrpc_client.Fault):
+          pass
+        else:
+          break
+
+    def addPythonScript(script_id, params, body):
+      with getXMLRPCClient() as erp5_xmlrpc_client:
+        custom = erp5_xmlrpc_client.portal_skins.custom
+        try:
+          custom.manage_addProduct.PythonScripts.manage_addPythonScript(
+              script_id)
+        except six.moves.xmlrpc_client.ProtocolError as e:
+          if e.errcode != 302:
+            raise
+        getattr(custom, script_id).ZPythonScriptHTML_editAction(
+            '',
+            '',
+            params,
+            body,
+        )
+
+    # a python script to verify activity processing
+    addPythonScript(
+        script_id='ERP5Site_verifyActivityProcessing',
+        params='mode',
+        body='''if 1:
+          import json
+          portal = context.getPortalObject()
+          if mode == "count":
+            return json.dumps(dict(count=len(portal.portal_activities.getMessageList())))
+          if mode == "activate":
+            for _ in range(10):
+              portal.portal_templates.activate(activity="SQLQueue").getTitle()
+            return "activated"
+          raise ValueError("Unknown mode: %s" % mode)
+        ''',
+    )
+    cls.zope_verify_activity_processing_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_verifyActivityProcessing',
+    )
+    # a python script logging to event log
+    addPythonScript(
+        script_id='ERP5Site_logMessage',
+        params='name',
+        body='''if 1:
+            from erp5.component.module.Log import log
+            return log("hello %s" % name)
+        ''',
+    )
+    cls.zope_log_message_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_logMessage',
+    )
+    # a python script issuing a long request
+    addPythonScript(
+        script_id='ERP5Site_executeLongRequest',
+        params='',
+        body='''if 1:
+          import time
+          for _ in range(5):
+            time.sleep(1)
+          return "done"    
+        ''',
+    )
+    cls.zope_long_request_url = six.moves.urllib_parse.urljoin(
+        cls.zope_base_url,
+        'ERP5Site_executeLongRequest',
+    )
+
+  def setUp(self):
+    super(ZopeTestMixin, self).setUp()
+    # run logrotate a first time so that it create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+  def tearDown(self):
+    super(ZopeTestMixin, self).tearDown()
+    # reset logrotate status
+    logrotate_status = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'logrotate.status',
+    )
+    if os.path.exists(logrotate_status):
+      os.unlink(logrotate_status)
+    for logfile in glob.glob(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'srv',
+            'backup',
+            'logrotate',
+            '*',
+        )):
+      os.unlink(logfile)
+    for logfile in glob.glob(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'srv',
+            'monitor',
+            'private',
+            'documents',
+            '*',
+        )):
+      os.unlink(logfile)
+
+  def _getCrontabCommand(self, crontab_name):
+    # type: (str) -> str
+    """Read a crontab and return the command that is executed.
+
+    overloaded to use crontab from zope partition
+    """
+    with open(
+        os.path.join(
+            self.getComputerPartitionPath('zope-default'),
+            'etc',
+            'cron.d',
+            crontab_name,
+        )) as f:
+      crontab_spec, = f.readlines()
+    self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
+    return crontab_spec.split(None, 5)[-1]
+
+  def test_event_log_rotation(self):
+    requests.get(
+        self.zope_log_message_url,
+        params={
+            "name": "world"
+        },
+        verify=False,
+    ).raise_for_status()
+
+    zope_event_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-event.log',
+    )
+    with open(zope_event_log_path) as f:
+      self.assertIn('hello world', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'zope-0-event.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('hello world', f.read())
+
+    requests.get(
+        self.zope_log_message_url,
+        params={
+            "name": "le monde"
+        },
+        verify=False,
+    ).raise_for_status()
+    with open(zope_event_log_path) as f:
+      self.assertNotIn('hello world', f.read())
+    with open(zope_event_log_path) as f:
+      self.assertIn('hello le monde', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_access_log_rotation(self):
+    requests.get(
+        self.zope_base_url,
+        verify=False,
+        headers={
+            'User-Agent': 'before rotation'
+        },
+    ).raise_for_status()
+
+    zope_access_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-Z2.log',
+    )
+    with open(zope_access_log_path) as f:
+      self.assertIn('before rotation', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'zope-0-Z2.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('before rotation', f.read())
+
+    requests.get(
+        self.zope_base_url,
+        verify=False,
+        headers={
+            'User-Agent': 'after rotation'
+        },
+    ).raise_for_status()
+    with open(zope_access_log_path) as f:
+      self.assertNotIn('before rotation', f.read())
+    with open(zope_access_log_path) as f:
+      self.assertIn('after rotation', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_long_request_log_rotation(self):
+    requests.get(self.zope_long_request_url,
+                 verify=False,
+                 params={
+                     'when': 'before rotation'
+                 }).raise_for_status()
+
+    zope_long_request_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'longrequest_logger_zope-0.log',
+    )
+    with open(zope_long_request_log_path) as f:
+      self.assertIn('before rotation', f.read())
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+    # this logrotate leaves the log for the day as non compressed
+    rotated_log_file = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'srv',
+        'backup',
+        'logrotate',
+        'longrequest_logger_zope-0.log-20500101',
+    )
+    with open(rotated_log_file) as f:
+      self.assertIn('before rotation', f.read())
+
+    requests.get(
+        self.zope_long_request_url,
+        verify=False,
+        params={
+            'when': 'after rotation'
+        },
+    ).raise_for_status()
+    with open(zope_long_request_log_path) as f:
+      self.assertNotIn('before rotation', f.read())
+    with open(zope_long_request_log_path) as f:
+      self.assertIn('after rotation', f.read())
+
+    # on next day execution of logrotate, log files are compressed
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+
+  def test_basic_authentication_user_in_access_log(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+    requests.get(self.zope_base_url,
+                 verify=False,
+                 auth=requests.auth.HTTPBasicAuth(
+                     param_dict['inituser-login'],
+                     param_dict['inituser-password'],
+                 )).raise_for_status()
+
+    zope_access_log_path = os.path.join(
+        self.getComputerPartitionPath('zope-default'),
+        'var',
+        'log',
+        'zope-0-Z2.log',
+    )
+    with open(zope_access_log_path) as f:
+      self.assertIn(param_dict['inituser-login'], f.read())
+
+  def test_deadlock_debugger(self):
+    dump_response = requests.get(
+        self.zope_deadlock_debugger_url,
+        verify=False,
+    )
+    dump_response.raise_for_status()
+    self.assertIn('Thread ', dump_response.text)
+
+  def test_activity_processing(self):
+    def wait_for_activities(max_retries):
+      for retry in range(max_retries):
+        time.sleep(10)
+        resp = requests.get(
+            self.zope_verify_activity_processing_url,
+            params={
+                'mode': 'count',
+                'retry': retry,
+            },
+            verify=False,
+        )
+        if not resp.ok:
+          # XXX we start by flushing existing activities from site creation
+          # and inital upgrader run. During this time it may happen that
+          # ERP5 replies with site errors, we tolerate these errors and only
+          # check the final state.
+          continue
+        count = resp.json()['count']
+        if not count:
+          break
+      else:
+        self.assertEqual(count, 0)
+
+    wait_for_activities(60)
+    requests.get(
+        self.zope_verify_activity_processing_url,
+        params={
+            'mode': 'activate'
+        },
+        verify=False,
+    ).raise_for_status()
+    wait_for_activities(10)
+
+
+class TestZopeMedusa(ZopeTestMixin, ERP5InstanceTestCase):
+  wsgi = False
+
+
+class TestZopeWSGI(ZopeTestMixin, ERP5InstanceTestCase):
+  wsgi = True
+
+  @unittest.expectedFailure
+  def test_long_request_log_rotation(self):
+    super(TestZopeWSGI, self).test_long_request_log_rotation(self)
+
+  @unittest.expectedFailure
+  def test_basic_authentication_user_in_access_log(self):
+    super(TestZopeWSGI, self).test_basic_authentication_user_in_access_log(self)

software/erp5/upgrade_test/test.py

@@ -39,13 +39,15 @@ from slapos.testing.testcase import installSoftwareUrlList
 from slapos.testing.testcase import SlapOSNodeCommandError
 from slapos.grid.utils import md5digest
 
-old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.4/software/erp5/software.cfg'
+old_software_release_url = 'https://lab.nexedi.com/nexedi/slapos/raw/1.0.167.5/software/erp5/software.cfg'
 new_software_release_url = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))
 
 _, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    old_software_release_url, software_id="upgrade_erp5")
-
+    old_software_release_url,
+    software_id="upgrade_erp5",
+    skip_software_check=True,
+)
 
 def setUpModule():
   installSoftwareUrlList(

software/fluentd/test/test.py

@@ -35,6 +35,7 @@ import subprocess
 import tempfile
 import time
 import six
+import sys
 
 from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
 from six.moves.socketserver import StreamRequestHandler, TCPServer
@@ -171,7 +172,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
 <source>
   @type exec
   tag tag.name
-  command python %s
+  command %s %s
   run_interval %ss
   <parse>
     keys pressure, humidity, temperature
@@ -186,7 +187,7 @@ class SensorConfTestCase(WendelinTutorialTestCase):
   <buffer>
     flush_mode immediate
   </buffer>
-</match>''' % (script_path, FLUSH_INTERVAL, cls._ipv6_address)
+</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
 
   @classmethod
   def sensor_script(cls, measurementList):
@@ -280,4 +281,4 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
       data,
       msgpack.unpackb(
         self.serve(self._wendelin_port, WendelinHTTPRequestHandler)),
-    )
+    )
\ No newline at end of file

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 087f226ba90928dcc5a722d7008c867a
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = b260fce887535fc69a259d1fd31af1b2
+md5sum = d949e6100fd4e01819cc2867a65c6e65
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in

software/kvm/instance-kvm.cfg.jinja2

@@ -1029,15 +1029,12 @@ config-command = ${ansible-vm-bin:rendered}
 
 {% if bootstrap_script_url -%}
 [download-bootstrap-script]
-recipe = slapos.recipe.build
-location = ${directory:public}/vm-bootstrap
-install =
-  from zc.buildout.download import check_md5sum, urlretrieve, ChecksumError
-  url, md5sum = {{ repr(bootstrap_script_url) }}.split('#')
-  urlretrieve(url, location)
-  if not check_md5sum(location, md5sum):
-    os.remove(location)
-    raise ChecksumError
+recipe = slapos.recipe.build:download
+offline = false
+destination = ${directory:public}/vm-bootstrap
+{% set bootstrap_script_url = bootstrap_script_url.split('#') -%}
+url = {{ bootstrap_script_url[0] }}
+md5sum = {{ bootstrap_script_url[1] }}
 {% endif -%}
 
 [logrotate-vm-bootstrap]

software/ors-amarisoft/buildout.hash.cfg

@@ -16,27 +16,27 @@
 
 [template]
 filename = instance.cfg
-md5sum = 96a76300b2f714b6c47157920fe79a53
+md5sum = 6ea4fa210a91c15278c847a809de5991
 
 [template-lte-enb-epc]
 _update_hash_filename_ = instance-enb-epc.jinja2.cfg
-md5sum = 762d55291e75e8b61e35f9f28d29915a
+md5sum = 25fae79db2b30be0f365ad967b278c3c
 
 [template-lte-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = 83c37b43d7b70584bf71aed9289ea13c
+md5sum = fe249168a3f50b0efe6aeae39afb03ae
 
 [template-lte-gnb-epc]
 _update_hash_filename_ = instance-gnb-epc.jinja2.cfg
-md5sum = e43a726dd3023a4bbaa474bb2d7a6ebe
+md5sum = 5ec4508b02f6e7fbdbe1f37c65b9d897
 
 [template-lte-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = fc59d15a7f7f942951f9e38d7a1cca2c
+md5sum = 6deb38b3de44f6e0a43b49fd13f0f072
 
 [template-lte-epc]
 _update_hash_filename_ = instance-epc.jinja2.cfg
-md5sum = 0585b4f8fd42538595e6abb61d07fd93
+md5sum = 089f62e736cdc620eafc2c47b050fe13
 
 [ue_db.jinja2.cfg]
 filename = config/ue_db.jinja2.cfg
@@ -44,11 +44,11 @@ md5sum = d33163012d6c98efc59161974c649557
 
 [enb.jinja2.cfg]
 filename = config/enb.jinja2.cfg
-md5sum = d841debc51d9f12555a47d1556a6a3c1
+md5sum = 97d9cdd07704cae36c5e4234c87025e8
 
 [gnb.jinja2.cfg]
 filename = config/gnb.jinja2.cfg
-md5sum = da64ea9c5003f40987a8bba3f18e8839
+md5sum = b0df7f3b679b25d5296dd67e074364b4
 
 [ltelogs.jinja2.sh]
 filename = ltelogs.jinja2.sh
@@ -61,3 +61,11 @@ md5sum = 518c71ce57204304b703b977c665a164
 [ims.jinja2.cfg]
 filename = config/ims.jinja2.cfg
 md5sum = e561ec26a70943c61557def1781cf65f
+
+[sdr-busy-promise]
+_update_hash_filename_ = promise/check_sdr_busy.py
+md5sum = 9e867282d7dd80c4255e87a82a47a19d
+
+[interface-up-promise]
+_update_hash_filename_ = promise/check_interface_up.py
+md5sum = 505efcbe04e717088924f2267b10c2b9

software/ors-amarisoft/config/enb.jinja2.cfg

@@ -56,15 +56,15 @@
     { 
       /* address of MME for S1AP connection. Must be modified if the MME
          runs on a different host. */
-      mme_addr: "127.0.1.100"
+      mme_addr: "{{ slapparameter_dict.get('mme_addr', '127.0.1.100') }}"
     },
   ],
   /* GTP bind address (=address of the ethernet interface connected to
      the MME). Must be modified if the MME runs on a different host. */
-  gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
+  gtp_addr: "{{ gtp_addr }}",
 
   /* high 20 bits of SIB1.cellIdentifier */
-  enb_id: 0x1A2D0,
+  enb_id: {{ slapparameter_dict.get('enb_id', '0x1A2D0') }},
 
   /* list of cells */
   cell_list: [

software/ors-amarisoft/config/gnb.jinja2.cfg

@@ -37,15 +37,15 @@
   amf_list: [
     {
       /* address of AMF for NGAP connection. Must be modified if the AMF runs on a different host. */
-      amf_addr: "127.0.1.100",
+      amf_addr: "{{ slapparameter_dict.get('amf_addr', '127.0.1.100') }}",
     },
   ],
   /* GTP bind address (=address of the ethernet interface connected to
      the AMF). Must be modified if the AMF runs on a different host. */
-  gtp_addr: "{{ slapparameter_dict.get('gtp_addr', '127.0.1.1') }}",
+  gtp_addr: "{{ gtp_addr }}",
 
   gnb_id_bits: 28,
-  gnb_id: 0x12345,
+  gnb_id: {{ slapparameter_dict.get('gnb_id', '0x12345') }},
 
   nr_support: true,
 

software/ors-amarisoft/instance-enb-epc.jinja2.cfg

@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
@@ -34,7 +34,7 @@ promise = ${:etc}/promise
 log = ${:var}/log
 
 [request-common-base]
-recipe = slapos.cookbook:request.serialised
+recipe = slapos.cookbook:request
 software-url = {{ slap_connection['software-release-url'] }}
 server-url = {{ slap_connection['server-url'] }}
 computer-id = {{ slap_connection['computer-id'] }}
@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-enb-input-schema.jinja2.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": {{ default_lte_n_rb_dl }}
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-enb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
 configuration.ims_ws_port = 9003
@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  sudo /opt/amarisoft/rm-tmp-lte | true;
+  {{ enb }}/lteenb ${directory:etc}/enb.cfg >> ${directory:log}/enb-output.cfg 2>> ${directory:log}/enb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/enb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-enb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -83,6 +93,7 @@ context =
   section directory directory
   section slap_configuration slap-configuration
   key slapparameter_dict slap-configuration:configuration
+  raw gtp_addr {{ local_ipv4 }}
   import  netaddr netaddr
 
 [lte-enb-config]
@@ -91,9 +102,21 @@ template = {{ enb_template }}
 rendered = ${directory:etc}/enb.cfg
 
 [publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
+<= monitor-publish
+recipe = slapos.cookbook:publish
+ipv4 = {{ local_ipv4 }}
 
 [monitor-instance-parameter]
+{% if slapparameter_dict.get("name", None) %}
 monitor-title = {{ slapparameter_dict['name'] | string }}
+{% endif %}
+{% if slapparameter_dict.get("monitor-password", None) %}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+{% endif %}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance-epc.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-mme-config
   lte-mme-service
-# Temporarily extend monitor-base until promises are added
+  tun-up-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.log_size = 50M
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
@@ -62,7 +62,7 @@ context =
 [lte-ims-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/ims.log; sleep 1
-command-line = rm -f ${directory:var}/lte_ue.db; {{ mme }}/lteims ${directory:etc}/ims.cfg
+command-line = {{ mme }}/lteims ${directory:etc}/ims.cfg
 wrapper-path = ${directory:service}/lte-ims
 mode = 0775
 pidfile = ${directory:run}/ims.pid
@@ -71,17 +71,29 @@ hash-files =
   {{ ue_db_path }}
 environment = AMARISOFT_PATH=/opt/amarisoft/.amarisoft
 
+[lte-mme-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  rm -f ${directory:var}/lte_ue.db;
+  {{ mme }}/ltemme ${directory:etc}/mme.cfg >> ${directory:log}/mme-output.cfg 2>> ${directory:log}/mme-output.cfg
+
 ### MME
 [lte-mme-service]
 recipe = slapos.cookbook:wrapper
+# When the machine shutdowns abruptly, lte_ue is not cleaned up which causes
+# amarisoft ltemme to fail. TODO: find a cleaner way to handle this
 init = ${ltelogs:rendered} ${directory:log}/mme.log
-command-line = {{ mme }}/ltemme ${directory:etc}/mme.cfg
+command-line = ${lte-mme-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-mme
 mode = 0775
 pidfile = ${directory:run}/mme.pid
 hash-files =
   ${lte-mme-config:rendered}
   {{ ue_db_path }}
+  ${lte-mme-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib:{{ nghttp2_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -119,17 +131,18 @@ context =
   import  netaddr netaddr
   key ifup_empty lte-mme-ifup-empty:wrapper-path
 
-[publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-
-{% if slapparameter_dict.get("monitor-password", None) %}
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
-{% else %}
-monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
-{% endif %}
-
-{% if slapparameter_dict.get("monitor-password", None) %}
 [monitor-instance-parameter]
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-password'] }}
-{% endif %}
+
+[publish-connection-information]
+<= monitor-publish
+recipe = slapos.cookbook:publish
+
+# Add custom promise to check if /dev/sdr0 is busy
+[tun-up-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ interface_up_promise }}
+output = ${directory:plugins}/check-tun-up.py
+config-ifname = ${slap-configuration:tun-name}

software/ors-amarisoft/instance-gnb-epc.jinja2.cfg

@@ -12,7 +12,7 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
@@ -34,7 +34,7 @@ promise = ${:etc}/promise
 log = ${:var}/log
 
 [request-common-base]
-recipe = slapos.cookbook:request.serialised
+recipe = slapos.cookbook:request
 software-url = {{ slap_connection['software-release-url'] }}
 server-url = {{ slap_connection['server-url'] }}
 computer-id = {{ slap_connection['computer-id'] }}
@@ -42,6 +42,8 @@ partition-id = {{ slap_connection['partition-id'] }}
 key-file = {{ slap_connection['key-file'] }}
 cert-file = {{ slap_connection['cert-file'] }}
 
+sla-computer_guid = {{ slap_connection['computer-id'] }}
+
 config-monitor-password = ${monitor-htpasswd:passwd}
 return = monitor-base-url
 

software/ors-amarisoft/instance-gnb-input-schema.jinja2.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": {{ default_nr_bandwidth }}
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -4,7 +4,7 @@ parts =
   ltelogs
   lte-gnb-config
   lte-enb-service
-# Temporarily extend monitor-base until promises are added
+  sdr-busy-promise
   monitor-base
   publish-connection-information
 
@@ -15,15 +15,15 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = {{ slap_connection['computer-id'] }}
 partition = {{ slap_connection['partition-id'] }}
 url = {{ slap_connection['server-url'] }}
 key = {{ slap_connection['key-file'] }}
 cert = {{ slap_connection['cert-file'] }}
 
-configuration.network_name = VIFIB
-configuration.domain = vifib.com
+configuration.network_name = RAPIDSPACE
+configuration.domain = rapid.space
 configuration.mme_ws_port = 9000
 configuration.enb_ws_port = 9002
 configuration.ims_ws_port = 9003
@@ -60,17 +60,27 @@ extensions = jinja2.ext.do
 context =
   section directory directory
 
+[lte-enb-sh-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:_buildout_section_name_}
+mode = 0700
+template =
+  inline:#!/bin/sh
+  sudo /opt/amarisoft/rm-tmp-lte | true;
+  {{ enb }}/lteenb ${directory:etc}/gnb.cfg >> ${directory:log}/gnb-output.cfg 2>> ${directory:log}/gnb-output.cfg
+
 ### eNodeB (enb)
 [lte-enb-service]
 recipe = slapos.cookbook:wrapper
 init = ${ltelogs:rendered} ${directory:log}/enb.log; sleep 2
-command-line = {{ enb }}/lteenb ${directory:etc}/gnb.cfg
+command-line = ${lte-enb-sh-wrapper:rendered}
 wrapper-path = ${directory:service}/lte-enb
 mode = 0775
 reserve-cpu = True
 pidfile = ${directory:run}/enb.pid
 hash-files =
   ${lte-gnb-config:rendered}
+  ${lte-enb-sh-wrapper:rendered}
 environment =
   LD_LIBRARY_PATH={{ openssl_location }}/lib
   AMARISOFT_PATH=/opt/amarisoft/.amarisoft
@@ -83,6 +93,7 @@ context =
   section directory directory
   section slap_configuration slap-configuration
   key slapparameter_dict slap-configuration:configuration
+  raw gtp_addr {{ local_ipv4 }}
   import  netaddr netaddr
 
 [lte-gnb-config]
@@ -91,9 +102,21 @@ template = {{ gnb_template }}
 rendered = ${directory:etc}/gnb.cfg
 
 [publish-connection-information]
-recipe = slapos.cookbook:publish.serialised
-monitor-base-url = ${monitor-instance-parameter:monitor-base-url}
+<= monitor-publish
+recipe = slapos.cookbook:publish
+ipv4 = {{ local_ipv4 }}
 
 [monitor-instance-parameter]
+{% if slapparameter_dict.get("name", None) %}
 monitor-title = {{ slapparameter_dict['name'] | string }}
+{% endif %}
+{% if slapparameter_dict.get("monitor-password", None) %}
 password = {{ slapparameter_dict['monitor-password'] | string }}
+{% endif %}
+
+# Add custom promise to check if /dev/sdr0 is busy
+[sdr-busy-promise]
+recipe = slapos.cookbook:promise.plugin
+eggs = slapos.core
+file = {{ sdr_busy_promise }}
+output = ${directory:plugins}/check-sdr-busy.py

software/ors-amarisoft/instance-tdd1900-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd1900-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd2600-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3500-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-enb-epc-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-enb-input-schema.json

@@ -26,6 +26,18 @@
       "description": "number of DL resource blocks",
       "type": "number",
       "default": 100
+    },
+    "mme_addr": {
+      "title": "MME Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "enb_id": {
+      "title": "eNB ID",
+      "description": "eNB ID",
+      "type": "string",
+      "default": "0x1A2D0"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-gnb-epc-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance-tdd3700-gnb-input-schema.json

@@ -32,6 +32,18 @@
       "description": "Downlink Bandwidth (in MHz)",
       "type": "number",
       "default": 40
+    },
+    "amf_addr": {
+      "title": "AMF Address",
+      "description": "IPv4 of the core network",
+      "type": "string",
+      "default": "127.0.0.100"
+    },
+    "gnb_id": {
+      "title": "gNB ID",
+      "description": "gNB ID",
+      "type": "string",
+      "default": "0x12345"
     }
   }
 }
\ No newline at end of file

software/ors-amarisoft/instance.cfg

@@ -7,7 +7,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
@@ -36,6 +36,20 @@ gnb = dynamic-template-lte-gnb:rendered
 epc = dynamic-template-lte-epc:rendered
 RootSoftwareInstance = $${:enb-epc}
 
+[local-ipv4-address]
+recipe = slapos.recipe.build
+init = 
+  import socket
+  s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+  try:
+      s.connect(('1.1.1.1', 1))
+      ip = s.getsockname()[0]
+  except Exception:
+      ip = '127.0.0.1'
+  finally:
+      s.close()
+  options['local-ipv4'] = ip
+
 [dynamic-template-lte-enb-epc]
 < = jinja2-template-base
 template = ${template-lte-enb-epc:target}
@@ -43,6 +57,7 @@ filename = instance-lte-enb-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    key local_ipv4 local-ipv4-address:local-ipv4
  
 [dynamic-template-lte-gnb-epc]
 < = jinja2-template-base
@@ -51,6 +66,7 @@ filename = instance-lte-gnb-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    key local_ipv4 local-ipv4-address:local-ipv4
  
 [dynamic-template-lte-enb]
 < = jinja2-template-base
@@ -62,6 +78,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw enb_template   ${enb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_earfcn ${enb:default-dl-earfcn}
     raw default_lte_dl_freq ${enb:default-lte-dl-freq}
@@ -71,6 +88,7 @@ extra-context =
     raw default_lte_rx_gain ${enb:default-lte-rx-gain}
     raw min_frequency ${enb:min-frequency}
     raw max_frequency ${enb:max-frequency}
+    key local_ipv4 local-ipv4-address:local-ipv4
 
 [dynamic-template-lte-gnb]
 < = jinja2-template-base
@@ -82,6 +100,7 @@ extra-context =
     raw enb   ${enb:destination}
     raw gnb_template   ${gnb.jinja2.cfg:target}
     raw ltelogs_template   ${ltelogs.jinja2.sh:target}
+    raw sdr_busy_promise   ${sdr-busy-promise:target}
     raw openssl_location ${openssl:location}
     raw default_dl_nr_arfcn ${enb:default-dl-nr-arfcn}
     raw default_nr_band ${enb:default-nr-band}
@@ -91,6 +110,7 @@ extra-context =
     raw default_nr_rx_gain ${enb:default-nr-rx-gain}
     raw min_frequency ${enb:min-frequency}
     raw max_frequency ${enb:max-frequency}
+    key local_ipv4 local-ipv4-address:local-ipv4
 
 [dynamic-template-lte-epc]
 < = jinja2-template-base
@@ -99,6 +119,7 @@ filename = instance-lte-epc.cfg
 extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:rendered}
+    raw interface_up_promise   ${interface-up-promise:target}
     raw mme   ${mme:destination}
     raw mme_template   ${mme.jinja2.cfg:target}
     raw ims_template   ${ims.jinja2.cfg:target}

software/ors-amarisoft/promise/check_interface_up.py

+import socket
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    ifname = self.getConfig('ifname')
+
+    f = open('/sys/class/net/%s/operstate' % ifname, 'r')
+    if f.read() == 'up\n':
+      self.logger.info("%s is up", ifname)
+    else:
+      self.logger.error("%s is down", ifname)
+    f.close()
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/promise/check_sdr_busy.py

+import os
+import errno
+
+from zope.interface import implementer
+from slapos.grid.promise import interface
+from slapos.grid.promise.generic import GenericPromise
+
+
+
+
+@implementer(interface.IPromise)
+class RunPromise(GenericPromise):
+
+  def __init__(self, config):
+    """
+      Called when initialising the promise before testing.
+      Sets the configuration and the periodicity.
+    """
+    super(RunPromise, self).__init__(config)
+    self.setPeriodicity(minute=2)
+
+
+  def sense(self):
+    """
+      Called every time the promise is tested.
+      Signals a positive or negative result.
+
+      In this case, check whether the file exists.
+    """
+    sdr_dev = '/dev/sdr0'
+    try:
+      open(sdr_dev, 'w').close()
+      self.logger.error("eNB is not using %s", sdr_dev)
+    except IOError as e:
+      if e.errno == errno.EBUSY:
+        self.logger.info("eNB is using %s", sdr_dev)
+
+  def test(self):
+    """
+      Called after sense() if the instance is still converging.
+      Returns success or failure based on sense results.
+
+      In this case, fail if the previous sensor result is negative.
+    """
+    return self._test(result_count=1, failure_amount=1)
+
+
+  def anomaly(self):
+    """
+      Called after sense() if the instance has finished converging.
+      Returns success or failure based on sense results.
+      Failure signals the instance has diverged.
+
+      In this case, fail if two out of the last three results are negative.
+    """
+    return self._anomaly(result_count=3, failure_amount=2)

software/ors-amarisoft/software.cfg

@@ -49,6 +49,12 @@ url = ${:_profile_base_location_}/${:_update_hash_filename_}
 [template-lte-epc]
 <= download-base
 
+[sdr-busy-promise]
+<= download-base
+
+[interface-up-promise]
+<= download-base
+
 [amarisoft]
 recipe = slapos.recipe.build
 path = /opt/amarisoft/lte

software/repman/buildout.hash.cfg

@@ -14,19 +14,19 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 8a08be95a04f1a47098c4fdef80bdfed
+md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 697a1b546c883da45c14dbcd2d73b2b9
+md5sum = 7782adde357c9444fed51f2dcc230a52
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 2ee2c4bc5f985c11c2167b819d261256
+md5sum = 0a4ce036ea524373a85fac10e81edb96
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = c78c6be9537d3dffbb526b0e819bacb5
+md5sum = 8cdeb63a2acc294801266f24599f6d32
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = a5c204cac552754520aee0570d379723
+md5sum = 938f1e8087395757f5ae861d20500658
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = df44c2d6fb1971df582345daedad280c
+md5sum = 47b76144e1b116580c8acf08274af976
+
+[template-publish-slave-information]
+_update_hash_filename_ = publish-database-slave-parameters.cfg.in
+md5sum = 9616477ff993c55c2b43bf8797db6994
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
-md5sum = d8a0a7d72b02f911dafda655941b805d
+md5sum = 6e5c02542568f250aacd84694249efc0
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in
-md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
+md5sum = c203f40a58386310a433b58fd345a341
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = d2ebd2ec55bf8489789a52c808729925
+md5sum = d5c19d866c5a76d3bc2f434a91b1553a
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in
-md5sum = e9bcee5dc1318fe3acda2663472214f5
+md5sum = 8f14054b3761b167cf8a86f4f084a574
 
 [proxy-need-start-stop.sh.in]
 _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
 md5sum = 455aaf369bf5141758dc57f2c0e67b08
+
+[slave-db-manage.in]
+_update_hash_filename_ = templates/slave-db-manage.in
+md5sum = b45313ae5fb06972cc7fc945e34e434a

software/repman/instance-mariadb.cfg.jinja2.in

@@ -17,17 +17,11 @@
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
--extends = publish-early
 database-host = {{ host }}:{{ port }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 partition-path = ${buildout:directory}
 receiver-port = ${dbjob-parameter:socat-port}
-
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-
-database-list = {{ dumps(database_list) }}
+csr-id = ${caucase-csr-id:csr-id}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -37,6 +31,22 @@ mode = 644
 < = jinja2-template-base
 mode = 755
 
+[jinja2-template-script-base]
+< = jinja2-template-executable
+context =
+  key tmp_dir            directory:tmp
+  key partition_dir      buildout:directory
+  raw username           {{ slapparameter_dict['repman-user'] }}
+  raw password           {{ slapparameter_dict['repman-passwd'] }}
+  raw cluster_name       {{ slapparameter_dict['cluster'] }}
+  raw secure_url         {{ slapparameter_dict['repman-secure-url'] }}
+  raw mysql_conf         ${directory:etc}/mysql/my.cnf
+  raw dash_bin           {{ dash }}
+  raw jq_bin             {{ jq_bin }}
+  raw curl_bin           {{ curl_bin }}
+  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  ${:extra-context}
+
 [simplefile]
 < = jinja2-template-base
 template = inline:{{ '{{ content }}' }}
@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+{% import "caucase" as caucase with context %}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-updater',
+     url=slapparameter_dict['caucase-url'],
+     data_dir='${mysql-directory:ssl}',
+     crt_path='${mysql-directory:ssl}/server-cert.pem',
+     ca_path='${mysql-directory:ssl}/ca-cert.pem',
+     crl_path='${mysql-directory:ssl}/crl.pem',
+     key_path='${mysql-directory:ssl}/server-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-updater') -%}
+{% do part_list.append('caucase-updater-promise') -%}
+
+[get-csr-id]
+recipe = plone.recipe.command
+output = ${directory:tmp}/csr_id
+command = 
+  if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
+    RESULT="None";
+  else
+    if [ -f "${caucase-updater-csr:csr}" ]; then
+      RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
+      if [ ! $? -eq 0 ]; then
+        RESULT="None";
+      fi
+    fi
+  fi
+  cat <<EOF > ${:output}
+  [caucase]
+  csr-id = $(echo $RESULT)
+  EOF
+update-command = ${:command}
+
+[caucase-csr-id]
+recipe = slapos.cookbook:zero-knowledge.read
+file-path = ${get-csr-id:output}
+csr-id =
+
 [my-cnf-parameters]
 socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
-data-directory = ${directory:srv}/mariadb
+data-directory = ${mysql-directory:data}
 pid-file = ${directory:run}/mariadb.pid
 plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
 groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-ssl-crt = ${directory:mariadb-ssl}/crt.pem
-ssl-key = ${directory:mariadb-ssl}/key.pem
-ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
 
 [my-cnf]
 < = jinja2-template-base
@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
 mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
+require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
 
 [init-script]
 < = jinja2-template-executable
+extensions = jinja2.ext.do
 # XXX: is there a better location ?
 rendered = ${directory:etc}/mariadb_initial_setup.sql
 template = {{ parameter_dict['template-mariadb-initial-setup'] }}
@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
 mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 
 [{{ section('update-mysql-script') }}]
-< = jinja2-template-executable
+< = jinja2-template-script-base
 rendered = ${directory:scripts}/mariadb_update
 init-password = ${directory:etc}/.init-passwd.done
 upgrade-done = ${directory:lib}/mariadb-update-done
-context =
+extra-context =
   key init_password_done :init-password
   key upgrade_done       :upgrade-done
   key init_root_sql      init-root-sql:rendered
   key mysql_update       update-mysql:output
-  raw mysql_conf         ${directory:etc}/mysql/my.cnf
-  raw dash_bin           {{ dash }}
-  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  key init_database_sql  init-script:rendered
 template = {{ parameter_dict['template-init-root-wrapper'] }}
 
+[{{ section('mysql-slave-db-cleanup') }}]
+< = jinja2-template-script-base
+rendered = ${directory:scripts}/manage-slave-db
+db-name = {{ slapparameter_dict['database-name'] }}
+extra-context =
+  key database_name      :db-name
+template = {{ parameter_dict['template-manage-db'] }}
+
 [mysqld]
 < = jinja2-template-executable
 rendered = ${directory:bin}/mysqld
@@ -136,19 +195,19 @@ environ =
   {{ variable }}
   {%- endfor %}
 
-[ca-mysqld]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-key-file = ${my-cnf-parameters:ssl-key}
-cert-file = ${my-cnf-parameters:ssl-crt}
-executable = ${mysqld:rendered}
-wrapper = ${directory:controller}/mariadb
+[mysqld-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = ${mysqld:rendered}
+wrapper-path = ${directory:controller}/mariadb
+wait-for-files =
+  ${mysql-directory:mysql}/my.cnf
+  ${mysql-directory:ssl}/server-cert.pem
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
 {% do part_list.append("supervisord-mariadb-ctl") -%}
-{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
-  "stopwaitsecs": 86400, "environment": [],
+{% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
+  "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${directory:log}/mariadb_stdout.log",
   "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
 
@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
 
 [binary-wrap-base]
-recipe = slapos.cookbook:wrapper
 # Note: --defaults-file must be the first argument, otherwise wrapped binary
 # will reject it.
-command-line = 
-  "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
-  --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
-wrapper-path = ${directory:bin}/${:command}
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:command}
+mode = 755
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+    --protocol=socket \
+  ${:extra-args} "$@"
+wrapper-path = ${:rendered}
 extra-args =
 
 [binary-wrap-mysql]
@@ -208,6 +272,11 @@ command = mysqladmin
 <= binary-wrap-base
 command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
 command = mysql_tzinfo_to_sql
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+  --skip-write-binlog "$@"
 
 [binary-wrap-pt-digest]
 <= binary-wrap-base
@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
-mariadb-backup-full = ${:backup}/mariadb-full
-mariadb-backup-incremental = ${:backup}/mariadb-incremental
-mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 lib = ${:var}/lib
-mysql = ${:lib}/mysql
 log = ${:var}/log
 run = ${:var}/run
 config-tmp = ${:tmp}/config
-custom = ${directory:etc}/mysql/custom
+
+[mysql-directory]
+recipe = slapos.cookbook:mkdirectory
+mysql = ${directory:etc}/mysql
+ssl = ${:mysql}/ssl
+data = ${directory:lib}/mysql
+custom = ${:mysql}/custom.d
 
 [dbjob-parameter]
 bash-bin = {{ bash_bin }}
 db-user = ${init-script-parameters:root-user}
 db-password = ${init-script-parameters:password}
-mysql-dir = ${directory:mysql}
-dbjob-cnf = ${directory:etc}/mysql/my.cnf
-log-dir = ${directory:mysql}/.system/logs
+mysql-dir = ${mysql-directory:data}
+dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
+log-dir = ${mysql-directory:data}/.system/logs
 tmp-dir = ${directory:tmp}
 mysqld-socket  = ${my-cnf-parameters:socket}
+mysql-client = ${binary-wrap-mysql:wrapper-path}
 socat-port = {{ int(port) + 9 }}
 restart-script = ${mysqld-restart-script:rendered}
 socat-location = {{ parameter_dict['socat-location'] }}
@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
   cd ${directory:config-tmp} &&
   {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
   tar -xzf config.tar.gz
-  cp -r data/.system ${directory:mysql}
-  rm -rf ${directory:etc}/mysql
-  cp -r etc/mysql ${directory:etc}
-  ln -sf ${directory:mysql}/.system ${directory:var}/system
-  ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
+  rm -r etc/mysql/ssl  # we don't need generated ssl files
+  cp -r data/.system ${mysql-directory:data}
+  cp -r etc/mysql/* ${mysql-directory:mysql}
+  ln -sf ${mysql-directory:data}/.system ${directory:var}/system
+  ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
 
 [{{ section('install-mysql-config') }}]
 recipe = plone.recipe.command

software/repman/instance-repman-input-schema.json

@@ -89,33 +89,6 @@
       "type": "string",
       "default": ""
     },
-    "tags": {
-      "title": "Provisioning db tag list",
-      "description": "Provisioning db tags. Set one tag per line.",
-      "type": "array",
-      "items": {
-        "type": "string"
-      },
-      "default": [
-        "gtidstrict",
-        "bind",
-        "pkg",
-        "innodb",
-        "noquerycache",
-        "slow",
-        "pfs",
-        "linux",
-        "readonly",
-        "diskmonitor",
-        "sqlerror",
-        "compressbinlog",
-        "bm4ci",
-        "mroonga",
-        "utctime",
-        "readcommitted",
-        "nohandshake"
-      ]
-    },
     "http-session-lifetime": {
       "title": "Web Session life time in s",
       "description": "Web interface Session life time in seconds. Default 86400",
@@ -128,17 +101,11 @@
       "type": "integer",
       "default": 4
     },
-    "autorejoin": {
-      "title": "Automatic rejoin a failed master",
-      "description": "Automatic rejoin a failed master (default true)",
+    "require-ssl": {
+      "title": "Enable REQUIRE SSL for repman database users",
+      "description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
       "type": "boolean",
-      "default": true
-    },
-    "autoseed": {
-      "title": "Automatic join a standalone node",
-      "description": "Automatic join a standalone node",
-      "type": "boolean",
-      "default": true
+      "default": false
     },
     "repman-cluster-dict": {
       "title": "Replication Manager clusters definition",
@@ -189,6 +156,34 @@
               "type": "string",
               "default": ""
             },
+            "tags": {
+              "title": "Provisioning db tag list",
+              "description": "Provisioning db tags. Set one tag per line.",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "default": [
+                "gtidstrict",
+                "bind",
+                "pkg",
+                "innodb",
+                "noquerycache",
+                "slow",
+                "pfs",
+                "linux",
+                "readonly",
+                "diskmonitor",
+                "sqlerror",
+                "compressbinlog",
+                "bm4ci",
+                "mroonga",
+                "utctime",
+                "readcommitted",
+                "nohandshake",
+                "ssl"
+              ]
+            },
             "proxysql-user": {
               "title": "Proxysql username",
               "description": "Proxysql external user, default is 'external'.",
@@ -197,7 +192,7 @@
             },
             "proxy-tags": {
               "title": "Proxy tag list",
-              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
+              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
               "type": "array",
               "items": {
                 "type": "string"
@@ -206,7 +201,8 @@
                 "pkg",
                 "masterslave",
                 "linux",
-                "noreadwritesplit"
+                "noreadwritesplit",
+                "ssl"
               ]
             },
             "logical-backup-cron": {
@@ -302,6 +298,18 @@
               "type": "boolean",
               "default": true
             },
+            "autorejoin": {
+              "title": "Automatic rejoin a failed master db",
+              "description": "Automatic rejoin a failed master (default: true).",
+              "type": "boolean",
+              "default": true
+            },
+            "autoseed": {
+              "title": "Automatic join a standalone node",
+              "description": "Automatic join a standalone mariadb node to the cluster.",
+              "type": "boolean",
+              "default": true
+            },
             "failover-mode": {
               "title": "Failover mode",
               "description": "Failover is manual or automatic (default \"manual\").",

software/repman/instance-repman-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "properties": {
+    "db_user": {
+      "description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
+      "title": "Database User",
+      "type": "string",
+      "default": "user"
+    },
+    "db_password": {
+      "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
+      "title": "Initial database password",
+      "type": "string",
+      "default": ""
+    },
+    "db_charset": {
+      "description": "Database Character set information. This parameter is set only if database is not created yet.",
+      "title": "Database Character set information",
+      "type": "string",
+      "enum": [
+        "latin1",
+        "latin2",
+        "latin5",
+        "latin7",
+        "ascii",
+        "latin5",
+        "utf8",
+        "utf8mb4",
+        "utf16",
+        "utf16le",
+        "utf32",
+        "binary",
+        "utf8_general_ci",
+        "utf8_unicode_ci",
+        "utf8_unicode_ci",
+        "utf8mb4_bin",
+        "utf8mb4_general_ci",
+        "utf8mb4_unicode_ci",
+        "latin1_general_ci",
+        "latin1_bin",
+        "ascii_general_ci",
+        "ascii_bin"
+      ],
+      "default": ""
+    }
+  },
+  "title": "Input Parameters",
+  "type": "object"
+}

software/repman/instance-repman.cfg.jinja2.in

@@ -8,8 +8,23 @@
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set cluster_list = [] -%}
-{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%}
+{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
+  "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
+  "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
+{% set database_slave_list = [] -%}
+{% set db_name_list = [] -%}
+{% set count = namespace(value=2) %}
+{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
+{% set caucase_url = 'http://' ~ caucase_bind -%}
+{% set csrid_list = [] -%}
+
+{% macro password(name) -%}
+[{{ name }}-password]
+recipe = slapos.cookbook:generate.password
+bytes = 12
+
+{% endmacro -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -25,11 +40,25 @@ log = ${:var}/log
 data = ${:var}/lib
 nginx-prefix = ${:var}/nginx
 tmp = ${:home}/tmp
+backup-caucased = ${:srv}/backup-caucased
+caucased = ${:srv}/caucased
+ssl = ${:etc}/ssl
+
+[proxysql-directory]
+recipe = slapos.cookbook:mkdirectory
+config = ${directory:etc}/proxysql
+ssl = ${:config}/ssl
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {% set proxysql_controller = "proxysql-ctl" -%}
 {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+
+[proxysql-controller-update]
+recipe = slapos.cookbook:wrapper
+command-line = ${proxysql-ctl-bin:wrapper-path} update
+wrapper-path = ${directory:scripts}/proxysql-ctl-update
 {% do part_list.append("supervisord-proxysql-ctl") -%}
+{% do part_list.append("proxysql-controller-update") -%}
 
 [request-common]
 recipe = slapos.cookbook:request.serialised
@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
+config-caucase-url = {{ caucase_url }}
 
 [download-proxy-config]
 recipe = slapos.recipe.template:jinja2
@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
   PORT=$3
   CONFIG=$4
   if [ -z "$CONFIG" ]; then
-    CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
+    CONFIG="${proxysql-directory:config}/proxysql.cnf"
   fi
   mkdir -p ${repman:config-tmp}/proxies
   cd ${repman:config-tmp}/proxies
   {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
   tar -xzf proxies-$NAME.tar.gz
-  cp conf/proxysql.cnf $CONFIG
+  cp etc/proxysql/proxysql.cnf $CONFIG
 rendered = ${directory:bin}/update-proxysql-config
 mode = 755
 
+{% for instance_dict in slave_instance_list -%}
+{% set slave_dict = {
+  'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
+  'user': instance_dict.get('db_user', 'user'),
+  'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
+  'slave_reference': instance_dict['slave_reference'],
+  'charset': instance_dict.get('db_charset', ''),
+  'require_ssl': True
+  } -%}
+{% do database_slave_list.append(slave_dict) -%}
+{% do db_name_list.append(slave_dict['name']) -%}
+{{ password(instance_dict['slave_reference']) }}
+{% endfor %}
+[database-slave-information]
+{% for slave_dict in database_slave_list -%}
+{{ slave_dict['name'] }} = !py!{{ slave_dict }}
+{% endfor %}
+{% set db_list = db_name_list | join(' ') -%}
+
 {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
 {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
   "database-amount": 2, "proxysql-user": "external",
@@ -67,22 +116,23 @@ mode = 755
 {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
 {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
 
-{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
 # Request mariadb instances
 {% set db_amount = parameter_dict.get('database-amount', 2) -%}
 {% if db_amount < 2 -%}
 {% set db_amount = 2 -%}
 {% endif -%}
 
+{% set count.value = count.value + db_amount %}
 {% for i in range(0, db_amount) -%}
 {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
 {% set section = 'request-mariadb-' ~ i -%}
-{% set dbname = 'Mariadb-' ~ i -%}
+{% set dbname = 'Mariadb-' ~ i %}
+
 
 [{{ section }}]
 <= request-common
 software-type = mariadb
-name = {{ dbname }}
+name = {{ dbname ~ '-' ~ name}}
 sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
 {% for key, value in mariadb_dict.items() -%}
 config-{{ key }} = {{ dumps(value) }}
@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
 config-root-password = ${publish-early:db-root-password}
 config-repman-user = ${repman-parameter:username}
 config-heartbeat-user = ${repman-parameter:heartbeat-user}
-#config-repman-passwd =  ${repman-parameter:password}
+config-repman-passwd =  ${repman-parameter:password}
 config-repman-url = ${nginx-parameter:backend-url}
 config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
 config-cluster = {{ name }}
 config-name = {{ dbname }}
-config-database-list = {{ dumps(database_list) }}
-return = 
+config-database-list = !py!{{ database_slave_list }}
+config-database-name = {{ dumps(db_list) }}
+config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
+return =
   database-host
   receiver-port
   monitor-base-url
   partition-path
+  csr-id
 
 {% do part_list.append(section) -%}
 {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
 {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
 {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
 {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
 
 {% endfor -%}
 
@@ -129,6 +183,7 @@ maximum = 7132
 {% for key, value in parameter_dict.items() -%}
 {{ key }} = {{ value }}
 {% endfor -%}
+name = {{ name }}
 proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
 proxy-port = {{ '${' ~ name ~ '-port:port}' }}
 proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
@@ -136,6 +191,8 @@ db-user = repman
 db-password = ${publish-early:db-root-password}
 db-list = {{ mariadb_server_list | join(',') }}
 
+autorejoin = {{ parameter_dict.get("autorejoin", True) }}
+autoseed = {{ parameter_dict.get("autoseed", True) }}
 heartbeat-user = ${repman-parameter:heartbeat-user}
 heartbeat-password = ${publish-early:db-root-password}
 partition-list = {{ mariadb_path_list | join(',') }}
@@ -150,8 +207,8 @@ proxysql-servers-ipv6 = [{{ ip }}]
 password = ${repman-parameter:password}
 proxysql-partition = ${buildout:directory}
 receiver-port-list = {{ receiver_port_list | join(',') }}
-enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
-proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }}
+enabled-tags = {{ parameter_dict.get("tag-list", tag_list) | join(',') }}
+proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
 logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
 physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
 proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
@@ -191,7 +248,7 @@ context =
 recipe = plone.recipe.command
 # if Repman is not started, cannot download config from server
 stop-on-error = false
-config = ${repman:proxies}/proxysql-{{ name }}.cnf
+config = ${proxysql-directory:config}/proxysql.cnf
 data = ${repman:proxy-data}/{{ name }}
 command =
   mkdir -p ${:data} &&
@@ -209,6 +266,7 @@ command-line =
 wrapper-path = ${directory:controller}/proxysql-{{ name }}
 wait-for-files =
   ${repman:bootstrap}/{{ name }}_bootstrapped
+  ${config-proxysql-{{ name }}:config}
 depends =
   {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
   {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
@@ -237,7 +295,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 {% set service_name = "proxysql-" ~ name -%}
 {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
-  "stopwaitsecs": 60, "environment": [],
+  "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
   "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
 
@@ -247,22 +305,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 {% do part_list.append('config-' ~ name) -%}
 {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
 
-{% set publish_database_list = [] -%}
-{% set publish_database_v6_list = [] -%}
-{% for database in database_list -%}
-{%   if database.get('user') -%}
-{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%     do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%   else -%}
-{%     do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%     do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%   endif -%}
-{% endfor -%}
 
-{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
-{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
+[{{ name}}-publish-slave-information]
+recipe = slapos.cookbook:switch-softwaretype
+default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
+RootSoftwareInstance = ${:default}
+
+[dynamic-{{ name}}-publish-slave-information]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/database-publish-slave-information.cfg
+extensions = jinja2.ext.do
+template = {{ template_publish_slave_information }}
+context =
+    import json_module          json
+    key proxy_port              {{ name }}-cluster-parameter:proxy-port
+    raw eggs_directory          {{ eggs_directory }}
+    raw develop_eggs_directory  {{ develop_eggs_directory }}
+    raw ipv6                    {{ ip }}
+    raw ipv4                    {{ ipv4 }}
+    section slave_dict          database-slave-information
+
+{% do part_list.append(name ~ '-publish-slave-information') -%}
 {% endfor -%}
 
+# deploy caucase
+{% import "caucase" as caucase with context %}
+{{ caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout_bin_directory,
+    caucased_path='${directory:service}/caucased',
+    backup_dir='${directory:backup-caucased}',
+    data_dir='${directory:caucased}',
+    netloc=caucase_bind,
+    tmp='${directory:tmp}',
+    service_auto_approve_count=count.value,
+    user_auto_approve_count=1,
+    key_len=2048,
+)}}
+{% do part_list.append('caucased') -%}
+{% do part_list.append('caucased-promise') -%}
+{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-updater',
+     url=caucase_url,
+     data_dir='${directory:srv}/caucase-updater',
+     crt_path='${directory:ssl}/repman-cert.crt',
+     ca_path='${directory:srv}/caucase-updater/ca.crt',
+     crl_path='${directory:srv}/caucase-updater/crl.pem',
+     key_path='${directory:ssl}/repman-cert.key',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-updater') -%}
+{% do part_list.append('caucase-updater-promise') -%}
+
+#caucase user certificate
+{{ caucase.updater(
+     prefix='caucase-user-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-user-updater',
+     url=caucase_url,
+     data_dir='${directory:srv}/caucase-user-updater',
+     crt_path='${directory:ssl}/caucase.user.crt',
+     ca_path='${directory:srv}/caucase-user-updater/ca.crt',
+     crl_path='${directory:srv}/caucase-user-updater/crl.pem',
+     key_path='${directory:ssl}/caucase.user.key',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+     mode='user',
+)}}
+{% do part_list.append('caucase-user-updater') -%}
+{% do part_list.append('caucase-user-updater-promise') -%}
+
+#caucase proxysql certificate
+{{ caucase.updater(
+     prefix='caucase-proxysql-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-proxysql-updater',
+     url=caucase_url,
+     data_dir='${proxysql-directory:ssl}',
+     crt_path='${proxysql-directory:ssl}/client-cert.pem',
+     ca_path='${proxysql-directory:ssl}/ca-cert.pem',
+     crl_path='${proxysql-directory:ssl}/crl.pem',
+     key_path='${proxysql-directory:ssl}/client-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-proxysql-updater') -%}
+{% do part_list.append('caucase-proxysql-updater-promise') -%}
+
+[caucase-sign-csr]
+recipe = slapos.recipe.template:jinja2
+mode = 755
+rendered = ${directory:bin}/caucase-sign
+template =
+  inline:#!/bin/sh
+  cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
+  cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
+  for csr_id in {{ csrid_list | join(' ') }}; do
+    if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
+      continue
+    fi
+    {{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
+  done
+
+[cron-caucase-sign-csr]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = caucase-sign-csr
+frequency = * * * * *
+command = ${caucase-sign-csr:rendered}
+{% do part_list.append('cron-caucase-sign-csr') -%}
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = ${slap-connection:computer-id}
@@ -286,7 +448,6 @@ etc      = ${directory:etc}/repman
 data-dir = ${directory:var}/lib
 root-dir = ${directory:srv}/repman
 clusters = ${:etc}/cluster.d
-proxies  = ${:etc}/proxy
 proxy-data = ${:data-dir}/proxy
 config-tmp = ${directory:tmp}/config
 bootstrap = ${:etc}/bootstrap
@@ -296,8 +457,9 @@ proxies-log = ${directory:log}/proxy
 ipv6 = ${instance-parameter:ipv6-random}
 port = ${instance-parameter:nginx-port}
 ssl-port = ${instance-parameter:nginx-ssl-port}
-ssl-certificate = ${ca-nginx:cert-file}
-ssl-key = ${ca-nginx:key-file}
+ssl-certificate = ${directory:ssl}/repman-cert.crt
+ca-certificate = ${directory:srv}/caucase-updater/ca.crt
+ssl-key = ${directory:ssl}/repman-cert.key
 pid-file = ${directory:run}/nginx.pid
 access-log = ${directory:log}/nginx_access.log
 error-log = ${directory:log}/nginx_error.log
@@ -326,8 +488,6 @@ username = admin
 heartbeat-user = heartbeat
 password = ${publish-early:repman-password}
 cluster-d = ${repman:clusters}
-autorejoin = {{ slapparameter_dict.get("autorejoin", True) }}
-autoseed = {{ slapparameter_dict.get("autoseed", True) }}
 mysql-bin-dir = {{ mariadb_location }}/bin
 mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
 mysqlclient-path = {{ mariadb_location }}/bin/mysql
@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
 
 http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
 http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
+ssl-cert = ${directory:ssl}/repman-cert.crt
+ssl-key = ${directory:ssl}/repman-cert.key
 
 [repman-config-folder]
 recipe = plone.recipe.command
@@ -392,6 +554,9 @@ depends =
   ${replication-manager-reload:recipe}
   ${repman-listen-promise:recipe}
   ${repman-listen-ssl-promise:recipe}
+wait-for-files =
+  ${directory:ssl}/repman-cert.crt
+  ${directory:ssl}/repman-cert.key
 
 [repman-config.toml]
 recipe = slapos.recipe.template:jinja2
@@ -428,11 +593,12 @@ command-line =
   {{ nginx_bin }}
   -p ${directory:nginx-prefix}
   -c ${nginx-conf:rendered}
-wrapper-path = ${directory:bin}/nginx-start
+wrapper-path = ${directory:services}/nginx
 wait-for-files =
-  ${ca-directory:certs}/nginx.key
-  ${ca-directory:certs}/nginx.crt
-  ${nginx-graceful-wrapper:rendered}
+  ${directory:ssl}/repman-cert.crt
+  ${directory:srv}/caucase-updater/ca.crt
+  ${directory:ssl}/repman-cert.key
+
 
 [nginx-graceful-wrapper]
 recipe = slapos.recipe.template:jinja2
@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
 context =
 mode = 755
 
-[ca-nginx]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-cert-file = ${ca-directory:certs}/nginx.crt
-key-file = ${ca-directory:certs}/nginx.key
-executable = ${nginx-launcher:wrapper-path}
-wrapper = ${directory:bin}/ca-nginx
-
-[ca-nginx-service]
-recipe = slapos.cookbook:wrapper
-command-line = ${ca-nginx:wrapper}
-wrapper-path = ${directory:services}/nginx
-hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-
 [logrotate-entry-nginx]
 <= logrotate-entry-base
 name = nginx
@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
   monitor-password monitor-htpasswd:passwd
   db-root-password gen-root-password:passwd
   repman-password repman-password:passwd
+database-list = {{ db_list }}
 
 [publish-connection-parameter]
 <= monitor-publish
@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
 backend-url = ${nginx-parameter:backend-ssl-url}
 url = ${repman-frontend:connection-secure_access}
 username = ${repman-parameter:username}
+database-list = {{ db_list }}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
 # Deploy replication-manager instance
 #
 #############################
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
 [buildout]
 extends =
   {{ template_monitor }}
@@ -552,7 +715,7 @@ parts =
   replication-manager
   monitor-base
   logrotate-entry-nginx
-  ca-nginx-service
+  nginx-launcher
   publish-connection-parameter
   repman-frontend-promise
   repman-backend-promise

software/repman/instance.cfg.in

@@ -28,8 +28,10 @@ mode    = 0644
 extensions = jinja2.ext.do
 rendered= ${buildout:directory}/${:_buildout_section_name_}
 supervisord-lib = {{ supervisord_lib }}
+causace-lib = {{ caucase_library }}
 import-list =
   file supervisord_lib :supervisord-lib
+  file caucase         :causace-lib
 context =
     key slapparameter_dict      slap-configuration:configuration
     key computer_id             slap-configuration:computer
@@ -48,6 +50,8 @@ context =
     raw bash_bin                {{ bash_location }}/bin/bash
     raw jq_bin                  {{ jq_location }}/bin/jq
     raw curl_bin                {{ curl_location }}/bin/curl
+    raw openssl_bin             {{ openssl_location }}/bin/openssl
+    raw caucase_bin_client      {{ caucase_bin_client }}
 
     ${:extra-context}
 extra-context =
@@ -56,6 +60,7 @@ extra-context =
 <= jinja2-template-base
 template= {{ template_repman_cfg }}
 extra-context =
+    key slave_instance_list         slap-configuration:slave-instance-list
     raw gowork_bin                  {{ gowork_bin }}
     raw haproxy_location            {{ haproxy_location }}
     raw nginx_bin                   {{ nginx_location }}/sbin/nginx
@@ -71,6 +76,7 @@ extra-context =
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}
     raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
+    raw template_publish_slave_information {{ template_publish_slave }}
 
 [template-mariadb-parameters]
 bash = {{ bash_location }}
@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
 template-init-root-wrapper = {{ template_init_root_wrapper }}
 template-mysqld-wrapper = {{ template_mysqld_wrapper }}
 template-mysqld-need-start = {{ mysqld_start_template }}
+template-manage-db = {{ template_slave_db_manage }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
 bin-directory = {{ bin_directory }}

software/repman/publish-database-slave-parameters.cfg.in

+{% set part_list = [] -%}
+
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
+
+{% for _, database in slave_dict.items() -%}
+{% set slave_reference = database['slave_reference'] -%}
+{% do part_list.append('publish-slave-' ~ slave_reference) -%}
+
+[publish-slave-{{ slave_reference }}]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_reference }}
+{%   if database.get('user') -%}
+url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
+url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
+{%   else -%}
+url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
+url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
+{%   endif -%}
+{% endfor %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+parts =
+{% for part in part_list -%}
+{{ '  %s' % part }}
+{% endfor -%}

software/repman/software.cfg

@@ -18,9 +18,10 @@ extends =
   ../../component/socat/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../component/jq/buildout.cfg
+  ../../component/openssl/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../neoppod/software-common.cfg
+  ../../stack/caucase/buildout.cfg
 
 parts =
   slapos-cookbook
@@ -28,14 +29,43 @@ parts =
   instance.cfg
   template-mariadb.cfg
   template-mysqld-wrapper
+  caucase-eggs
   gowork
-  
+
+[mariadb]
+location = ${mariadb-10.4:location}
+
 [python]
 part = python3
 
-[gowork]
-# replication-manager does not build on golang 1.17
-golang = ${golang1.16:location}
+[template-mysqld-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
+mode = 644
+template =
+  inline:{% raw %}#!/bin/sh -e
+  basedir='${mariadb:location}'
+  datadir='{{datadir}}'
+  marker=$datadir/.slapos_initializing
+  system=$datadir/.system
+  [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
+    find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
+    touch "$marker"
+    "$basedir/scripts/mysql_install_db" \
+      --defaults-file='{{defaults_file}}' \
+      --skip-name-resolve \
+      --auth-root-authentication-method=normal \
+      --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
+      --datadir="$datadir"
+    rm "$marker"
+  }
+  {%- if environ is defined %}
+  {%-   for variable in environ.splitlines() %}
+  {{ variable }} \
+  {%-   endfor %}
+  {%- endif %}
+  exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
+  {% endraw %}
 
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
@@ -45,6 +75,7 @@ mode = 0644
 context =
     key bash_location bash:location
     key bin_directory buildout:bin-directory
+    key caucase_library caucase-jinja2-library:target
     key config_toml_in config-toml.in:target
     key config_cluster_toml_in config-cluster-toml.in:target
     key coreutils_location coreutils:location
@@ -70,6 +101,7 @@ context =
     key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
+    key openssl_location openssl:location
     key percona_toolkit_location percona-toolkit:location
     key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
     key repman_src_location git.signal18.io_signal18_repman:location
@@ -87,10 +119,12 @@ context =
     key template_init_root_sql mariadb-init-root-sql:target
     key template_init_root_wrapper init-root-wrapper-in:target
     key template_repman_cfg instance-repman.cfg:target
+    key template_publish_slave template-publish-slave-information:target
+    key template_slave_db_manage slave-db-manage.in:target
     key unixodbc_location unixodbc:location
     key sysbench_location sysbench:location
     key proxysql_location proxysql:location
-
+    raw caucase_bin_client ${buildout:bin-directory}/caucase
 
 [download-file]
 recipe  = slapos.recipe.build:download
@@ -126,6 +160,9 @@ link-binary =
 [template-mariadb-initial-setup]
 <= download-file
 
+[template-publish-slave-information]
+<= download-file
+
 [template-my-cnf]
 <= download-file
 
@@ -146,3 +183,6 @@ link-binary =
 
 [proxy-need-start-stop.sh.in]
 <= download-file
+
+[slave-db-manage.in]
+<= download-file

software/repman/software.cfg.json

@@ -10,6 +10,15 @@
       "request": "instance-repman-input-schema.json",
       "response": "instance-repman-output-schema.json",
       "index": 0
+    },
+    "default-slave": {
+      "title": "Replication Manager Database",
+      "description": "Replication Manager Database",
+      "software-type": "default",
+      "request": "instance-repman-slave-input-schema.json",
+      "response": "instance-repman-output-schema.json",
+      "shared": true,
+      "index": 1
     }
   }
 }

software/repman/templates/cluster-config.toml.in

@@ -12,6 +12,11 @@ monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{
 db-servers-connect-timeout = 1
 slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
 slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
+autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
+autoseed = {{ setbool(parameter_dict['autoseed']) }}
+{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
+autorejoin-logical-backup = true
+{% endif -%}
 
 proxysql = true
 proxysql-port = {{ parameter_dict['proxy-port'] }}
@@ -40,6 +45,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
 scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
+#scheduler-db-servers-receiver-use-ssl   ## set to true when ssl enabled on socat
 
 prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
 prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
@@ -49,6 +55,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
 prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
 prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
 
+replication-use-ssl = true
 test-inject-traffic = true
 # failover
 failover-mode = "{{ parameter_dict['failover-mode'] }}"

software/repman/templates/config.toml.in

-{% macro setbool(value) -%}
-{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
-{% endmacro -%}
 [Default]
 api-bind = "{{ parameter_dict['ipv4'] }}"
 http-bind-address = "{{ parameter_dict['ipv4'] }}"
@@ -11,17 +8,14 @@ monitoring-save-config = false
 api-https-bind = true
 api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
 include = "{{ parameter_dict['cluster-d'] }}"
-autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
-autoseed = {{ setbool(parameter_dict['autoseed']) }}
-{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
-autorejoin-logical-backup = true
-{% endif -%}
 db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 # Database list of hosts to ignore in election
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
-monitoring-wait-retry = 40
+monitoring-wait-retry = 1200
+monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
+monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
@@ -48,6 +42,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
 backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
 backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
 backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
 
 # Mail configuration
 # Alert email sender (default "mrm@localhost")

software/repman/templates/dbjobs.in

 #!/bin/bash
 
-USER={{ parameter_dict['db-user'] }}
+
+DBUSER={{ parameter_dict['db-user'] }}
 PASSWORD={{ parameter_dict['db-password'] }}
 ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
 SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
 BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
 DATADIR={{ parameter_dict['mysql-dir'] }}/
+MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
 {% if parameter_dict['use-ipv6'] == True -%}
 {% set listen = "TCP6-LISTEN" -%}
 {% else -%}
@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
 
 doneJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 pauseJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 partialRestore()
 {
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
  for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
 
 
   for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
     mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
     mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
     mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
     mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file=/etc/mysql/dbjob.cnf  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
  done
  for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
  done
- cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}   -e"flush privileges;start slave;"
+ cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
+ $MYSQL_CLIENT -e "flush privileges;start slave;"
 }
 
 kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
 for job in "${JOBS[@]}"
 do
 
- TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
+ TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
 
  ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
  ID=($(echo $TASK | awk -F@ '{ print $1 }'))
  #purge de past
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
+ $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
 
   if [ "$ADDRESS" == "" ]; then
     echo "No $job needed"
@@ -77,14 +79,14 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-        mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -e 'start slave;'
       ;;
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-       mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT  -e 'start slave;'
       ;;
       reseedmariabackup)
        rm -rf $BACKUPDIR
@@ -107,7 +109,7 @@ do
       ;;
       mariabackup)
        cd {{ parameter_dict['tmp-dir'] }}
-       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       error)
        cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
@@ -116,10 +118,10 @@ do
        cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       optimize)
-       mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       restart)
-        {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
+        {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
   esac
   doneJob

software/repman/templates/init_root_wrapper.in

@@ -2,7 +2,7 @@
 
 run_mysql () {
   {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
-  --protocol=socket -uroot -hlocalhost $@
+  --protocol=socket -hlocalhost "$@"
 }
 
 if [ ! -f "{{ init_password_done }}" ]; then
@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
   fi
   echo "Setting mariabdb root password...";
   run_mysql  < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
+  echo "remove unwanted users...";
+  run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
+  run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
   echo "done"
 fi
 
@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
   if [ $? -eq 0 ]; then
     touch {{ upgrade_done }};
   fi
+else
+  # only create databases now
+  run_mysql  < {{ init_database_sql }} || exit 1;
 fi

software/repman/templates/mariadb_init_root.sql.in

@@ -2,16 +2,18 @@
 --  or products like mysql-fabric won't work
 SET @@SESSION.SQL_LOG_BIN=0;
 
+# {{ parameter_dict["require-ssl"] }}
+{% set ssl = parameter_dict["require-ssl"] -%}
 CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
 CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
 
 CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
 
 DROP DATABASE IF EXISTS test ;

software/repman/templates/mariadb_initial_setup.sql.in

 SET @@SESSION.SQL_LOG_BIN=0;
 
 USE mysql;
-{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}
-SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
-{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
-CREATE DATABASE IF NOT EXISTS `{{ name }}`;
+CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
+
+{% macro database(name, user, password, ssl, charset="") -%}
+{% set charset_collate = "" -%}
+{% set charset_dict = {'utf8_general_ci': 'utf8',
+                       'utf8_unicode_ci': 'utf8',
+                       'utf8mb4_bin': 'utf8mb4',
+                       'utf8mb4_general_ci': 'utf8mb4',
+                       'utf8mb4_unicode_ci': 'utf8mb4',
+                       'latin1_general_ci': 'latin1',
+                       'latin1_bin': 'latin1',
+                       'ascii_general_ci': 'ascii',
+                       'ascii_bin': 'ascii'} -%}
+{% if charset %}
+{%   set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
+{%   if charset_dict.get(charset) -%}
+{%     set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
+{%   endif -%}
+{% endif -%}
+CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
 {% if user -%}
-GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
 {% endfor -%}

software/repman/templates/mysqld-need-start.sh.in

@@ -4,13 +4,11 @@ curl () {
   {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
 }
 
-# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
-
 # Checking if mariadb start is needed
-#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 
 if [ $CODE -eq 200 ]; then
+  sleep 5
   echo "$CODE: Updating mysql configuration..."
   # update mysql configuration
   {{ update_config }}
@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
   sleep 5
   # check again if the service is still up...
   {{ mariadb_controller }} status mariadb
+else
+  echo "CODE: $CODE. No need start!"
 fi

software/repman/templates/slave-db-manage.in

+#!/bin/bash
+
+set -e
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+get_token () {
+  curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
+}
+
+run_mysql () {
+  {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
+}
+
+TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \   {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }}  -r '.slaposDatadir')
+
+# Only write or delete on master database else, we break replication.
+if [ "$DATADIR" = "{{ partition_dir }}" ]; then
+
+  cat << EOF > {{ tmp_dir }}/.script.sql
+use repman_slave_definition;
+
+CREATE TABLE IF NOT EXISTS \`slave\` (
+  \`name\` varchar(80) NOT NULL,
+  \`state\` tinyint(1) DEFAULT NULL,
+  PRIMARY KEY (\`name\`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+UPDATE \`slave\` set \`state\`=false;
+{% for name in database_name.split(' ') -%}
+{%  if name -%}
+REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
+{%  endif -%}
+{% endfor -%}
+
+EOF
+
+  # Update requested slaves database
+  run_mysql < {{ tmp_dir }}/.script.sql
+  rm -f {{ tmp_dir }}/.script.sql
+
+  DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
+  RET=$?
+
+  if [ ! "$RET" = "0" ]; then
+    echo "Mysql command failed: $DBNAME"
+    exit $RET
+  fi
+  if [ -z "$DBNAME" ]; then
+    echo "No database for slave to remove.";
+  fi
+
+  for NAME in $DBNAME; do
+    if [ ! -z "$NAME" ]; then
+      echo "Deleting database $NAME..."
+      run_mysql -e "DROP DATABASE IF EXISTS $NAME";
+      run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
+      echo "Done."
+    fi
+  done
+fi

software/repman/test/test.py

@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
 
     token = resp.json()['token']
     headers = {"authorization": "Bearer " + token}
-    resp = requests.get(
-        urljoin(self.url, '/api/monitor'),
-        headers=headers,
-        verify=False,
-    )
-    self.assertEqual(resp.status_code, requests.codes.ok)
 
     for i in range(20):
       resp = requests.get(
@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
     self.assertTrue(cluster['isProvision'])
     self.assertTrue(cluster['isFailable'])
     self.assertFalse(cluster['isDown'])
+
+    resp = requests.get(
+        urljoin(self.url, '/api/clusters/cluster1/status'),
+        headers=headers,
+        verify=False,
+    )
+    self.assertEqual(resp.status_code, requests.codes.ok)
+    self.assertEqual(resp.json(), {"alive": "running"})

software/slapos-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 597c29546519aabe7259e416d0b92095
+md5sum = ba6c572678687081ce111d3805be3cee

software/slapos-testing/instance.cfg

@@ -35,10 +35,10 @@ repository = ${caucase-repository:location}
 [caucase-test-runner]
 recipe = slapos.recipe.template:jinja2
 template = inline:#!/bin/sh
-  export HOSTS="$(mktemp)"
-  trap 'rm "$HOSTS"' EXIT
-  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
-  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
+  export HOSTS_FILE="$(mktemp)"
+  trap 'rm "$HOSTS_FILE"' EXIT
+  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS_FILE"
+  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/libuserhosts.so:$LD_PRELOAD
   exec python -m unittest discover -v
 rendered = $${caucase:location}/host_setting.sh
 

software/theia/buildout.hash.cfg

@@ -47,7 +47,7 @@ md5sum = 1a668d6203d42b4d46d56e24c7606cb2
 
 [python-language-server-requirements.txt]
 _update_hash_filename_ = python-language-server-requirements.txt
-md5sum = 9f478fd1b03b7738f3de549cb899bf54
+md5sum = e16f1414f2657fa5ffc949839207f41b
 
 [slapos.css.in]
 _update_hash_filename_ = slapos.css.in

software/theia/python-language-server-requirements.txt

 appdirs==1.4.4
-astroid==2.5.2
-attrs==20.3.0
+astroid==2.6.2
+attrs==21.2.0
 black==20.8b1
-certifi==2020.12.5
+cachetools==4.2.4
+certifi==2021.5.30
 chardet==4.0.0
 click==7.1.2
 future==0.18.2
 idna==2.10
 importlib-metadata==3.10.0
-isort==5.8.0
+iniconfig==1.1.1
+isort==5.9.1
 jedi==0.18.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.6.0
 mccabe==0.6.1
-mypy==0.812
 mypy-extensions==0.4.3
+mypy==0.930
+packaging==21.0
 parso==0.8.2
 pathspec==0.8.1
 pluggy==0.13.1
-pydantic==1.8.1
+pydantic==1.8.2
 pydocstyle==6.0.0
 pyflags==0.1
 pyflakes==2.1.0
-pygls==0.10.2
-pylint==2.7.4
+pygls==0.11.1
+pylint==2.9.3
+pyparsing==2.4.7
 regex==2021.4.4
 requests==2.25.1
 rope==0.11.0
-six==1.12.0
+six==1.16.0
 snowballstemmer==1.2.1
-toml==0.10.2
+tomli==1.2.3
 typed-ast==1.4.1
-typeguard==2.12.0
-typing-extensions==3.7.4.3
-urllib3==1.26.4
-wrapt==1.11.2
-yapf==0.29.0
+typeguard==2.12.1
+typing-extensions==4.0.1
+urllib3==1.26.6
+wrapt==1.12.1
+yapf==0.31.0
+zc.buildout.languageserver==0.6.2
 zc.buildout==2.13.4
-zc.buildout.languageserver==0.5.0
-zipp==3.4.1
+zipp==3.4.1
\ No newline at end of file

stack/erp5/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = c82ea00c4514b72fb97a6fa7ac36ec52
+md5sum = f553aa7d6596dcf98e7e61bfb6bd81c7
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -46,7 +46,7 @@ md5sum = 1de449e8c0c4a85c5ce2b447785b7654
 
 [template-mariadb-initial-setup]
 filename = mariadb_initial_setup.sql.in
-md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
+md5sum = f928b9dc99f7f970caadfe7dd6f95d34
 
 [template-postfix]
 filename = instance-postfix.cfg.in
@@ -70,7 +70,7 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in
-md5sum = bbef65b4edeb342f08309604ca3717d5
+md5sum = 0444d646607f8643e987bb62d1cbbc1d
 
 [template-erp5]
 filename = instance-erp5.cfg.in
@@ -86,11 +86,11 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = f3121380ab4d31ba5f4984aec74d0a2f
+md5sum = 09895deebca3206425fada3caf1d92cf
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d6166515fda7b09df754672536b131be
+md5sum = cbcbe2daf3a51dfab50446a18c0be76e
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -189,7 +189,7 @@ cert = ${haproxy-conf-ssl:certificate}
 ca-cert = ${haproxy-conf-ssl:ca-cert}
 crl = ${haproxy-conf-ssl:crl}
 {% endif %}
-stats-socket = ${directory:run}/haproxy.sock
+stats-socket = ${directory:run}/ha.sock
 path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
 family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
 pidfile = ${directory:run}/haproxy.pid

stack/erp5/instance-mariadb.cfg.in

 {% set part_list = [] -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
-{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure'}]) -%}
+{% set database_list = slapparameter_dict.get('database-list', [{'name': 'erp5', 'user': 'user', 'password': 'insecure', 'with-process-privilege': True}]) -%}
 {% set test_database_list = [] %}
 {% for database_count in range(slapparameter_dict.get('test-database-amount', 1)) -%}
 {%   do test_database_list.append({'name': 'erp5_test_' ~ database_count, 'user': 'testuser_' ~ database_count, 'password': 'testpassword' ~ database_count}) -%}

stack/erp5/instance-zope.cfg.in

@@ -185,7 +185,7 @@ context = key host_dict hosts-parameter:host-dict
 recipe = slapos.recipe.template:jinja2
 mode = 755
 template = inline:#!{{ parameter_dict['dash'] }}/bin/dash
-  export HOSTS=${hosts:rendered}
+  export HOSTS_FILE=${hosts:rendered}
   export LD_PRELOAD={{ parameter_dict['userhosts'] }}:$LD_PRELOAD
   exec "$@"
 rendered = ${directory:bin}/userhosts

stack/erp5/instance.cfg.in

@@ -110,7 +110,7 @@ link-binary = {{ dumps(zope_link_binary) }}
 fonts = {{ dumps(zope_fonts) }}
 fontconfig-includes = {{ dumps(zope_fontconfig_includes) }}
 template-fonts-conf = {{ dumps(template_fonts_conf) }}
-userhosts = {{ userhosts_location }}/lib/userhosts.so
+userhosts = {{ userhosts_location }}/lib/libuserhosts.so
 site-zcml = {{ site_zcml }}
 extra-path-list = {{ dumps(extra_path_list) }}
 matplotlibrc = {{ matplotlibrc_location }}

stack/erp5/mariadb_initial_setup.sql.in

@@ -4,16 +4,19 @@ USE mysql;
 SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
 {% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
-#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
+{% macro database(name, user, password, with_process_privilege) -%}
 CREATE DATABASE IF NOT EXISTS `{{ name }}`;
 {% if user -%}
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%   if with_process_privilege %}
+GRANT PROCESS ON *.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT PROCESS ON *.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+{%-   endif %}
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('with-process-privilege')) }}
 {% endfor -%}

stack/slapos.cfg

@@ -132,7 +132,7 @@ eggs =
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos014
+zc.buildout = 2.7.1+slapos015
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 
@@ -167,7 +167,7 @@ importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
 itsdangerous = 0.24
 lock-file = 2.0
-lxml = 4.6.3
+lxml = 4.6.5
 meld3 = 1.0.2
 mock = 3.0.5
 more-itertools = 5.0.0
@@ -189,14 +189,14 @@ setproctitle = 1.1.10
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.4.3
 six = 1.12.0
-slapos.cookbook = 1.0.220
+slapos.cookbook = 1.0.226
 slapos.core = 1.7.2
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.22
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.52
-slapos.recipe.cmmi = 0.18
+slapos.recipe.build = 0.53
+slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 4.6
 slapos.toolbox = 0.126
 stevedore = 1.21.0:whl