Update Release Candidate

component/ZODB/test-zodb4-wc2.cfg

@@ -109,7 +109,6 @@ template = inline:
 
 
 [versions]
-mock = 3.0.5
 random2 = 1.0.1
 zope.testing = 4.7
 zope.testrunner = 5.2

component/cmake/buildout.cfg

@@ -14,8 +14,3 @@ md5sum = c0feb5855604f68b09bdb3acb623619e
 environment =
   CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
   CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
-
-[cmake-3.18]
-<= cmake
-url = https://cmake.org/files/v3.18/cmake-3.18.4.tar.gz
-md5sum = 0380beaee1c39a22455db02651abe7be

component/java-jdk/buildout.cfg

@@ -26,8 +26,7 @@ configure-command = :
 make-binary = :
 post-install =
   mv * %(location)s
-  for file in %(location)s/bin/* %(location)s/lib/*.so ; do
-    echo appending rpath to $file
-    ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s $file
-  done
+  cd %(location)s
+  set -x
+  ${patchelf:location}/bin/patchelf --set-rpath %(rpath)s bin/* lib/*.so
 rpath = ${zlib:location}/lib:${alsa:location}/lib:${freetype:location}/lib:${libpng:location}/lib:${libXrender:location}/lib:${libXtst:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXext:location}/lib:${libXdmcp:location}/lib:${libXi:location}/lib:${libxcb:location}/lib:@@LOCATION@@/lib:@@LOCATION@@/lib/server

component/libiconv/buildout.cfg

@@ -10,5 +10,6 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/libiconv.gets.patch#8a20d8afe0617fce56f77537d2b84621
 recipe = slapos.recipe.cmmi
+shared = true
 url = http://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.14.tar.gz
 md5sum = e34509b1623cec449dfeb73d7ce9c6c6

component/libmicrohttpd/buildout.cfg

@@ -10,6 +10,7 @@ parts =
 
 [libmicrohttpd]
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
 md5sum = dcd6045ecb4ea18c120afedccbd1da74
 configure-options =

component/m4/buildout.cfg

 [buildout]
 extends =
   ../xz-utils/buildout.cfg
-  ../patch/buildout.cfg
 parts =
   m4
 
 [m4]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.xz
-md5sum = 730bb15d96fffe47e148d1e09235af82
+url = http://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz
+md5sum = 0d90823e1426f1da2fd872df0311298d
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
-patch-binary = ${patch:location}/bin/patch
-patch-options = -p1
-patches =
-  https://sources.debian.org/data/main/m/m4/1.4.18-5/debian/patches/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac

component/ninja/buildout.cfg

 [buildout]
+extends =
+  ../defaults.cfg
 parts =
   ninja
 
@@ -8,8 +10,8 @@ shared = true
 version = 1.10.2
 url = https://github.com/ninja-build/ninja/archive/refs/tags/v${:version}.tar.gz
 md5sum = 639f75bc2e3b19ab893eaf2c810d4eb4
-configure-command = ./configure.py
-configure-options = --bootstrap
+configure-command = ${python:executable} configure.py
+configure-options = --bootstrap --verbose
 make-binary = true
 post-install =
   set %(location)s/bin

component/nxdtest/buildout.cfg

@@ -9,15 +9,11 @@ extends =
 # nxdtest is bin/ program to run nxdtest.
 # use ${nxdtest:exe} to run it.
 [nxdtest]
-recipe  = slapos.recipe.template:jinja2
+<= pyprog
 exe     = ${buildout:bin-directory}/nxdtest
-rendered= ${:exe}
-mode    = 755
-eggs    = ${.nxdtest.pyexe:eggs}
-template=
-  inline:
-  #!${.nxdtest.pyexe:exe}
-
+entry   = nxdtest:main
+eggs    = ${nxdtest-egg:egg}
+initialization =
   # $PATH for unshare and mount
   import os
   path = os.environ.get('PATH', '')
@@ -25,19 +21,6 @@ template=
     path = ':' + path
     os.environ['PATH'] = '${util-linux:location}/bin' + path
 
-  from nxdtest import main; main()
-
-
-# .nxdtest.pyexe is python interpreter used by nxdtest.
-# the interpreter is located at ${.nxdtest.pyexe:exe}.
-[.nxdtest.pyexe]
-<= python-interpreter
-eggs +=
-  ${pygolang:egg}
-  ${nxdtest-egg:egg}
-interpreter = ${:_buildout_section_name_}
-exe     = ${buildout:bin-directory}/${:interpreter}
-
 
 [nxdtest-egg]
 recipe  = zc.recipe.egg:develop
@@ -48,6 +31,6 @@ depends = ${python-prctl:egg}
 [nxdtest-repository]
 recipe  = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/nxdtest.git
-revision = 9f413221
+revision = 6f75fa90
 location = ${buildout:parts-directory}/nxdtest
 git-executable = ${git:location}/bin/git

component/proxysql/buildout.cfg

@@ -2,54 +2,49 @@
 extends =
   ../autoconf/buildout.cfg
   ../automake/buildout.cfg
-  ../libtool/buildout.cfg
-  ../cmake/buildout.cfg
-  ../openssl/buildout.cfg
-  ../patch/buildout.cfg
-  ../git/buildout.cfg
-  ../openssl/buildout.cfg
   ../bzip2/buildout.cfg
-  ../perl/buildout.cfg
-  ../gnutls/buildout.cfg
+  ../cmake/buildout.cfg
   ../curl/buildout.cfg
+  ../git/buildout.cfg
   ../gnutls/buildout.cfg
-  ../libzip/buildout.cfg
-  ../m4/buildout.cfg
-  ../pcre/buildout.cfg
   ../jemalloc/buildout.cfg
   ../libmicrohttpd/buildout.cfg
+  ../libtool/buildout.cfg
+  ../m4/buildout.cfg
+  ../openssl/buildout.cfg
+  ../patch/buildout.cfg
+  ../pcre/buildout.cfg
+  ../perl/buildout.cfg
+# https://github.com/libinjection/libinjection/issues/4
+  ../python-2.7/buildout.cfg
 
 parts =
   proxysql
 
 [proxysql]
 recipe = slapos.recipe.cmmi
-version = v2.0.12
+shared = true
+version = v2.3.2
 url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
-md5sum = 70ec17fe73703a25730fdd44b6bc3ef5
-prefix = @@LOCATION@@
-# Patch installation path for SlapOS
-pre-configure =
-  mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
-  sed -ri "
-    s#(\s)/usr/bin#\1${:prefix}/bin#g
-    s#(\s)/etc#\1${:prefix}/etc#g
-    s#(\s)/usr/lib#\1${:prefix}/lib#
-    s#(\s)/var/lib#\1${:prefix}/lib#g
-    s#(\s)useradd#\1echo useradd#g
-    s#(\s)systemctl#\1echo systemctl#g
-    s#(\s)chkconfig#\1echo chkconfig#g
-    s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
-
+md5sum = 969129ac43c9f64641509891a116e0e1
 configure-command = true
+make-options = GIT_VERSION=${:version}
+# The install target does not honor DESTDIR:
+# let's forget it and only install what we need.
+make-targets = src/proxysql
+post-install =
+  install -Dt %(location)s/bin %(make-targets)s
 environment =
-  GIT_VERSION=${:version}
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
-  PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake-3.18:location}/bin:%(PATH)s
+  PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s:${python2.7:location}/bin
   CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
   CFLAGS=-I${gnutls:location}/include
   LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
-  CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include
-  CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib
+  CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include:${zlib:location}/include
+  CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib:${zlib:location}/lib
   LIBTOOL=libtool
   ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
+patch-options = -p1
+patches =
+# PR #3402 (mariadb_client: backport patch to fix syntax error in cmake 3.20)
+  https://github.com/sysown/proxysql/commit/a3cfa56d257219f7610cd5711045bb5d84485a91.patch

component/pygolang/buildout.cfg

@@ -49,5 +49,64 @@ initialization =
 scripts = ${:interpreter}
 
 
+# pyprog provides macro recipe to build python programs.
+#
+# Contrary to zc.recipe.egg:scripts it generates scripts that are run with
+# sys.executable being correctly set. In particular it is valid to spawn
+# sys.executable from inside the program and assume that all specified eggs are
+# still accessible and can be imported.
+#
+# Usage example:
+#
+#   [myprog]
+#   <= pyprog
+#   exe = ${buildout:bin-directory}/myprog
+#   entry = my.py.mod:main
+#   eggs = ...
+[pyprog]
+recipe = slapos.recipe.build
+initialization =
+depends =
+_name = ${:_buildout_section_name_}
+init =
+  name   = options['_name']  # options['_buildout_section_name_'] does not work
+  exe    = options['exe']
+  entry  = options['entry']
+  eggs   = options['eggs']
+  pyinit = options['initialization']
+
+  options['depends'] += '$${.%s.pyprog:recipe}' % name
+
+  # mod:func -> 'from mod import func; sys.exit(func())'
+  mod, func = entry.split(':')
+  entry_run = 'from %s import %s; sys.exit(%s())' % (mod, func, func)
+
+  # indent pyinit with '  '
+  __pyinit = '\n'.join(['  '+_ for _ in pyinit.splitlines()])
+
+  self.buildout.parse("""
+  # .X.pyprog is python program to start and run entry
+  # it uses .X.pyexe as underlying python interpreter
+  [.%(name)s.pyprog]
+  recipe  = slapos.recipe.template:jinja2
+  exe     = %(exe)s
+  rendered= $${:exe}
+  mode    = 755
+  template=
+    inline:
+    #!$${.%(name)s.pyexe:exe}
+    import sys
+  %(__pyinit)s
+    %(entry_run)s
+
+  # .X.pyexe is python interpreter used by .X.pyprog
+  [.%(name)s.pyexe]
+  <= python-interpreter
+  eggs += %(eggs)s
+  interpreter = $${:_buildout_section_name_}
+  exe = $${buildout:bin-directory}/$${:interpreter}
+  """ % locals())
+
+
 [versions]
 pygolang = 0.0.9

component/pytest/buildout.cfg

@@ -16,3 +16,4 @@ eggs    = ${pytest:eggs}
 [versions]
 pytest  = 4.6.11:whl
 pytest-timeout = 1.4.2
+pytest-mock = 2.0.0:whl

component/replication-manager/buildout.cfg

@@ -28,5 +28,6 @@ command = set -e
 <= go-git-package
 go.importpath = github.com/signal18/replication-manager
 repository =  https://github.com/signal18/replication-manager
-branch = 2.1
-revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
+#branch = 2.1
+branch = develop
+revision = 838ffeb205ea4477f1c1bda607940fa07d8bcf51
\ No newline at end of file

component/slapos/obs.cfg

 [buildout]
 # You need to define rootdir and destdir on an upper level
+#
+# rootdir is where the parts will be installed by the debian package (e.g. /opt/slapos)
+# destdir is where we can put the files during compilation (e.g. /tmp/build-slapos)
+# buildout MUST BE launched in $destdir/$rootdir (this is enforced by an assert)
+#
+# in this file, we teach many packages to use prefix = $rootdir/parts/package and we
+# set DESTDIR = $destdir so in reality the files land in $destdir/$rootdir/parts/package
+# which is EXACTLY what cmmi recipe uses by default (buildout run dir/parts/package)
+# that's why everything is working
+#
+# please be careful when touching this file
+
 #rootdir = %TARGET_DIRECTORY%
-#destdir = %BUILD_ROOT_DIRECTORY% 
-#builddir = %BUILD_DIRECTORY%
+#destdir = %BUILD_ROOT_DIRECTORY%
+
 extends =
   buildout.cfg
 
@@ -13,34 +25,24 @@ extensions =
 extends-cache = extends-cache
 download-cache = download-cache
 
-# Uguu, upstream buildout.cfg must be patched as it works the other way
-# around from a packager point of view at least, thus at the end static
-# path, such as Python HOME directory, are wrong...
-#
-# Currently:
-# ./configure --prefix=BUILD_DIRECTORY && make install
-# Instead of:
-# ./configure --prefix=INSTALL_DIRECTORY && make install DESTDIR=BUILD_DIRECTORY
+[python]
+init +=
+  buildout = self.buildout['buildout']
+  assert buildout['directory'] == buildout['destdir'] + buildout['rootdir'], (
+    "Buildout MUST BE launched in destdir/rootdir (currently launched in %s but should be launched in %s)",
+	 buildout['directory'], buildout['destdir'] + buildout['rootdir'])
+
 [python2.7]
 configure-options +=
   --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
 environment +=
   DESTDIR=${buildout:destdir}
 
-[gettext]
-# Add gettext library path to RPATH as its binaries are used to build
-# glib for example
-environment =
-  PATH=${perl:location}/bin:${lunzip:location}/bin:%(PATH)s
-  CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include
-  LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${buildout:builddir}/parts/${:_buildout_section_name_}/lib
-
 [bison]
 configure-options +=
   --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
 make-options +=
   DESTDIR=${buildout:destdir}
-
 environment +=
   PERL5LIB=${perl:location}/lib/5.28.1/
 
@@ -62,7 +64,6 @@ environment +=
 
 
 [dbus]
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-options +=
   --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
 make-options +=
@@ -74,7 +75,6 @@ post-install =
   mkdir -p ${buildout:destdir}/parts/${:_buildout_section_name_}/var/run/dbus
 
 [dbus-glib]
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment +=
   LD_LIBRARY_PATH=${dbus:location}/lib
   DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
@@ -92,21 +92,13 @@ post-install =
   sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
 
 [openssl]
+shared = false
 prefix = ${buildout:rootdir}/parts/${:_buildout_section_name_}
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 make-options +=
   INSTALL_PREFIX=${buildout:destdir}
   DESTDIR=${buildout:destdir}
- 
-environment =
+environment +=
   PERL5LIB=${perl:location}/lib/5.28.1/
-  PERL=${perl:location}/bin/perl
-
-[bison-go]
-<= bison
-configure-options =
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-make-options =
 
 [gobject-introspection]
 pre-configure =
@@ -115,10 +107,8 @@ configure-options +=
   --enable-shared
 environment +=
   PERL5LIB=${perl:location}/lib/5.28.1/
-
-post-install = 
+post-install =
   sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
-  rm -rf ${bison-go:location}
 
 [pygobject3]
 pre-configure +=
@@ -126,18 +116,6 @@ pre-configure +=
 post-install =
   sed -i 's#!${python:location}/bin/python${python:version}#!/opt/slapos/parts/python${python:version}/bin/python${python:version}#' ${python:location}/bin/python${python:version}-config
 
-[ncurses]
-configure-options =
-  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-  --with-shared
-  --without-ada
-  --without-manpages
-  --without-tests
-  --without-normal
-  --without-debug
-  --without-gpm
-  --enable-rpath
-
 [flex]
 environment +=
  BISON_PKGDATADIR=${bison:location}/share/bison/
@@ -154,7 +132,6 @@ post-install =
   sed -i "s#'${buildout:destdir}/parts/site_perl#'/opt/slapos/parts/site_perl#" ${perl:location}/lib/5.28.1/*-linux-thread-multi/Config.pm
 
 [perl]
-location = ${buildout:destdir}/parts/perl
 configure-command =
   sh Configure -des \
     -Dprefix=${buildout:rootdir}/parts/${:_buildout_section_name_} \

component/zodbtools/test-common.cfg

@@ -42,5 +42,4 @@ template = inline:
 
 
 [versions]
-mock = 3.0.5
 freezegun = 1.0.0

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.220'
+version = '1.0.226'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/generic_mysql/mysql.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
@@ -16,35 +18,39 @@ def updateMysql(mysql_upgrade_binary, mysql_binary, mysql_script_file):
   while True:
     while True:
       mysql_upgrade = subprocess.Popen(mysql_upgrade_binary,
-        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        universal_newlines=True)
       result = mysql_upgrade.communicate()[0]
       if mysql_upgrade.returncode:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_binary, result)
+        print("Command %r failed with result:\n%s" % (mysql_upgrade_binary, result))
         break
-      print "MySQL database upgraded with result:\n%s" % result
+      print("MySQL database upgraded with result:\n%s" % result)
       mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
-          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+          universal_newlines=True)
       result = mysql.communicate(mysql_script)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
       # import timezone database
       mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                               universal_newlines=True)
       timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
       if mysql_tzinfo_to_sql.returncode != 0:
-        print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result))
         break
       mysql = subprocess.Popen(mysql_list + ('mysql',), stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                               universal_newlines=True)
       result = mysql.communicate(timezone_sql)[0]
       if mysql.returncode:
-        print 'Command %r failed with:\n%s' % (mysql_list, result)
+        print('Command %r failed with:\n%s' % (mysql_list, result))
         break
-      print 'SlapOS initialisation script succesfully applied on database.'
+      print('SlapOS initialisation script succesfully applied on database.')
       return
     sleep = min(sleep+1, 30)
-    print 'Sleeping for %ss and retrying' % sleep
+    print('Sleeping for %ss and retrying' % sleep)
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

software/erp5/instance-erp5-input-schema.json

@@ -217,7 +217,7 @@
             },
             "timerserver-interval": {
               "description": "Timerserver tick period, in seconds, or 0 to disable",
-              "default": 5,
+              "default": 1,
               "type": "number"
             },
             "private-dev-shm": {

software/erp5/software.cfg.json

@@ -5,6 +5,7 @@
   "software-type": {
     "default": {
       "title": "Default",
+      "software-type": "default",
       "description": "No automated database modification (ERP5Site is not automatically created).",
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",

software/erp5testnode/software.cfg.json

@@ -5,9 +5,11 @@
   "software-type": {
     "default": {
       "title": "Default",
+      "software-type": "default",
       "description": "Default setup for ERP5TestNode Instance.",
       "request": "instance-erp5testnode-input-schema.json",
-      "response": "instance-output-schema.json"
+      "response": "instance-output-schema.json",
+      "index": 0
     }
   }
 }

software/erp5testnode/testsuite/deploy-test/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [deploy-script-controller-script]
 filename = deploy-script-controller
-md5sum = 8288e59eb442c662544daffbf446a033
+md5sum = 8357771b70efd0740561b1cb46f6955e
 
 [template-deploy-test]
 filename = instance.cfg.in

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

@@ -83,8 +83,6 @@ function upload ()
   add_checks $LOG_FILE
   t=`date '+%Y%m%d%H%S'`
   mv $LOG_FILE ${LOG_FILE}.$t
-  # just to be sure flush all disk operations before uploading
-  flush
   curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
 }
 

software/neoppod/software-common.cfg

@@ -125,7 +125,6 @@ template =
 
 [versions]
 coverage = 4.5.1
-mock = 3.0.5
 ecdsa = 0.13
 mysqlclient = 1.3.12
 pycrypto = 2.6.1

software/repman/buildout.hash.cfg

@@ -14,19 +14,19 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 8a08be95a04f1a47098c4fdef80bdfed
+md5sum = af2fc4a7a0f782fed2cb1112ef3cb397
 
 [instance-repman.cfg]
 _update_hash_filename_ = instance-repman.cfg.jinja2.in
-md5sum = 697a1b546c883da45c14dbcd2d73b2b9
+md5sum = c6e41783b8e3e844b6319953a8d665ec
 
 [config-toml.in]
 _update_hash_filename_ = templates/config.toml.in
-md5sum = 2ee2c4bc5f985c11c2167b819d261256
+md5sum = dc9fdcf6c3c775da6b0b72f83c4a4105
 
 [config-cluster-toml.in]
 _update_hash_filename_ = templates/cluster-config.toml.in
-md5sum = c78c6be9537d3dffbb526b0e819bacb5
+md5sum = ff44b81dc1177e32019ddbf4f1c6b616
 
 [nginx.conf.in]
 _update_hash_filename_ = templates/nginx.conf.in
@@ -34,7 +34,7 @@ md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
 
 [template-mariadb.cfg]
 _update_hash_filename_ = instance-mariadb.cfg.jinja2.in
-md5sum = a5c204cac552754520aee0570d379723
+md5sum = 938f1e8087395757f5ae861d20500658
 
 [template-my-cnf]
 _update_hash_filename_ = templates/my.cnf.in
@@ -42,15 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
 
 [template-mariadb-initial-setup]
 _update_hash_filename_ = templates/mariadb_initial_setup.sql.in
-md5sum = df44c2d6fb1971df582345daedad280c
+md5sum = 47b76144e1b116580c8acf08274af976
+
+[template-publish-slave-information]
+_update_hash_filename_ = publish-database-slave-parameters.cfg.in
+md5sum = 9616477ff993c55c2b43bf8797db6994
 
 [mariadb-init-root-sql]
 _update_hash_filename_ = templates/mariadb_init_root.sql.in
-md5sum = d8a0a7d72b02f911dafda655941b805d
+md5sum = 6e5c02542568f250aacd84694249efc0
 
 [init-root-wrapper-in]
 _update_hash_filename_ = templates/init_root_wrapper.in
-md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
+md5sum = c203f40a58386310a433b58fd345a341
 
 [repman-manager-sh.in]
 _update_hash_filename_ = templates/repman-manager.sh.in
@@ -58,12 +62,16 @@ md5sum = 852dfab6d798aa1382eec4de2fd624f9
 
 [dbjobs-in]
 _update_hash_filename_ = templates/dbjobs.in
-md5sum = d2ebd2ec55bf8489789a52c808729925
+md5sum = d5c19d866c5a76d3bc2f434a91b1553a
 
 [mysqld-need-start.sh.in]
 _update_hash_filename_ = templates/mysqld-need-start.sh.in
-md5sum = e9bcee5dc1318fe3acda2663472214f5
+md5sum = 8f14054b3761b167cf8a86f4f084a574
 
 [proxy-need-start-stop.sh.in]
 _update_hash_filename_ = templates/proxy-need-start-stop.sh.in
 md5sum = 455aaf369bf5141758dc57f2c0e67b08
+
+[slave-db-manage.in]
+_update_hash_filename_ = templates/slave-db-manage.in
+md5sum = b45313ae5fb06972cc7fc945e34e434a

software/repman/instance-mariadb.cfg.jinja2.in

@@ -17,17 +17,11 @@
 
 [{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
--extends = publish-early
 database-host = {{ host }}:{{ port }}
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 partition-path = ${buildout:directory}
 receiver-port = ${dbjob-parameter:socat-port}
-
-[publish-early]
-recipe = slapos.cookbook:publish-early
--init =
-
-database-list = {{ dumps(database_list) }}
+csr-id = ${caucase-csr-id:csr-id}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -37,6 +31,22 @@ mode = 644
 < = jinja2-template-base
 mode = 755
 
+[jinja2-template-script-base]
+< = jinja2-template-executable
+context =
+  key tmp_dir            directory:tmp
+  key partition_dir      buildout:directory
+  raw username           {{ slapparameter_dict['repman-user'] }}
+  raw password           {{ slapparameter_dict['repman-passwd'] }}
+  raw cluster_name       {{ slapparameter_dict['cluster'] }}
+  raw secure_url         {{ slapparameter_dict['repman-secure-url'] }}
+  raw mysql_conf         ${directory:etc}/mysql/my.cnf
+  raw dash_bin           {{ dash }}
+  raw jq_bin             {{ jq_bin }}
+  raw curl_bin           {{ curl_bin }}
+  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  ${:extra-context}
+
 [simplefile]
 < = jinja2-template-base
 template = inline:{{ '{{ content }}' }}
@@ -53,10 +63,54 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+{% import "caucase" as caucase with context %}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-updater',
+     url=slapparameter_dict['caucase-url'],
+     data_dir='${mysql-directory:ssl}',
+     crt_path='${mysql-directory:ssl}/server-cert.pem',
+     ca_path='${mysql-directory:ssl}/ca-cert.pem',
+     crl_path='${mysql-directory:ssl}/crl.pem',
+     key_path='${mysql-directory:ssl}/server-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-updater') -%}
+{% do part_list.append('caucase-updater-promise') -%}
+
+[get-csr-id]
+recipe = plone.recipe.command
+output = ${directory:tmp}/csr_id
+command = 
+  if [ -s "${mysql-directory:ssl}/server-cert.pem" ]; then
+    RESULT="None";
+  else
+    if [ -f "${caucase-updater-csr:csr}" ]; then
+      RESULT=$({{ caucase_bin_client }} --ca-url {{ slapparameter_dict['caucase-url'] }} --send-csr ${caucase-updater-csr:csr} | cut -d ' ' -f1)
+      if [ ! $? -eq 0 ]; then
+        RESULT="None";
+      fi
+    fi
+  fi
+  cat <<EOF > ${:output}
+  [caucase]
+  csr-id = $(echo $RESULT)
+  EOF
+update-command = ${:command}
+
+[caucase-csr-id]
+recipe = slapos.cookbook:zero-knowledge.read
+file-path = ${get-csr-id:output}
+csr-id =
+
 [my-cnf-parameters]
 socket = ${directory:run}/mariadb.sock
 ip = {{ ip }}
-data-directory = ${directory:srv}/mariadb
+data-directory = ${mysql-directory:data}
 pid-file = ${directory:run}/mariadb.pid
 plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
 groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
@@ -66,9 +120,6 @@ innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0
 innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
 innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
 relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
-ssl-crt = ${directory:mariadb-ssl}/crt.pem
-ssl-key = ${directory:mariadb-ssl}/key.pem
-ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
 
 [my-cnf]
 < = jinja2-template-base
@@ -89,9 +140,11 @@ database-list = {{ dumps(database_list) }}
 mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
 root-user = repman
 heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
+require-ssl = {{ dumps(slapparameter_dict['require-ssl']) }}
 
 [init-script]
 < = jinja2-template-executable
+extensions = jinja2.ext.do
 # XXX: is there a better location ?
 rendered = ${directory:etc}/mariadb_initial_setup.sql
 template = {{ parameter_dict['template-mariadb-initial-setup'] }}
@@ -106,20 +159,26 @@ init-script = ${init-script:rendered}
 mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 
 [{{ section('update-mysql-script') }}]
-< = jinja2-template-executable
+< = jinja2-template-script-base
 rendered = ${directory:scripts}/mariadb_update
 init-password = ${directory:etc}/.init-passwd.done
 upgrade-done = ${directory:lib}/mariadb-update-done
-context =
+extra-context =
   key init_password_done :init-password
   key upgrade_done       :upgrade-done
   key init_root_sql      init-root-sql:rendered
   key mysql_update       update-mysql:output
-  raw mysql_conf         ${directory:etc}/mysql/my.cnf
-  raw dash_bin           {{ dash }}
-  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+  key init_database_sql  init-script:rendered
 template = {{ parameter_dict['template-init-root-wrapper'] }}
 
+[{{ section('mysql-slave-db-cleanup') }}]
+< = jinja2-template-script-base
+rendered = ${directory:scripts}/manage-slave-db
+db-name = {{ slapparameter_dict['database-name'] }}
+extra-context =
+  key database_name      :db-name
+template = {{ parameter_dict['template-manage-db'] }}
+
 [mysqld]
 < = jinja2-template-executable
 rendered = ${directory:bin}/mysqld
@@ -136,19 +195,19 @@ environ =
   {{ variable }}
   {%- endfor %}
 
-[ca-mysqld]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-key-file = ${my-cnf-parameters:ssl-key}
-cert-file = ${my-cnf-parameters:ssl-crt}
-executable = ${mysqld:rendered}
-wrapper = ${directory:controller}/mariadb
+[mysqld-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = ${mysqld:rendered}
+wrapper-path = ${directory:controller}/mariadb
+wait-for-files =
+  ${mysql-directory:mysql}/my.cnf
+  ${mysql-directory:ssl}/server-cert.pem
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
 {% do part_list.append("supervisord-mariadb-ctl") -%}
-{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
-  "stopwaitsecs": 86400, "environment": [],
+{% set maradb_program_dict = {"name": "mariadb", "command": "${mysqld-launcher:wrapper-path}",
+  "stopwaitsecs": 86400, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${directory:log}/mariadb_stdout.log",
   "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
 
@@ -178,13 +237,18 @@ target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
 
 [binary-wrap-base]
-recipe = slapos.cookbook:wrapper
 # Note: --defaults-file must be the first argument, otherwise wrapped binary
 # will reject it.
-command-line = 
-  "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
-  --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
-wrapper-path = ${directory:bin}/${:command}
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/${:command}
+mode = 755
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+    --protocol=socket \
+  ${:extra-args} "$@"
+wrapper-path = ${:rendered}
 extra-args =
 
 [binary-wrap-mysql]
@@ -208,6 +272,11 @@ command = mysqladmin
 <= binary-wrap-base
 command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
 command = mysql_tzinfo_to_sql
+template =
+  inline:#!/bin/sh
+  exec {{ parameter_dict['mariadb-location'] }}/bin/${:command} \
+  --defaults-file="${directory:etc}/mysql/my.cnf" \
+  --skip-write-binlog "$@"
 
 [binary-wrap-pt-digest]
 <= binary-wrap-base
@@ -226,26 +295,29 @@ plugin = ${:etc}/plugin
 srv = ${buildout:directory}/srv
 tmp = ${buildout:directory}/tmp
 backup = ${:srv}/backup
-mariadb-backup-full = ${:backup}/mariadb-full
-mariadb-backup-incremental = ${:backup}/mariadb-incremental
-mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 lib = ${:var}/lib
-mysql = ${:lib}/mysql
 log = ${:var}/log
 run = ${:var}/run
 config-tmp = ${:tmp}/config
-custom = ${directory:etc}/mysql/custom
+
+[mysql-directory]
+recipe = slapos.cookbook:mkdirectory
+mysql = ${directory:etc}/mysql
+ssl = ${:mysql}/ssl
+data = ${directory:lib}/mysql
+custom = ${:mysql}/custom.d
 
 [dbjob-parameter]
 bash-bin = {{ bash_bin }}
 db-user = ${init-script-parameters:root-user}
 db-password = ${init-script-parameters:password}
-mysql-dir = ${directory:mysql}
-dbjob-cnf = ${directory:etc}/mysql/my.cnf
-log-dir = ${directory:mysql}/.system/logs
+mysql-dir = ${mysql-directory:data}
+dbjob-cnf = ${mysql-directory:mysql}/dbjob.cnf
+log-dir = ${mysql-directory:data}/.system/logs
 tmp-dir = ${directory:tmp}
 mysqld-socket  = ${my-cnf-parameters:socket}
+mysql-client = ${binary-wrap-mysql:wrapper-path}
 socat-port = {{ int(port) + 9 }}
 restart-script = ${mysqld-restart-script:rendered}
 socat-location = {{ parameter_dict['socat-location'] }}
@@ -309,11 +381,11 @@ template = inline:#!{{ bash_bin }}
   cd ${directory:config-tmp} &&
   {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
   tar -xzf config.tar.gz
-  cp -r data/.system ${directory:mysql}
-  rm -rf ${directory:etc}/mysql
-  cp -r etc/mysql ${directory:etc}
-  ln -sf ${directory:mysql}/.system ${directory:var}/system
-  ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
+  rm -r etc/mysql/ssl  # we don't need generated ssl files
+  cp -r data/.system ${mysql-directory:data}
+  cp -r etc/mysql/* ${mysql-directory:mysql}
+  ln -sf ${mysql-directory:data}/.system ${directory:var}/system
+  ln -sf ${my-cnf:rendered} ${mysql-directory:custom}/01_mariadb.cnf
 
 [{{ section('install-mysql-config') }}]
 recipe = plone.recipe.command

software/repman/instance-repman-input-schema.json

@@ -113,7 +113,8 @@
         "mroonga",
         "utctime",
         "readcommitted",
-        "nohandshake"
+        "nohandshake",
+        "ssl"
       ]
     },
     "http-session-lifetime": {
@@ -140,6 +141,12 @@
       "type": "boolean",
       "default": true
     },
+    "require-ssl": {
+      "title": "Enable REQUIRE SSL for repman database users",
+      "description": "Enable secure connection between repman and mariadb databases. This option is an initial value, used only when creating database users.",
+      "type": "boolean",
+      "default": false
+    },
     "repman-cluster-dict": {
       "title": "Replication Manager clusters definition",
       "description": "Replication Manager clusters definition",
@@ -197,7 +204,7 @@
             },
             "proxy-tags": {
               "title": "Proxy tag list",
-              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\"]",
+              "description": "playbook configuration tags. Default: [\"pkg\", \"masterslave\", \"linux\", \"noreadwritesplit\", \"ssl\"]",
               "type": "array",
               "items": {
                 "type": "string"
@@ -206,7 +213,8 @@
                 "pkg",
                 "masterslave",
                 "linux",
-                "noreadwritesplit"
+                "noreadwritesplit",
+                "ssl"
               ]
             },
             "logical-backup-cron": {

software/repman/instance-repman-slave-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "properties": {
+    "db_user": {
+      "description": "Database User, default is 'user'. This parameter is set only if database is not created yet.",
+      "title": "Database User",
+      "type": "string",
+      "default": "user"
+    },
+    "db_password": {
+      "description": "Database password. If no password set, a password will be generated. This parameter is set only if database is not created yet.",
+      "title": "Initial database password",
+      "type": "string",
+      "default": ""
+    },
+    "db_charset": {
+      "description": "Database Character set information. This parameter is set only if database is not created yet.",
+      "title": "Database Character set information",
+      "type": "string",
+      "enum": [
+        "latin1",
+        "latin2",
+        "latin5",
+        "latin7",
+        "ascii",
+        "latin5",
+        "utf8",
+        "utf8mb4",
+        "utf16",
+        "utf16le",
+        "utf32",
+        "binary",
+        "utf8_general_ci",
+        "utf8_unicode_ci",
+        "utf8_unicode_ci",
+        "utf8mb4_bin",
+        "utf8mb4_general_ci",
+        "utf8mb4_unicode_ci",
+        "latin1_general_ci",
+        "latin1_bin",
+        "ascii_general_ci",
+        "ascii_bin"
+      ],
+      "default": ""
+    }
+  },
+  "title": "Input Parameters",
+  "type": "object"
+}

software/repman/instance-repman.cfg.jinja2.in

@@ -8,8 +8,23 @@
 {% set ip = (ipv6_set | list)[0] -%}
 {% set ipv4 = (ipv4_set | list)[0] -%}
 {% set cluster_list = [] -%}
-{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow", "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci", "mroonga", "utctime", "readcommitted", "nohandshake"] -%}
+{% set tag_list = ["gtidstrict", "bind", "pkg", "innodb", "noquerycache", "slow",
+  "pfs", "linux", "readonly", "diskmonitor", "sqlerror", "compressbinlog", "bm4ci",
+  "mroonga", "utctime", "readcommitted", "nohandshake", "ssl"] -%}
 {% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
+{% set database_slave_list = [] -%}
+{% set db_name_list = [] -%}
+{% set count = namespace(value=2) %}
+{% set caucase_bind = '[' ~ ip ~ ']:8890' -%}
+{% set caucase_url = 'http://' ~ caucase_bind -%}
+{% set csrid_list = [] -%}
+
+{% macro password(name) -%}
+[{{ name }}-password]
+recipe = slapos.cookbook:generate.password
+bytes = 12
+
+{% endmacro -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -25,11 +40,25 @@ log = ${:var}/log
 data = ${:var}/lib
 nginx-prefix = ${:var}/nginx
 tmp = ${:home}/tmp
+backup-caucased = ${:srv}/backup-caucased
+caucased = ${:srv}/caucased
+ssl = ${:etc}/ssl
+
+[proxysql-directory]
+recipe = slapos.cookbook:mkdirectory
+config = ${directory:etc}/proxysql
+ssl = ${:config}/ssl
 
 {% import "supervisord_lib" as supervisord_lib with context %}
 {% set proxysql_controller = "proxysql-ctl" -%}
 {{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+
+[proxysql-controller-update]
+recipe = slapos.cookbook:wrapper
+command-line = ${proxysql-ctl-bin:wrapper-path} update
+wrapper-path = ${directory:scripts}/proxysql-ctl-update
 {% do part_list.append("supervisord-proxysql-ctl") -%}
+{% do part_list.append("proxysql-controller-update") -%}
 
 [request-common]
 recipe = slapos.cookbook:request.serialised
@@ -39,6 +68,7 @@ key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
+config-caucase-url = {{ caucase_url }}
 
 [download-proxy-config]
 recipe = slapos.recipe.template:jinja2
@@ -48,16 +78,35 @@ template = inline:#!{{ bash_bin }}
   PORT=$3
   CONFIG=$4
   if [ -z "$CONFIG" ]; then
-    CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
+    CONFIG="${proxysql-directory:config}/proxysql.cnf"
   fi
   mkdir -p ${repman:config-tmp}/proxies
   cd ${repman:config-tmp}/proxies
   {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
   tar -xzf proxies-$NAME.tar.gz
-  cp conf/proxysql.cnf $CONFIG
+  cp etc/proxysql/proxysql.cnf $CONFIG
 rendered = ${directory:bin}/update-proxysql-config
 mode = 755
 
+{% for instance_dict in slave_instance_list -%}
+{% set slave_dict = {
+  'name': 'db_%s' % instance_dict['slave_reference'].replace('-', '_').lower(),
+  'user': instance_dict.get('db_user', 'user'),
+  'password': instance_dict.get('db_password', '${' ~ instance_dict['slave_reference'] ~ '-password:passwd}'),
+  'slave_reference': instance_dict['slave_reference'],
+  'charset': instance_dict.get('db_charset', ''),
+  'require_ssl': True
+  } -%}
+{% do database_slave_list.append(slave_dict) -%}
+{% do db_name_list.append(slave_dict['name']) -%}
+{{ password(instance_dict['slave_reference']) }}
+{% endfor %}
+[database-slave-information]
+{% for slave_dict in database_slave_list -%}
+{{ slave_dict['name'] }} = !py!{{ slave_dict }}
+{% endfor %}
+{% set db_list = db_name_list | join(' ') -%}
+
 {% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
 {% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
   "database-amount": 2, "proxysql-user": "external",
@@ -67,22 +116,23 @@ mode = 755
 {% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
 {% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
 
-{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
 # Request mariadb instances
 {% set db_amount = parameter_dict.get('database-amount', 2) -%}
 {% if db_amount < 2 -%}
 {% set db_amount = 2 -%}
 {% endif -%}
 
+{% set count.value = count.value + db_amount %}
 {% for i in range(0, db_amount) -%}
 {% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
 {% set section = 'request-mariadb-' ~ i -%}
-{% set dbname = 'Mariadb-' ~ i -%}
+{% set dbname = 'Mariadb-' ~ i %}
+
 
 [{{ section }}]
 <= request-common
 software-type = mariadb
-name = {{ dbname }}
+name = {{ dbname ~ '-' ~ name}}
 sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
 {% for key, value in mariadb_dict.items() -%}
 config-{{ key }} = {{ dumps(value) }}
@@ -91,23 +141,27 @@ config-monitor-passwd = ${publish-early:monitor-password}
 config-root-password = ${publish-early:db-root-password}
 config-repman-user = ${repman-parameter:username}
 config-heartbeat-user = ${repman-parameter:heartbeat-user}
-#config-repman-passwd =  ${repman-parameter:password}
+config-repman-passwd =  ${repman-parameter:password}
 config-repman-url = ${nginx-parameter:backend-url}
 config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
 config-cluster = {{ name }}
 config-name = {{ dbname }}
-config-database-list = {{ dumps(database_list) }}
-return = 
+config-database-list = !py!{{ database_slave_list }}
+config-database-name = {{ dumps(db_list) }}
+config-require-ssl = {{ dumps(slapparameter_dict.get('require-ssl', False)) }}
+return =
   database-host
   receiver-port
   monitor-base-url
   partition-path
+  csr-id
 
 {% do part_list.append(section) -%}
 {% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
 {% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
 {% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
 {% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+{% do csrid_list.append('${' ~ section ~ ':connection-csr-id}') -%}
 
 {% endfor -%}
 
@@ -129,6 +183,7 @@ maximum = 7132
 {% for key, value in parameter_dict.items() -%}
 {{ key }} = {{ value }}
 {% endfor -%}
+name = {{ name }}
 proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
 proxy-port = {{ '${' ~ name ~ '-port:port}' }}
 proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
@@ -151,7 +206,7 @@ password = ${repman-parameter:password}
 proxysql-partition = ${buildout:directory}
 receiver-port-list = {{ receiver_port_list | join(',') }}
 enabled-tags = {{ slapparameter_dict.get("tag-list", tag_list) | join(',') }}
-proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit"]) | join(',') }}
+proxy-tags = {{ parameter_dict.get("proxy-tags", ["pkg", "masterslave", "linux", "noreadwritesplit", "ssl"]) | join(',') }}
 logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
 physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
 proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
@@ -191,7 +246,7 @@ context =
 recipe = plone.recipe.command
 # if Repman is not started, cannot download config from server
 stop-on-error = false
-config = ${repman:proxies}/proxysql-{{ name }}.cnf
+config = ${proxysql-directory:config}/proxysql.cnf
 data = ${repman:proxy-data}/{{ name }}
 command =
   mkdir -p ${:data} &&
@@ -209,6 +264,7 @@ command-line =
 wrapper-path = ${directory:controller}/proxysql-{{ name }}
 wait-for-files =
   ${repman:bootstrap}/{{ name }}_bootstrapped
+  ${config-proxysql-{{ name }}:config}
 depends =
   {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
   {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
@@ -237,7 +293,7 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 
 {% set service_name = "proxysql-" ~ name -%}
 {% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
-  "stopwaitsecs": 60, "environment": [],
+  "stopwaitsecs": 60, "startretries": 10, "autorestart": True, "environment": [],
   "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
   "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
 
@@ -247,22 +303,126 @@ config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
 {% do part_list.append('config-' ~ name) -%}
 {% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
 
-{% set publish_database_list = [] -%}
-{% set publish_database_v6_list = [] -%}
-{% for database in database_list -%}
-{%   if database.get('user') -%}
-{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%     do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%   else -%}
-{%     do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%     do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
-{%   endif -%}
-{% endfor -%}
 
-{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
-{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
+[{{ name}}-publish-slave-information]
+recipe = slapos.cookbook:switch-softwaretype
+default = {{ 'dynamic-' ~name ~ '-publish-slave-information:rendered' }}
+RootSoftwareInstance = ${:default}
+
+[dynamic-{{ name}}-publish-slave-information]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/database-publish-slave-information.cfg
+extensions = jinja2.ext.do
+template = {{ template_publish_slave_information }}
+context =
+    import json_module          json
+    key proxy_port              {{ name }}-cluster-parameter:proxy-port
+    raw eggs_directory          {{ eggs_directory }}
+    raw develop_eggs_directory  {{ develop_eggs_directory }}
+    raw ipv6                    {{ ip }}
+    raw ipv4                    {{ ipv4 }}
+    section slave_dict          database-slave-information
+
+{% do part_list.append(name ~ '-publish-slave-information') -%}
 {% endfor -%}
 
+# deploy caucase
+{% import "caucase" as caucase with context %}
+{{ caucase.caucased(
+    prefix='caucased',
+    buildout_bin_directory=buildout_bin_directory,
+    caucased_path='${directory:service}/caucased',
+    backup_dir='${directory:backup-caucased}',
+    data_dir='${directory:caucased}',
+    netloc=caucase_bind,
+    tmp='${directory:tmp}',
+    service_auto_approve_count=count.value,
+    user_auto_approve_count=1,
+    key_len=2048,
+)}}
+{% do part_list.append('caucased') -%}
+{% do part_list.append('caucased-promise') -%}
+{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
+{{ caucase.updater(
+     prefix='caucase-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-updater',
+     url=caucase_url,
+     data_dir='${directory:srv}/caucase-updater',
+     crt_path='${directory:ssl}/repman-cert.crt',
+     ca_path='${directory:srv}/caucase-updater/ca.crt',
+     crl_path='${directory:srv}/caucase-updater/crl.pem',
+     key_path='${directory:ssl}/repman-cert.key',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-updater') -%}
+{% do part_list.append('caucase-updater-promise') -%}
+
+#caucase user certificate
+{{ caucase.updater(
+     prefix='caucase-user-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-user-updater',
+     url=caucase_url,
+     data_dir='${directory:srv}/caucase-user-updater',
+     crt_path='${directory:ssl}/caucase.user.crt',
+     ca_path='${directory:srv}/caucase-user-updater/ca.crt',
+     crl_path='${directory:srv}/caucase-user-updater/crl.pem',
+     key_path='${directory:ssl}/caucase.user.key',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+     mode='user',
+)}}
+{% do part_list.append('caucase-user-updater') -%}
+{% do part_list.append('caucase-user-updater-promise') -%}
+
+#caucase proxysql certificate
+{{ caucase.updater(
+     prefix='caucase-proxysql-updater',
+     buildout_bin_directory=buildout_bin_directory,
+     updater_path='${directory:services}/caucase-proxysql-updater',
+     url=caucase_url,
+     data_dir='${proxysql-directory:ssl}',
+     crt_path='${proxysql-directory:ssl}/client-cert.pem',
+     ca_path='${proxysql-directory:ssl}/ca-cert.pem',
+     crl_path='${proxysql-directory:ssl}/crl.pem',
+     key_path='${proxysql-directory:ssl}/client-key.pem',
+     on_renew=None,
+     max_sleep=None,
+     template_csr_pem=None,
+     openssl=openssl_bin,
+)}}
+{% do part_list.append('caucase-proxysql-updater') -%}
+{% do part_list.append('caucase-proxysql-updater-promise') -%}
+
+[caucase-sign-csr]
+recipe = slapos.recipe.template:jinja2
+mode = 755
+rendered = ${directory:bin}/caucase-sign
+template =
+  inline:#!/bin/sh
+  cp ${directory:ssl}/caucase.user.key ${directory:ssl}/caucase-full.key
+  cat ${directory:ssl}/caucase.user.crt >> ${directory:ssl}/caucase-full.key
+  for csr_id in {{ csrid_list | join(' ') }}; do
+    if [ "$csr_id" = "None" ] || [ -z "$csr_id"]; then
+      continue
+    fi
+    {{ buildout_bin_directory }}/caucase --ca-url {{ caucase_url }} --user-key ${directory:ssl}/caucase-full.key --sign-csr $csr_id
+  done
+
+[cron-caucase-sign-csr]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = caucase-sign-csr
+frequency = * * * * *
+command = ${caucase-sign-csr:rendered}
+{% do part_list.append('cron-caucase-sign-csr') -%}
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = ${slap-connection:computer-id}
@@ -286,7 +446,6 @@ etc      = ${directory:etc}/repman
 data-dir = ${directory:var}/lib
 root-dir = ${directory:srv}/repman
 clusters = ${:etc}/cluster.d
-proxies  = ${:etc}/proxy
 proxy-data = ${:data-dir}/proxy
 config-tmp = ${directory:tmp}/config
 bootstrap = ${:etc}/bootstrap
@@ -296,8 +455,9 @@ proxies-log = ${directory:log}/proxy
 ipv6 = ${instance-parameter:ipv6-random}
 port = ${instance-parameter:nginx-port}
 ssl-port = ${instance-parameter:nginx-ssl-port}
-ssl-certificate = ${ca-nginx:cert-file}
-ssl-key = ${ca-nginx:key-file}
+ssl-certificate = ${directory:ssl}/repman-cert.crt
+ca-certificate = ${directory:srv}/caucase-updater/ca.crt
+ssl-key = ${directory:ssl}/repman-cert.key
 pid-file = ${directory:run}/nginx.pid
 access-log = ${directory:log}/nginx_access.log
 error-log = ${directory:log}/nginx_error.log
@@ -345,6 +505,8 @@ mail-to = {{ slapparameter_dict.get("mail-to", "") }}
 
 http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
 http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
+ssl-cert = ${directory:ssl}/repman-cert.crt
+ssl-key = ${directory:ssl}/repman-cert.key
 
 [repman-config-folder]
 recipe = plone.recipe.command
@@ -392,6 +554,9 @@ depends =
   ${replication-manager-reload:recipe}
   ${repman-listen-promise:recipe}
   ${repman-listen-ssl-promise:recipe}
+wait-for-files =
+  ${directory:ssl}/repman-cert.crt
+  ${directory:ssl}/repman-cert.key
 
 [repman-config.toml]
 recipe = slapos.recipe.template:jinja2
@@ -428,11 +593,12 @@ command-line =
   {{ nginx_bin }}
   -p ${directory:nginx-prefix}
   -c ${nginx-conf:rendered}
-wrapper-path = ${directory:bin}/nginx-start
+wrapper-path = ${directory:services}/nginx
 wait-for-files =
-  ${ca-directory:certs}/nginx.key
-  ${ca-directory:certs}/nginx.crt
-  ${nginx-graceful-wrapper:rendered}
+  ${directory:ssl}/repman-cert.crt
+  ${directory:srv}/caucase-updater/ca.crt
+  ${directory:ssl}/repman-cert.key
+
 
 [nginx-graceful-wrapper]
 recipe = slapos.recipe.template:jinja2
@@ -442,20 +608,6 @@ rendered = ${directory:scripts}/nginx-graceful
 context =
 mode = 755
 
-[ca-nginx]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-cert-file = ${ca-directory:certs}/nginx.crt
-key-file = ${ca-directory:certs}/nginx.key
-executable = ${nginx-launcher:wrapper-path}
-wrapper = ${directory:bin}/ca-nginx
-
-[ca-nginx-service]
-recipe = slapos.cookbook:wrapper
-command-line = ${ca-nginx:wrapper}
-wrapper-path = ${directory:services}/nginx
-hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-
 [logrotate-entry-nginx]
 <= logrotate-entry-base
 name = nginx
@@ -468,6 +620,7 @@ recipe = slapos.cookbook:publish-early
   monitor-password monitor-htpasswd:passwd
   db-root-password gen-root-password:passwd
   repman-password repman-password:passwd
+database-list = {{ db_list }}
 
 [publish-connection-parameter]
 <= monitor-publish
@@ -476,6 +629,7 @@ recipe = slapos.cookbook:publish
 backend-url = ${nginx-parameter:backend-ssl-url}
 url = ${repman-frontend:connection-secure_access}
 username = ${repman-parameter:username}
+database-list = {{ db_list }}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
@@ -545,6 +699,15 @@ command = ${template-proxysql-need-stop-start:rendered}
 # Deploy replication-manager instance
 #
 #############################
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
 [buildout]
 extends =
   {{ template_monitor }}
@@ -552,7 +715,7 @@ parts =
   replication-manager
   monitor-base
   logrotate-entry-nginx
-  ca-nginx-service
+  nginx-launcher
   publish-connection-parameter
   repman-frontend-promise
   repman-backend-promise

software/repman/instance.cfg.in

@@ -28,8 +28,10 @@ mode    = 0644
 extensions = jinja2.ext.do
 rendered= ${buildout:directory}/${:_buildout_section_name_}
 supervisord-lib = {{ supervisord_lib }}
+causace-lib = {{ caucase_library }}
 import-list =
   file supervisord_lib :supervisord-lib
+  file caucase         :causace-lib
 context =
     key slapparameter_dict      slap-configuration:configuration
     key computer_id             slap-configuration:computer
@@ -48,6 +50,8 @@ context =
     raw bash_bin                {{ bash_location }}/bin/bash
     raw jq_bin                  {{ jq_location }}/bin/jq
     raw curl_bin                {{ curl_location }}/bin/curl
+    raw openssl_bin             {{ openssl_location }}/bin/openssl
+    raw caucase_bin_client      {{ caucase_bin_client }}
 
     ${:extra-context}
 extra-context =
@@ -56,6 +60,7 @@ extra-context =
 <= jinja2-template-base
 template= {{ template_repman_cfg }}
 extra-context =
+    key slave_instance_list         slap-configuration:slave-instance-list
     raw gowork_bin                  {{ gowork_bin }}
     raw haproxy_location            {{ haproxy_location }}
     raw nginx_bin                   {{ nginx_location }}/sbin/nginx
@@ -71,6 +76,7 @@ extra-context =
     raw proxysql_location          {{ proxysql_location }}
     raw template_repman_manager_sh {{ template_repman_manager_sh }}
     raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
+    raw template_publish_slave_information {{ template_publish_slave }}
 
 [template-mariadb-parameters]
 bash = {{ bash_location }}
@@ -83,6 +89,7 @@ template-mariadb-init-root = {{ template_init_root_sql }}
 template-init-root-wrapper = {{ template_init_root_wrapper }}
 template-mysqld-wrapper = {{ template_mysqld_wrapper }}
 template-mysqld-need-start = {{ mysqld_start_template }}
+template-manage-db = {{ template_slave_db_manage }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
 bin-directory = {{ bin_directory }}

software/repman/publish-database-slave-parameters.cfg.in

+{% set part_list = [] -%}
+
+[slap_connection]
+# Kept for backward compatibility
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
+
+{% for _, database in slave_dict.items() -%}
+{% set slave_reference = database['slave_reference'] -%}
+{% do part_list.append('publish-slave-' ~ slave_reference) -%}
+
+[publish-slave-{{ slave_reference }}]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_reference }}
+{%   if database.get('user') -%}
+url_v4 = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":" ~ proxy_port ~ '/' ~ database['name'] }}
+url = {{ "mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ipv6 ~ "]:" ~ proxy_port ~ '/' ~ database['name'] }}
+{%   else -%}
+url = {{ "mysql://" ~ ipv4 ~ ":" ~ proxy_port ~ database['name'] }}
+url_v6 = {{ "mysql://[" ~ ip ~ "]:" ~ proxy_port ~ database['name'] }}
+{%   endif -%}
+{% endfor %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+parts =
+{% for part in part_list -%}
+{{ '  %s' % part }}
+{% endfor -%}

software/repman/software.cfg

@@ -18,9 +18,10 @@ extends =
   ../../component/socat/buildout.cfg
   ../../component/rsync/buildout.cfg
   ../../component/jq/buildout.cfg
+  ../../component/openssl/buildout.cfg
   ../../stack/supervisord/buildout.cfg
   ../../stack/monitor/buildout.cfg
-  ../neoppod/software-common.cfg
+  ../../stack/caucase/buildout.cfg
 
 parts =
   slapos-cookbook
@@ -28,14 +29,43 @@ parts =
   instance.cfg
   template-mariadb.cfg
   template-mysqld-wrapper
+  caucase-eggs
   gowork
-  
+
+[mariadb]
+location = ${mariadb-10.4:location}
+
 [python]
 part = python3
 
-[gowork]
-# replication-manager does not build on golang 1.17
-golang = ${golang1.16:location}
+[template-mysqld-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/mysqld.in
+mode = 644
+template =
+  inline:{% raw %}#!/bin/sh -e
+  basedir='${mariadb:location}'
+  datadir='{{datadir}}'
+  marker=$datadir/.slapos_initializing
+  system=$datadir/.system
+  [ -d "$datadir/mysql" ] && [ ! -f "$marker" ] || {
+    find "$datadir/" -mindepth 1 ! \( -path $marker -o -path $system -o -path "*$system/*" \) -delete || mkdir "$datadir"
+    touch "$marker"
+    "$basedir/scripts/mysql_install_db" \
+      --defaults-file='{{defaults_file}}' \
+      --skip-name-resolve \
+      --auth-root-authentication-method=normal \
+      --basedir="$basedir" --plugin_dir="$basedir/lib/plugin" \
+      --datadir="$datadir"
+    rm "$marker"
+  }
+  {%- if environ is defined %}
+  {%-   for variable in environ.splitlines() %}
+  {{ variable }} \
+  {%-   endfor %}
+  {%- endif %}
+  exec "$basedir/bin/mysqld" --defaults-file='{{defaults_file}}' "$@"
+  {% endraw %}
 
 [instance.cfg]
 recipe = slapos.recipe.template:jinja2
@@ -45,6 +75,7 @@ mode = 0644
 context =
     key bash_location bash:location
     key bin_directory buildout:bin-directory
+    key caucase_library caucase-jinja2-library:target
     key config_toml_in config-toml.in:target
     key config_cluster_toml_in config-cluster-toml.in:target
     key coreutils_location coreutils:location
@@ -70,6 +101,7 @@ context =
     key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
     key nginx_conf_in nginx.conf.in:target
     key nginx_location nginx:location
+    key openssl_location openssl:location
     key percona_toolkit_location percona-toolkit:location
     key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
     key repman_src_location git.signal18.io_signal18_repman:location
@@ -87,10 +119,12 @@ context =
     key template_init_root_sql mariadb-init-root-sql:target
     key template_init_root_wrapper init-root-wrapper-in:target
     key template_repman_cfg instance-repman.cfg:target
+    key template_publish_slave template-publish-slave-information:target
+    key template_slave_db_manage slave-db-manage.in:target
     key unixodbc_location unixodbc:location
     key sysbench_location sysbench:location
     key proxysql_location proxysql:location
-
+    raw caucase_bin_client ${buildout:bin-directory}/caucase
 
 [download-file]
 recipe  = slapos.recipe.build:download
@@ -126,6 +160,9 @@ link-binary =
 [template-mariadb-initial-setup]
 <= download-file
 
+[template-publish-slave-information]
+<= download-file
+
 [template-my-cnf]
 <= download-file
 
@@ -146,3 +183,6 @@ link-binary =
 
 [proxy-need-start-stop.sh.in]
 <= download-file
+
+[slave-db-manage.in]
+<= download-file

software/repman/software.cfg.json

@@ -10,6 +10,15 @@
       "request": "instance-repman-input-schema.json",
       "response": "instance-repman-output-schema.json",
       "index": 0
+    },
+    "default-slave": {
+      "title": "Replication Manager Database",
+      "description": "Replication Manager Database",
+      "software-type": "default",
+      "request": "instance-repman-slave-input-schema.json",
+      "response": "instance-repman-output-schema.json",
+      "shared": true,
+      "index": 1
     }
   }
 }

software/repman/templates/cluster-config.toml.in

@@ -40,6 +40,7 @@ scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backu
 backup-physical-type = "mariabackup"
 backup-logical-type = "mysqldump"
 scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
+#scheduler-db-servers-receiver-use-ssl   ## set to true when ssl enabled on socat
 
 prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
 prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
@@ -49,6 +50,7 @@ prov-db-memory = {{ parameter_dict['db-memory'] }}
 prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
 prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
 
+replication-use-ssl = true
 test-inject-traffic = true
 # failover
 failover-mode = "{{ parameter_dict['failover-mode'] }}"

software/repman/templates/config.toml.in

@@ -21,7 +21,9 @@ db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
 #db-servers-ignored-hosts =
 # Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
 monitoring-address = "{{ parameter_dict['ipv4'] }}"
-monitoring-wait-retry = 40
+monitoring-wait-retry = 1200
+monitoring-ssl-cert = "{{ parameter_dict['ssl-cert'] }}"
+monitoring-ssl-key = "{{ parameter_dict['ssl-key'] }}"
 
 #haproxy = true
 #haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
@@ -48,6 +50,7 @@ backup-restic-password = "{{ parameter_dict['password'] }}"
 backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
 backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
 backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+backup-mysqldump-options = "--hex-blob --single-transaction --verbose --all-databases --add-drop-database"
 
 # Mail configuration
 # Alert email sender (default "mrm@localhost")

software/repman/templates/dbjobs.in

 #!/bin/bash
 
-USER={{ parameter_dict['db-user'] }}
+
+DBUSER={{ parameter_dict['db-user'] }}
 PASSWORD={{ parameter_dict['db-password'] }}
 ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
 SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
 BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
 DATADIR={{ parameter_dict['mysql-dir'] }}/
+MYSQL_CLIENT={{ parameter_dict['mysql-client'] }}
 {% if parameter_dict['use-ipv6'] == True -%}
 {% set listen = "TCP6-LISTEN" -%}
 {% else -%}
@@ -16,45 +18,45 @@ JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "
 
 doneJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 pauseJob()
 {
-  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+  $MYSQL_CLIENT  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
 }
 
 partialRestore()
 {
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
  for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
+ $MYSQL_CLIENT  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
 
 
   for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
     mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
     mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
     mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
     mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file=/etc/mysql/dbjob.cnf  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
   for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
   done
  done
  for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
     mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
-    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
+    $MYSQL_CLIENT  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
  done
- cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}   -e"flush privileges;start slave;"
+ cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER
+ $MYSQL_CLIENT -e "flush privileges;start slave;"
 }
 
 kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
@@ -62,12 +64,12 @@ kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
 for job in "${JOBS[@]}"
 do
 
- TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
+ TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER -N))
 
  ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
  ID=($(echo $TASK | awk -F@ '{ print $1 }'))
  #purge de past
- mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
+ $MYSQL_CLIENT -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
 
   if [ "$ADDRESS" == "" ]; then
     echo "No $job needed"
@@ -77,14 +79,14 @@ do
       reseedmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-        mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER --init-command="reset master;set sql_log_bin=0" &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -e 'start slave;'
       ;;
       flashbackmysqldump)
        echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
        pauseJob
-       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
-       mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$DBUSER &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT  -e 'start slave;'
       ;;
       reseedmariabackup)
        rm -rf $BACKUPDIR
@@ -107,7 +109,7 @@ do
       ;;
       mariabackup)
        cd {{ parameter_dict['tmp-dir'] }}
-       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=root --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       error)
        cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
@@ -116,10 +118,10 @@ do
        cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       optimize)
-       mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
+       $MYSQL_CLIENT -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
       restart)
-        {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
+        {{ parameter_dict['restart-script'] }} &>{{ parameter_dict['log-dir'] }}/dbjob.out
       ;;
   esac
   doneJob

software/repman/templates/init_root_wrapper.in

@@ -2,7 +2,7 @@
 
 run_mysql () {
   {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
-  --protocol=socket -uroot -hlocalhost $@
+  --protocol=socket -hlocalhost "$@"
 }
 
 if [ ! -f "{{ init_password_done }}" ]; then
@@ -19,6 +19,9 @@ if [ ! -f "{{ init_password_done }}" ]; then
   fi
   echo "Setting mariabdb root password...";
   run_mysql  < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
+  echo "remove unwanted users...";
+  run_mysql -e "DROP USER IF EXISTS 'root'@'$HOSTNAME';"
+  run_mysql -e "DROP USER IF EXISTS ''@'$HOSTNAME';"
   echo "done"
 fi
 
@@ -29,4 +32,7 @@ if [ ! -f "{{ upgrade_done }}" ]; then
   if [ $? -eq 0 ]; then
     touch {{ upgrade_done }};
   fi
+else
+  # only create databases now
+  run_mysql  < {{ init_database_sql }} || exit 1;
 fi

software/repman/templates/mariadb_init_root.sql.in

@@ -2,16 +2,18 @@
 --  or products like mysql-fabric won't work
 SET @@SESSION.SQL_LOG_BIN=0;
 
+# {{ parameter_dict["require-ssl"] }}
+{% set ssl = parameter_dict["require-ssl"] -%}
 CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
 CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
 
 CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
-CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' {% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
 
 DROP DATABASE IF EXISTS test ;

software/repman/templates/mariadb_initial_setup.sql.in

 SET @@SESSION.SQL_LOG_BIN=0;
 
 USE mysql;
-{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
-{% if mroonga %}
-SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
-{% endif %}
 DROP FUNCTION IF EXISTS sphinx_snippets;
 #CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
 
-{% macro database(name, user, password) -%}
-CREATE DATABASE IF NOT EXISTS `{{ name }}`;
+CREATE DATABASE IF NOT EXISTS `repman_slave_definition`;
+
+{% macro database(name, user, password, ssl, charset="") -%}
+{% set charset_collate = "" -%}
+{% set charset_dict = {'utf8_general_ci': 'utf8',
+                       'utf8_unicode_ci': 'utf8',
+                       'utf8mb4_bin': 'utf8mb4',
+                       'utf8mb4_general_ci': 'utf8mb4',
+                       'utf8mb4_unicode_ci': 'utf8mb4',
+                       'latin1_general_ci': 'latin1',
+                       'latin1_bin': 'latin1',
+                       'ascii_general_ci': 'ascii',
+                       'ascii_bin': 'ascii'} -%}
+{% if charset %}
+{%   set charset_collate = "DEFAULT CHARACTER SET %s" % charset -%}
+{%   if charset_dict.get(charset) -%}
+{%     set charset_collate = "DEFAULT CHARACTER SET %s COLLATE %s" % (charset_dict[charset], charset) -%}
+{%   endif -%}
+{% endif -%}
+CREATE DATABASE IF NOT EXISTS `{{ name }}` {{ charset_collate }};
 {% if user -%}
-GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}'{% if ssl %} REQUIRE SSL{% endif %};
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
 GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
 {%- endif %}
 {% endmacro -%}
 
 {% for entry in parameter_dict['database-list'] -%}
-{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{{   database(entry['name'], entry.get('user'), entry.get('password'), entry.get('require_ssl', True) ~ '', entry.get('charset')) }}
 {% endfor -%}

software/repman/templates/mysqld-need-start.sh.in

@@ -4,13 +4,11 @@ curl () {
   {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
 }
 
-# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
-
 # Checking if mariadb start is needed
-#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
 
 if [ $CODE -eq 200 ]; then
+  sleep 5
   echo "$CODE: Updating mysql configuration..."
   # update mysql configuration
   {{ update_config }}
@@ -21,4 +19,6 @@ if [ $CODE -eq 200 ]; then
   sleep 5
   # check again if the service is still up...
   {{ mariadb_controller }} status mariadb
+else
+  echo "CODE: $CODE. No need start!"
 fi

software/repman/templates/slave-db-manage.in

+#!/bin/bash
+
+set -e
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+get_token () {
+  curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
+}
+
+run_mysql () {
+  {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" --protocol=socket "$@"
+}
+
+TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+DATADIR=$(curl -H "Authorization: Bearer ${TOKEN}" \   {{ secure_url }}/api/clusters/{{ cluster_name }}/topology/master | {{ jq_bin }}  -r '.slaposDatadir')
+
+# Only write or delete on master database else, we break replication.
+if [ "$DATADIR" = "{{ partition_dir }}" ]; then
+
+  cat << EOF > {{ tmp_dir }}/.script.sql
+use repman_slave_definition;
+
+CREATE TABLE IF NOT EXISTS \`slave\` (
+  \`name\` varchar(80) NOT NULL,
+  \`state\` tinyint(1) DEFAULT NULL,
+  PRIMARY KEY (\`name\`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+UPDATE \`slave\` set \`state\`=false;
+{% for name in database_name.split(' ') -%}
+{%  if name -%}
+REPLACE INTO \`slave\` VALUES ('{{ name }}', true);
+{%  endif -%}
+{% endfor -%}
+
+EOF
+
+  # Update requested slaves database
+  run_mysql < {{ tmp_dir }}/.script.sql
+  rm -f {{ tmp_dir }}/.script.sql
+
+  DBNAME=$(run_mysql --skip-column-names -Be "SELECT name FROM repman_slave_definition.slave WHERE state=false");
+  RET=$?
+
+  if [ ! "$RET" = "0" ]; then
+    echo "Mysql command failed: $DBNAME"
+    exit $RET
+  fi
+  if [ -z "$DBNAME" ]; then
+    echo "No database for slave to remove.";
+  fi
+
+  for NAME in $DBNAME; do
+    if [ ! -z "$NAME" ]; then
+      echo "Deleting database $NAME..."
+      run_mysql -e "DROP DATABASE IF EXISTS $NAME";
+      run_mysql -e "DELETE FROM repman_slave_definition.slave WHERE name='$NAME'";
+      echo "Done."
+    fi
+  done
+fi

software/repman/test/test.py

@@ -62,12 +62,6 @@ class TestRepman(SlapOSInstanceTestCase):
 
     token = resp.json()['token']
     headers = {"authorization": "Bearer " + token}
-    resp = requests.get(
-        urljoin(self.url, '/api/monitor'),
-        headers=headers,
-        verify=False,
-    )
-    self.assertEqual(resp.status_code, requests.codes.ok)
 
     for i in range(20):
       resp = requests.get(
@@ -87,3 +81,11 @@ class TestRepman(SlapOSInstanceTestCase):
     self.assertTrue(cluster['isProvision'])
     self.assertTrue(cluster['isFailable'])
     self.assertFalse(cluster['isDown'])
+
+    resp = requests.get(
+        urljoin(self.url, '/api/clusters/cluster1/status'),
+        headers=headers,
+        verify=False,
+    )
+    self.assertEqual(resp.status_code, requests.codes.ok)
+    self.assertEqual(resp.json(), {"alive": "running"})

software/theia/test/project_tests.py

@@ -60,7 +60,7 @@ def setUpModule():
 
 class ERP5Mixin(object):
   _test_software_url = erp5_software_release_url
-  _connexion_parameters_regex = re.compile(r"{\s*'_'\s*:\s*'(.*)'\s*}")
+  _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
 
   def _getERP5ConnexionParameters(self, software_type='export'):
     slapos = self._getSlapos(software_type)
@@ -69,7 +69,7 @@ class ERP5Mixin(object):
       stderr=subprocess.STDOUT,
     )
     print(out)
-    return json.loads(self._connexion_parameters_regex.search(out).group(1))
+    return json.loads(self._connexion_parameters_regex.search(out).group(0).replace("'", '"'))
 
   def _getERP5Url(self, connexion_parameters, path=''):
     return urljoin(connexion_parameters['family-default-v6'], path)

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = bbef65b4edeb342f08309604ca3717d5
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = c10634353841bb09a847168b4add8d2f
+md5sum = cb41e15a1585973896a7645dc2ae3a5c
 
 [template-zeo]
 filename = instance-zeo.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -297,7 +297,7 @@ config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict['instance-count']) }}
 config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
 config-thread-amount = {{ dumps(zope_parameter_dict['thread-amount']) }}
-config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
+config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 1)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}

stack/slapos.cfg

@@ -169,6 +169,7 @@ itsdangerous = 0.24
 lock-file = 2.0
 lxml = 4.6.3
 meld3 = 1.0.2
+mock = 3.0.5
 more-itertools = 5.0.0
 netaddr = 0.7.19
 pathlib2 = 2.3.5