Update Release Candidate

component/fontconfig/buildout.cfg

@@ -18,12 +18,12 @@ shared = true
 url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
 md5sum = 733f5e2371ca77b69707bd7b30cc2163
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
-# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
 configure-options =
   --disable-static
   --disable-docs
   --enable-libxml2
   --with-default-fonts=${fonts:location}
+  --with-add-fonts=no
 environment =
   PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}

component/fonts/buildout.cfg

@@ -9,6 +9,7 @@ parts =
   ipa-fonts
   ocrb-fonts
   android-fonts
+  dejavu-fonts
 
 [fonts]
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -50,6 +51,13 @@ md5sum = 2d41d5342eb5f61591ddeec5b80da74d
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
 
+# The DejaVu fonts are a font family based upon Bitstream Vera v1.10. Its purpose is to
+# provide a wider range of characters while maintaining the original look-and-feel
+[dejavu-fonts]
+<= fonts-base
+url = https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-fonts-ttf-2.37.tar.bz2
+md5sum = d0efec10b9f110a32e9b8f796e21782c
+
 # Microsoft's TrueType core fonts
 # non-free so not enabled by default
 [msttcore-fonts]

component/graphviz/buildout.cfg

@@ -9,12 +9,13 @@ extends =
   ../gtk-2/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
+  ../libexpat/buildout.cfg
 
 [graphviz]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz
-md5sum = 5b6a829b2ac94efcd5fa3c223ed6d3ae
+url = https://ftp.osuosl.org/pub/blfs/conglomeration/graphviz/graphviz-2.40.1.tar.gz
+md5sum = 4ea6fd64603536406166600bcc296fc8
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}
 configure-options =
   --with-included-ltdl
@@ -22,6 +23,7 @@ configure-options =
   --with-zlibdir=${zlib:location}/lib
   --with-freetype2
   --with-fontconfig
+  --with-expat
   --disable-swig
   --disable-sharp
   --disable-go
@@ -37,7 +39,6 @@ configure-options =
   --disable-ruby
   --disable-tcl
   --without-x
-  --without-expat
   --without-devil
   --without-webp
   --without-poppler
@@ -59,5 +60,5 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  CPPFLAGS=-I${zlib:location}/include -I${libexpat:location}/include
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libexpat:location}/lib -Wl,-rpath=${libexpat:location}/lib

component/msgpack-numpy/buildout.cfg

+[buildout]
+parts =
+  msgpack-numpy
+
+[msgpack-numpy]
+recipe = zc.recipe.egg:custom
+egg = msgpack-numpy

component/msgpack-python/buildout.cfg

@@ -4,4 +4,4 @@ parts =
 
 [msgpack-python]
 recipe = zc.recipe.egg:custom
-egg = msgpack-python
+egg = msgpack

component/openssl/buildout.cfg

@@ -43,7 +43,7 @@ make-options =
 make-targets =
   install_sw && x=${:location}/etc/ssl/certs && rm -f $x/* &&
   for i in ${ca-certificates:location}/certs/*/*.crt; do
-    ln -sv $i $x/`${:location}/bin/openssl x509 -hash -noout -in $i`.0
+    ln -sfv $i $x/`${:location}/bin/openssl x509 -hash -noout -in $i`.0
   ; done
 environment =
   PERL=${perl:location}/bin/perl

component/perl/buildout.cfg

@@ -54,6 +54,9 @@ install-inc =
 #  - extra arguments passed to perl Makefile.PL
 extra-configure-args =
 
+# - extra environ varialbles for configure and make
+extra-env =
+
 # Outputs:
 #  - site_perl, to use in inc in dependent packages
 site_perl=${:location}/lib/site_perl/${perl:version}:${:location}:${:inc}
@@ -64,12 +67,14 @@ perl-PATH = ${:location}/bin/
 #  dependencies
 perl-bin = ${:perl-PATH}/perl
 
+perl-PERL5LIB = 
 
 # Implementation
 recipe = slapos.recipe.cmmi
 url = https://www.cpan.org/modules/by-module/${:module}-${:version}.tar.gz
 configure-command =
-  PERL5LIB="${:inc}:${:install-inc}" \
+  ${:extra-env}
+  PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" \
   ${perl:location}/bin/perl \
   Makefile.PL \
   PREFIX=${:location} \
@@ -80,7 +85,7 @@ make-options =
   INSTALLSITESCRIPT=${:location}/perl-bin/
 
 make-binary=
-  PERL5LIB="${:inc}:${:install-inc}" make
+  ${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
 
 # this post-make-hook is same for all users of the macro.
 post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook

component/python-cryptography/buildout.cfg

@@ -12,6 +12,10 @@ recipe = zc.recipe.egg:custom
 egg = cryptography
 environment = python-cryptography-env
 setup-eggs = ${python-cffi:egg}
+library-dirs =
+  ${openssl:location}/lib/
+rpath =
+  ${openssl:location}/lib/
 
 [python-cryptography-env]
 PATH = ${pkgconfig:location}/bin:%(PATH)s

component/slapos/build.sh

@@ -29,7 +29,7 @@ unset CONFIG_SITE
 # Bootstrap SlapOS, using forked version of buildout.
 #
 wget https://bootstrap.pypa.io/bootstrap-buildout.py
-python -S bootstrap-buildout.py --buildout-version 2.5.2+slapos009 \
+python -S bootstrap-buildout.py --buildout-version 2.5.2+slapos013 \
   -f http://www.nexedi.org/static/packages/source/slapos.buildout/
 
 #

component/slapos/obs.cfg

+[buildout]
+# You need to define rootdir and destdir on an upper level
+#rootdir = %TARGET_DIRECTORY%
+#destdir = %BUILD_ROOT_DIRECTORY% 
+#builddir = %BUILD_DIRECTORY%
+extends =
+  buildout.cfg
+
+# Don't load extensions
+extensions =
+  slapos.rebootstrap
+
+extends-cache = extends-cache
+download-cache = download-cache
+
+# Uguu, upstream buildout.cfg must be patched as it works the other way
+# around from a packager point of view at least, thus at the end static
+# path, such as Python HOME directory, are wrong...
+#
+# Currently:
+# ./configure --prefix=BUILD_DIRECTORY && make install
+# Instead of:
+# ./configure --prefix=INSTALL_DIRECTORY && make install DESTDIR=BUILD_DIRECTORY
+[python2.7]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+environment +=
+  DESTDIR=${buildout:destdir}
+
+[gettext]
+# Add gettext library path to RPATH as its binaries are used to build
+# glib for example
+environment =
+  PATH=${perl:location}/bin:${lunzip:location}/bin:%(PATH)s
+  CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include
+  LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${buildout:builddir}/parts/${:_buildout_section_name_}/lib
+
+[bison]
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+make-options +=
+  DESTDIR=${buildout:destdir}
+
+environment +=
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+[intltool]
+environment +=
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+[autoconf]
+environment +=
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+[automake]
+environment +=
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+[firewalld]
+environment +=
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+
+[dbus]
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+configure-options +=
+  --prefix=${buildout:rootdir}/parts/${:_buildout_section_name_}
+make-options +=
+  DESTDIR=${buildout:destdir}
+environment +=
+  LDFLAGS=-L${libexpat:location}/lib -L${buildout:parts-directory}/${:_buildout_section_name_}/lib -Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib
+  LD_LIBRARY_PATH=${buildout:parts-directory}/${:_buildout_section_name_}/lib
+post-install =
+  mkdir -p ${buildout:destdir}/parts/${:_buildout_section_name_}/var/run/dbus
+
+[dbus-glib]
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+environment +=
+  CPPFLAGS=-I${libexpat:location}/include -I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
+  LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
+  LD_LIBRARY_PATH=${dbus:location}/lib
+  DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
+
+
+[dbus-python]
+pre-configure =
+  sed -i 's#/opt/slapos/parts/dbus/lib/libdbus-1.la#${dbus:location}/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
+environment +=
+  LD_LIBRARY_PATH=${dbus:location}/lib
+  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${dbus:location}/lib -Wl,-rpath=${dbus:location}/lib
+post-install =
+  sed -i 's#${dbus:location}/lib/libdbus-1.la#/opt/slapos/parts/dbus/lib/libdbus-1.la#' ${dbus-glib:location}/lib/libdbus-glib-1.la
+
+[openssl]
+prefix = ${buildout:rootdir}/parts/${:_buildout_section_name_}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+make-options +=
+  INSTALL_PREFIX=${buildout:destdir}
+ 
+environment =
+  PERL5LIB=${perl:location}/lib/5.26.1/
+  PERL=${perl:location}/bin/perl
+
+[bison-go]
+<= bison
+configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+make-options =
+
+[gobject-introspection]
+pre-configure =
+  ln -s ${python2.7:location}/bin/python2.7 ${python2.7:location}/bin/python2.
+  sed -i 's#!/opt/slapos/parts/python2.7/bin/python2.7#!${python2.7:location}/bin/python2.7#' ${python2.7:location}/bin/python-config
+  libtoolize -c -f
+  aclocal -I${pkgconfig:location}/share/aclocal -I${gettext:location}/share/aclocal -I${libtool:location}/share/aclocal -I${glib:location}/share/aclocal
+  ./autogen.sh
+configure-options +=
+  --enable-shared
+environment +=
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${flex:location}/bin:${bison-go:location}/bin:%(PATH)s
+  GIR_DIR=${buildout:parts-directory}/${:_buildout_section_name_}/share/gir-1.0
+  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${python2.7:location}/include/python2.7
+  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi -L${python2.7:location}/lib
+  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal:${glib:location}/share/aclocal:${intltool:location}/share/aclocal
+  M4=${m4:location}/bin/m4
+  PERL5LIB=${perl:location}/lib/5.26.1/
+
+post-install = 
+  sed -i 's#!${python2.7:location}/bin/python2.7#!/opt/slapos/parts/python2.7/bin/python2.7#' ${python2.7:location}/bin/python-config
+  rm -rf ${bison-go:location}
+
+[pygobject3]
+pre-configure +=
+  sed -i 's#!/opt/slapos/parts/python2.7/bin/python2.7#!${python2.7:location}/bin/python2.7#' ${python2.7:location}/bin/python-config
+environment +=
+  CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include -I${gettext:location}/include -I${libffi:location}/include -I${python2.7:location}/include/python2.7
+  LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${python2.7:location}/lib
+post-install =
+  sed -i 's#!${python2.7:location}/bin/python2.7#!/opt/slapos/parts/python2.7/bin/python2.7#' ${python2.7:location}/bin/python-config
+
+[ncurses]
+configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-shared
+  --without-ada
+  --without-manpages
+  --without-tests
+  --without-normal
+  --without-debug
+  --without-gpm
+  --enable-rpath
+
+[flex]
+environment +=
+ BISON_PKGDATADIR=${bison:location}/share/bison/
+
+
+[perl-CPAN-package]
+perl-PERL5LIB=${perl:location}/lib/5.26.1/
+
+pre-configure =
+  sed -i "s#'/opt/slapos/parts/perl#'${perl:location}#" ${perl:location}/lib/5.26.1/*-linux-thread-multi/Config.pm
+  sed -i "s#'/opt/slapos/parts/site_perl#'${buildout:destdir}/parts/site_perl#" ${perl:location}/lib/5.26.1/*-linux-thread-multi/Config.pm
+post-install =
+  sed -i "s#'${perl:location}#'/opt/slapos/parts/perl#" ${perl:location}/lib/5.26.1/*-linux-thread-multi/Config.pm
+  sed -i "s#'${buildout:destdir}/parts/site_perl#'/opt/slapos/parts/site_perl#" ${perl:location}/lib/5.26.1/*-linux-thread-multi/Config.pm
+
+[perl]
+location = ${buildout:destdir}/parts/perl
+configure-command =
+  sh Configure -des \
+    -Dprefix=${buildout:rootdir}/parts/${:_buildout_section_name_} \
+    -Dsiteprefix=${buildout:rootdir}/parts/site_${:_buildout_section_name_} \
+    -Dcflags=-I${gdbm:location}/include \
+    -Dldflags="-L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib" \
+    -Ui_db \
+    -Dnoextensions=ODBM_File \
+    -Dusethreads
+environment +=
+  DESTDIR=${buildout:destdir}
+
+[versions]
+# More recently version of cliff uses pbr which will break package build on OBS.
+cliff = 1.4.5

component/tomcat/buildout.cfg

@@ -23,6 +23,13 @@ strip-top-level-dir = true
 url = http://www-us.apache.org/dist/tomcat/tomcat-7/v7.0.91/bin/apache-tomcat-7.0.91.tar.gz
 md5sum = 8bfbb358b51f90374067879f8db1e91c
 
+[tomcat9]
+recipe = hexagonit.recipe.download
+ignore-existing = true
+strip-top-level-dir = true
+url = https://www-eu.apache.org/dist/tomcat/tomcat-9/v9.0.12/bin/apache-tomcat-9.0.12.tar.gz
+md5sum = 7283da4a3a6e939adcd8f919be4ba41a
+
 [tomcat7-output]
 # Shared binary location to ease migration
 recipe = plone.recipe.command

slapos/test/metaschema.json

-{
-  "id": "http://json-schema.org/draft-04/schema#",
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "description": "Core schema meta-schema",
-  "definitions": {
-    "schemaArray": {
-      "type": "array",
-      "minItems": 1,
-      "items": {
-        "$ref": "#"
-      }
-    },
-    "positiveInteger": {
-      "type": "integer",
-      "minimum": 0
-    },
-    "positiveIntegerDefault0": {
-      "allOf": [
-        {
-          "$ref": "#/definitions/positiveInteger"
-        },
-        {
-          "default": 0
-        }
-      ]
-    },
-    "simpleTypes": {
-      "enum": [
-        "array",
-        "boolean",
-        "integer",
-        "null",
-        "number",
-        "object",
-        "string"
-      ]
-    },
-    "stringArray": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      },
-      "minItems": 1,
-      "uniqueItems": true
-    }
-  },
-  "type": "object",
-  "properties": {
-    "id": {
-      "type": "string",
-      "format": "uri"
-    },
-    "$schema": {
-      "type": "string",
-      "format": "uri"
-    },
-    "title": {
-      "type": "string"
-    },
-    "description": {
-      "type": "string"
-    },
-    "default": {},
-    "multipleOf": {
-      "type": "number",
-      "minimum": 0,
-      "exclusiveMinimum": true
-    },
-    "maximum": {
-      "type": "number"
-    },
-    "exclusiveMaximum": {
-      "type": "boolean",
-      "default": false
-    },
-    "minimum": {
-      "type": "number"
-    },
-    "exclusiveMinimum": {
-      "type": "boolean",
-      "default": false
-    },
-    "maxLength": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minLength": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "pattern": {
-      "type": "string",
-      "format": "regex"
-    },
-    "additionalItems": {
-      "anyOf": [
-        {
-          "type": "boolean"
-        },
-        {
-          "$ref": "#"
-        }
-      ],
-      "default": {}
-    },
-    "items": {
-      "anyOf": [
-        {
-          "$ref": "#"
-        },
-        {
-          "$ref": "#/definitions/schemaArray"
-        }
-      ],
-      "default": {}
-    },
-    "maxItems": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minItems": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "uniqueItems": {
-      "type": "boolean",
-      "default": false
-    },
-    "maxProperties": {
-      "$ref": "#/definitions/positiveInteger"
-    },
-    "minProperties": {
-      "$ref": "#/definitions/positiveIntegerDefault0"
-    },
-    "required": {
-      "$ref": "#/definitions/stringArray"
-    },
-    "additionalProperties": {
-      "anyOf": [
-        {
-          "type": "boolean"
-        },
-        {
-          "$ref": "#"
-        }
-      ],
-      "default": {}
-    },
-    "definitions": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "properties": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "patternProperties": {
-      "type": "object",
-      "additionalProperties": {
-        "$ref": "#"
-      },
-      "default": {}
-    },
-    "dependencies": {
-      "type": "object",
-      "additionalProperties": {
-        "anyOf": [
-          {
-            "$ref": "#"
-          },
-          {
-            "$ref": "#/definitions/stringArray"
-          }
-        ]
-      }
-    },
-    "enum": {
-      "type": "array",
-      "minItems": 1,
-      "uniqueItems": true
-    },
-    "type": {
-      "anyOf": [
-        {
-          "$ref": "#/definitions/simpleTypes"
-        },
-        {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/simpleTypes"
-          },
-          "minItems": 1,
-          "uniqueItems": true
-        }
-      ]
-    },
-    "allOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "anyOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "oneOf": {
-      "$ref": "#/definitions/schemaArray"
-    },
-    "not": {
-      "$ref": "#"
-    }
-  },
-  "dependencies": {
-    "exclusiveMaximum": [
-      "maximum"
-    ],
-    "exclusiveMinimum": [
-      "minimum"
-    ]
-  },
-  "default": {}
-}

slapos/test/test_json_schema.py

@@ -34,23 +34,36 @@ import slapos.test
 import jsonschema
 
 
-def getSchemaValidator(filename):
-  schema_json_file = "/".join(slapos.test.__file__.split("/")[:-1])
-  schema_json_file += "/%s" % filename
-  with open(schema_json_file, "r") as json_file:
-    json_dict = json.loads(json_file.read())
-    json_file.close()
-  return json_dict
-
-def createValidatorTest(path, json_dict):
-  # Test that json is valid
+def createInstanceParameterSchemaValidatorTest(path):
+  # Test that json is a valid json schema, supports several
+  # validator, depending on the `$schema` defined in the json.
+  validator_dict = {
+    "http://json-schema.org/draft-03/schema#": jsonschema.Draft3Validator,
+    "http://json-schema.org/draft-04/schema": jsonschema.Draft4Validator,
+    "http://json-schema.org/draft-04/schema#": jsonschema.Draft4Validator,
+    "http://json-schema.org/draft-06/schema#": jsonschema.Draft6Validator,
+    "http://json-schema.org/draft-07/schema#": jsonschema.Draft7Validator,
+  }
   def run(self, *args, **kwargs):
     with open(path, "r") as json_file:
-      self.assertEqual(jsonschema.validate(json.load(json_file), json_dict), None)
+      json_dict = json.load(json_file)
+      validator = validator_dict.get(
+        json_dict.get('$schema'),
+        jsonschema.Draft7Validator)
+      validator.check_schema(json_dict)
+  return run
 
+
+def createSoftwareCfgValidatorTest(path, software_cfg_schema):
+  # Test that software json follows the schema for softwares json,
+  # which is defined in schema.json in this directory
+  def run(self, *args, **kwargs):
+    with open(path, "r") as json_file:
+      jsonschema.validate(json.load(json_file), software_cfg_schema)
   return run
 
-def createFormatTest(path, json_dict):
+
+def createFormatTest(path):
   # Test that json match our formatting rules
   def run(self, *args, **kwargs):
     with open(path, "r") as json_file:
@@ -69,23 +82,26 @@ def createFormatTest(path, json_dict):
 
 
 def generateSoftwareCfgTest():
-  json_dict = getSchemaValidator("schema.json")
+  software_cfg_schema = json.load(
+    open(os.path.join(
+        os.path.dirname(slapos.test.__file__),
+        "schema.json"), 'r'))
   base_path = "/".join(slapos.test.__file__.split("/")[:-3])
   for path in glob.glob("%s/software/*/software.cfg.json" % base_path):
     test_name = "test_%s_software_cfg_json" % path.split("/")[-2]
-    setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
-    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
+    setattr(TestJSONSchemaValidation, test_name, createSoftwareCfgValidatorTest(path, software_cfg_schema))
+    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path))
 
 
 def generateJSONSchemaTest():
-  json_dict = getSchemaValidator("metaschema.json")
   base_path = "/".join(slapos.test.__file__.split("/")[:-3])
   for path in glob.glob("%s/software/*/*schema.json" % base_path):
     software_type = path.split("/")[-2]
     filename = path.split("/")[-1].replace("-", "_").replace(".", "_")
     test_name = "test_schema_%s_%s" % (software_type, filename)
-    setattr(TestJSONSchemaValidation, test_name, createValidatorTest(path, json_dict))
-    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path, json_dict))
+    setattr(TestJSONSchemaValidation, test_name, createInstanceParameterSchemaValidatorTest(path))
+    setattr(TestJSONSchemaValidation, test_name + '_format', createFormatTest(path))
+
 
 class TestJSONSchemaValidation(unittest.TestCase):
   pass

software/caddy-frontend/TODO.rst

@@ -2,6 +2,9 @@ Generally things to be done with ``caddy-frontend``:
 
  * return warning on not implemented keys (from ``apache-frontend`` perspective) in master and slave request
  * tests: add assertion with results of promises in etc/promise for each partition
+ * tests: compare python objects where possible instead of instead of strings (eg. load ``rejected-slave-dict``)
+ * tests: swich to `cryptography <https://pypi.org/project/cryptography/>`_ for certificate management
+ * README: cleanup the documentation, explain various specifics
  * check the whole frontend slave snippet with ``caddy -validate`` during buildout run, and reject if does not pass validation
  * ``apache-ca-certificate`` shall be merged with ``apache-certificate``
 

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = ab1795f92e32655d05c662c965d2b1f5
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 44d50bf8391b5a73b2ab72923efe6437
+md5sum = 6a86edb96b171fbd0a59d0adc9cc906b
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 69992ad1dffe1c23237f3ef97193c95c
+md5sum = b30bcd11c545d86c30d05db9cecb4f80
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -46,7 +46,7 @@ md5sum = 7c987ad75fcce6f5b925c7696ff41971
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 69992ad1dffe1c23237f3ef97193c95c
+md5sum = b30bcd11c545d86c30d05db9cecb4f80
 
 [caddy-backend-url-validator]
 filename = templates/caddy-backend-url-validator.in
@@ -98,7 +98,7 @@ md5sum = 176cbca2070734a185a7ae5a4d1181c5
 
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
-md5sum = e018935e2cec2368991f743cab725741
+md5sum = 2b765db72191197122554df17ad471d1
 
 [template-apache-lazy-script-call]
 filename = templates/apache-lazy-script-call.sh.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -122,6 +122,9 @@ context =
 {%       do slave_error_list.append('slave https-url %r invalid' % (slave['https-url'],)) %}
 {%     endif %}
 {%   endif %}
+{%   if slave.get('ssl_ca_crt') and not (slave.get('ssl_crt') and slave.get('ssl_key')) %}
+{%     do slave_error_list.append('ssl_ca_crt is present, so ssl_crt and ssl_key are required')  %}
+{%   endif %}
 {%   if slave.get('ssl_key') and slave.get('ssl_crt') %}
 {%     set key_popen = popen([openssl, 'rsa', '-noout', '-modulus']) %}
 {%     set crt_popen = popen([openssl, 'x509', '-noout', '-modulus']) %}

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -139,7 +139,7 @@ bytes = 8
 {# Set Slave Certificates if needed                   #}
 
 {# Set ssl certificates for each slave #}
-{%   for cert_name in ('ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')%}
+{%   for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
 {%     if cert_name in slave_instance %}
 {%       set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}
 {%       set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) %}

software/caddy-frontend/templates/nginx-notebook-slave.conf.in

@@ -13,9 +13,6 @@ https://{{ slave_parameter.get('custom_domain') }}:{{ slave_parameter['nginx_htt
   errors {{ slave_parameter.get('error_log') }}
 
   tls {{ slave_parameter.get('path_to_ssl_crt', slave_parameter.get('login_certificate')) }} {{ slave_parameter.get('path_to_ssl_key', slave_parameter.get('login_key')) }} {
-{%-   if slave_parameter.get('path_to_ssl_ca_crt') %}
-    clients {{ slave_parameter.get('path_to_ssl_ca_crt') }}
-{%-   endif %}
     alpn http/1.1
   }
 

software/caddy-frontend/test/CA.wildcard.example.com.crt

+-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCCQDaYBkI56KXrjANBgkqhkiG9w0BAQsFADA4MQswCQYDVQQGEwJY
+WDEOMAwGA1UECAwFU3RhdGUxGTAXBgNVBAoMEFdpbGRjYXJkIFJvb3QgQ0EwHhcN
+MTgxMTIxMTAwMzM4WhcNMjgxMTE4MTAwMzM4WjBdMQswCQYDVQQGEwJBVTETMBEG
+A1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkg
+THRkMRYwFAYDVQQDDA0qLmV4YW1wbGUuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEA1A+TzPOPC3qPLq3KfmFpoa6Iubh1OhNFfH7fvCJQ1czTcFLe
+npG4uTjXwcoHbPb6CWq5hfUSY/ucI6mW93gsoW5vsbPFmUFe96fA6uJV17j2IppM
+vNHhcNxNbKxpGnStNO5HTW7q1Qk2yvcx4cL/bPCEaNFwBK1O+NeNz0ZDW8dGifYU
+aVj+qpVlvTi//j8P4FOwSVYvXdMqGH5WaaTquJPVEGg+704tzUxDbCUXrbCVzFgM
+d69sQg0sJQ9MddrsWkQSgcE7cffdC1JdGHCJ/B87iO3pjH2VjFth8EFMcQCn8V8e
+Nz0OpkcsZXuFg3L/3EtMV3ZXSlT6GDxaEcNuvQIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQB4mIEwylSudRONRRMgHDkhMlb8/O2MERYrBmsqatg3eU6/LYZAk/okUI6p
+aBkQ3GnUmA+gQnkhk4hffRk7NqtMq3r5MEcWunu61i45sXnsQh9myHEAeGfDw3wz
+2rkdXAY2jNeQhTBEsErgwKuN86BTFML9cNg2gTKLBbNC1rSJjoMqcKHxrAcsUBip
+bXDQMmNIQkzsc3ml6+17/qfu8+mTZ7J5kEkSbbwRD690LiR6Ltua22GAvuddt53S
+ieyOVVxCDlItquuGfuQ3ay8zlyQjYmoPI5AXE5Wv9W4mF7agc+SYe3myL3xlRbC+
+RD/fQtvjbf/bt9lkgs9DQoITaYvc
+-----END CERTIFICATE-----

software/caddy-frontend/test/CA.wildcard.example.com.key

+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA1A+TzPOPC3qPLq3KfmFpoa6Iubh1OhNFfH7fvCJQ1czTcFLe
+npG4uTjXwcoHbPb6CWq5hfUSY/ucI6mW93gsoW5vsbPFmUFe96fA6uJV17j2IppM
+vNHhcNxNbKxpGnStNO5HTW7q1Qk2yvcx4cL/bPCEaNFwBK1O+NeNz0ZDW8dGifYU
+aVj+qpVlvTi//j8P4FOwSVYvXdMqGH5WaaTquJPVEGg+704tzUxDbCUXrbCVzFgM
+d69sQg0sJQ9MddrsWkQSgcE7cffdC1JdGHCJ/B87iO3pjH2VjFth8EFMcQCn8V8e
+Nz0OpkcsZXuFg3L/3EtMV3ZXSlT6GDxaEcNuvQIDAQABAoIBAQCWdkcEUHvaRSd6
+k0ztxuhQE6pnO/3RKwNOhibxMdfxGtebBvF1yScsJKzRjysdoU9fhx4DchOOZWQv
+2ZCIHfhswhL2HvvA9aUQSzKSde06lr3tZ1WzU6eFkIpO5TXd05Nhzv9AbcapSVRb
+RnFaIiVhgnYweQnmB6HU5fx0aQI6BytP34t3rEZqdy+eYqtq1ZgYC7iXQJct08Sy
+0syR5boW2fKZZin78I+uOWfhD3uUDz7SnetwIEWuaJ/oYXv2YFqm+68XRSo4yi2G
+FlF3CgwecJCaHyEhxMQojlgM61EvEZ0v1FvoMyyQiNmWVSAtbd6BAD5YrdUzk1QO
+mzr3LuTxAoGBAPbnNp/0g25IYj857Q++hSjWsyjLLMfX6+hPsOv2ICL46+xU9P4s
+EeIe8PGgRvUkXiNZ7LRtipsFOB+qNYHknIRtLICyYXfumJH4+05XawHIPWdZNw1X
+762VsiLEHj1nx3tbEpiCApxYJTXat5/3skjibsNjkuV5JAfsiDHPHeOrAoGBANvf
+u1GI2mtUDZ1EJbxJUm3pCUg1aw8jL83OC9miTT36m3V/TiZvc6NEbWM5S8xprwJ1
+FG+MHchTgG3rZR6UhfK7OPb7jD0r1aVnA30NX3NS1zKfMEp2Ry83w9LJX70JbZsg
+ipo09UXSyE1EaeGGIEQ0xqCN/nRiy2KDh4h1gY83AoGAUH50ypU2vB+RGDfUV4uv
+ce79HdGPWd/FI0nHzkXBmGU61SOlc6/+bI/V0ZCFUap3nmLUzsXfqEZ9U6V0KFLV
+zD6jgZmmOSlqSDy6AYJyenRDwIvPbOQ8WYUyPC9gBHjvCgJY/6tzGnGKQBJ8RwTD
+9QsNPVobLADghEzS4ho6Dl0CgYBjCBxIlwk5ujv/j4gnjCbSVlnV6il0QfbwDVQN
+DCsaNVv7ygEbEqvU56cVP+NCCH/I7Y7sxwFLD0ETQSjkYyUJtQXtSFNb4fhybTmH
+A5TwTmma5VRM1YUuYUGUGRtD+5Egg8GpvxyR/GQ3WQ8PgufZkKO+APaQ2Uad8nwD
+HFnkdQKBgG0OlIKuVeLTVhPcQvOmiDEBeAVo1zc4zmA/JLk/euxesEVL9A8xuxsF
+ao0pLvpk/EWQGElxNLNJxbn7AB4uXlpsAvV3xBM88pQIuj2paix1CqSlgfR2F2jE
+t3470UVZV22ECV8vQK/of2byELrMscLExLgKW1lAIqqZ77BntFO9
+-----END RSA PRIVATE KEY-----

software/caddy-frontend/test/CA.wildcard.example.com.root.crt

+-----BEGIN CERTIFICATE-----
+MIIDQzCCAiugAwIBAgIJAOShMXfabB7nMA0GCSqGSIb3DQEBCwUAMDgxCzAJBgNV
+BAYTAlhYMQ4wDAYDVQQIDAVTdGF0ZTEZMBcGA1UECgwQV2lsZGNhcmQgUm9vdCBD
+QTAeFw0xODExMjExMDAxMjVaFw00NjEyMDQxMDAxMjVaMDgxCzAJBgNVBAYTAlhY
+MQ4wDAYDVQQIDAVTdGF0ZTEZMBcGA1UECgwQV2lsZGNhcmQgUm9vdCBDQTCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKq/9H+MvMt5B0vIC2/uEz25jqxT
+Wv36v9/HldZLvEwzOiEd0n53ZZFqVlfOoYyCIoWw1+SPwaAc8Oad8ELfoPUasV65
+xWki9F/tesgPZpyTO7vgpQfX5JVWNw28s13BgRkOO95h4t2S2t1K6sckC0M/B0o3
+wDs/M+74i6wUTHNNXVRejeNPlj9ZSKyfe8rwvY4aNkvW/TKKbaY1yXpQhbeZfU8j
+bk4tv4VOpIIoK7wWnSOcFHMANPqrIhygazI1zdsyySEssQ2TAepUb/zgZgk2IQ61
+GT+h7NVIoYZJKcAYlLapsZJV1d3Ec9y57zTpyfbWsQhmKHCasZeZK5gYqXECAwEA
+AaNQME4wHQYDVR0OBBYEFLZTKR+QsKR9ivZi4uFssy6sB80XMB8GA1UdIwQYMBaA
+FLZTKR+QsKR9ivZi4uFssy6sB80XMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAI8O1u6LnmBkWw3rPOTp+9DD2l0+tU3e51KZfp98Fm3Lo8qF5spAB+Ue
+OiBax9uENzjcHm7T7KdJrQNK6Mmat0VsD1WTR0TK1eBr9+hOh9EE1H5mEmSL0LOs
+ABcCW8DlDv9axWMkFEaJjLfYRUQdvUkb3BwlXo2oq8ectk5ZqS1IF873htYStkvc
+SvrzFpaMhYUIr2e7bvFEJ8XTz9l4eymdOBg3j89gf9OkmPa3FE6Qf7etTkVyOr1t
+7DIuucv2JkWqHnABIWsLgj4bdLWWULASA7FkI0lHxp5/9/OBb7kVlcgQg6c9Wed7
+VA9NaSB3jnBubpEbijnekHqPYO0Bf38=
+-----END CERTIFICATE-----

software/caddy-frontend/test/CA.wildcard.example.com.root.key

+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAqr/0f4y8y3kHS8gLb+4TPbmOrFNa/fq/38eV1ku8TDM6IR3S
+fndlkWpWV86hjIIihbDX5I/BoBzw5p3wQt+g9RqxXrnFaSL0X+16yA9mnJM7u+Cl
+B9fklVY3DbyzXcGBGQ473mHi3ZLa3UrqxyQLQz8HSjfAOz8z7viLrBRMc01dVF6N
+40+WP1lIrJ97yvC9jho2S9b9MoptpjXJelCFt5l9TyNuTi2/hU6kgigrvBadI5wU
+cwA0+qsiHKBrMjXN2zLJISyxDZMB6lRv/OBmCTYhDrUZP6Hs1UihhkkpwBiUtqmx
+klXV3cRz3LnvNOnJ9taxCGYocJqxl5krmBipcQIDAQABAoIBAHOAnbeaUCujlxfg
+Hjx8428hki1nxWmAsUKDFAx99sXk8TFtpvH9eis/r2B+WjFd5lRhJ+lohSX17c9S
+jy/tbkfe4pSdPbi8+Gnbju693D+WKRYSBBCmLe4G//6+4uZM+zMjucPYm0ofCQYg
+o2hKLYQzoo7F37c0LcE9R94DbSOg2tY6HkKNhXoo0KBY424nIlZ82ZvsS7Q+dduB
+txbFvzj0cnrYAJv8QIzcedaFGPN7tkLMZ+PS2rXPG/CkPevKgGwjOx+YyKgQk8GU
+O+YvMQ9/zhOz4ak76UOZF+a21DrVqsKj0BK90SEq10MUnd1riQJ2z2jD8SSChgb6
+rRc4+AECgYEA1NTSkhfxwfgWIjucnWPrtzNLyVqg5Ss+X/XZfN0LNvMje+AZ4wfz
+pLAf8cxnewgNqpR1PeUrjoUwcKrHX7M/MhfEhPKo2LyYDsNtNDcRk3JP047FrDnL
+beVc7lIfsCzuuQHgnUFhBE+8qP69VsHWBq1iQF2NUdW8HLjxTSuAQFECgYEAzWIR
+U/r5QijUE6fcev4FvPBCCF3+c64UEuXV/W4ZURWzOEAcKh9TAHiTKSQ3MDQK7IQY
+YtRbgePnA8Tc0Xj0jSxMtWTX1FanxftosRNgsZD1VKnBViwImuOeZHZYq83qPGT9
+FNUvGMAEAHevNdSdI2k9RSzrB2Lhei6wEYHJryECgYEAoAOWgYKBID2enoRFHsw2
+N5nYe/2ohEQ79DfKGaezO9AXuJXnwJqE4ygMDGaK0qReagaOE0gOtGuM3Nh5Z4lD
+lSzrcq1ipvk8NbVWkHBqxXmnbL6l/fPB79EHSqLx8ioGHZC8yF6US4KLrF9CCU1Y
+1dJb0VrE2mcgtFOUEFoJZdECgYAsGxVRjaIdvRreJbxJhWfCDW6A0X6lZQrWjBkK
+VayGJzzXpZzmxtdSUJJ50VcwuNxnsm5yOtxz5ndj7dDmAy2xa4QFqGRZK0rYT4dK
+D7lCKLkmt1XXpZkreho3xNqB+rSEx8M5yBZXIFU7rHgp/UDJq/4GbwECExAM5x3U
+hKTFQQKBgQCPGglKFvkRkvYonwDmLRiCjBPnpK5YeL/AYJxeD3V9b8V5Arc9ce9y
+PgFgk93RjGlEfLSN0Xbqc6GPIGwg6f5qyHvQ1BpCwupr3lhdsTxwIKbGpAH4Zvmz
+4COcrvkF9gAHaIiH97nLy/9h2EawKqKgJv3R0wfdKvRw4iW/4j4aPw==
+-----END RSA PRIVATE KEY-----

software/caddy-frontend/test/test.py

@@ -385,7 +385,8 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
         partition_parameter_kw=partition_parameter_kw,
         shared=True
       )
-    cls.runComputerPartition()
+    # run partition 4 more times for slaves to be setup
+    cls.runComputerPartition(max_quantity=4)
     for slave_reference, partition_parameter_kw in cls\
             .getSlaveParameterDictDict().items():
       slave_instance = request(
@@ -625,6 +626,29 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'ssl_crt': open('customdomainsslcrtsslkey.example.com.crt').read(),
         'ssl_key': open('customdomainsslcrtsslkey.example.com.key').read(),
       },
+      'custom_domain_ssl_crt_ssl_key_ssl_ca_crt': {
+        'url': cls.backend_url,
+        'custom_domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
+        'ssl_crt': open('CA.wildcard.example.com.crt').read(),
+        'ssl_key': open('CA.wildcard.example.com.key').read(),
+        'ssl_ca_crt': open('CA.wildcard.example.com.root.crt').read(),
+      },
+      'ssl_ca_crt_only': {
+        'url': cls.backend_url,
+        'ssl_ca_crt': open('CA.wildcard.example.com.root.crt').read(),
+      },
+      'ssl_ca_crt_garbage': {
+        'url': cls.backend_url,
+        'ssl_crt': open('CA.wildcard.example.com.crt').read(),
+        'ssl_key': open('CA.wildcard.example.com.key').read(),
+        'ssl_ca_crt': 'some garbage',
+      },
+      'ssl_ca_crt_does_not_match': {
+        'url': cls.backend_url,
+        'ssl_crt': open('wildcard.example.com.crt').read(),
+        'ssl_key': open('wildcard.example.com.key').read(),
+        'ssl_ca_crt': open('CA.wildcard.example.com.root.crt').read(),
+      },
       'type-zope': {
         'url': cls.backend_url,
         'type': 'zope',
@@ -774,13 +798,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     expected_parameter_dict = {
       'monitor-base-url': None,
       'domain': 'example.com',
-      'accepted-slave-amount': '37',
-      'rejected-slave-amount': '3',
-      'slave-amount': '40',
+      'accepted-slave-amount': '40',
+      'rejected-slave-amount': '4',
+      'slave-amount': '44',
       'rejected-slave-dict':
       '{"_apache_custom_http_s-rejected": ["slave not authorized"], '
+      '"_caddy_custom_http_s": ["slave not authorized"], '
       '"_caddy_custom_http_s-rejected": ["slave not authorized"], '
-      '"_caddy_custom_http_s": ["slave not authorized"]}'
+      '"_ssl_ca_crt_only": ["ssl_ca_crt is present, so ssl_crt and ssl_key '
+      'are required"]}'
     }
 
     self.assertEqual(
@@ -1198,13 +1224,96 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     # Caddy: Need to implement similar thing like check-error-on-apache-log
     raise NotImplementedError(self.id())
 
-  @skip('Feature postponed')
   def test_ssl_ca_crt(self):
-    raise NotImplementedError(self.id())
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'custom_domain_ssl_crt_ssl_key_ssl_ca_crt']
+    self.assertLogAccessUrlWithPop(
+      parameter_dict, 'custom_domain_ssl_crt_ssl_key_ssl_ca_crt')
+    self.assertEqual(
+      {
+        'domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
+        'replication_number': '1',
+        'url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
+        'site_url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
+        'secure_access':
+        'https://customdomainsslcrtsslkeysslcacrt.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
 
-  @skip('Feature postponed')
-  def test_path_to_ssl_ca_crt(self):
-    raise NotImplementedError(self.id())
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('CA.wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_ssl_ca_crt_only(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'ssl_ca_crt_only']
+    self.assertEqual(
+      parameter_dict,
+      {
+        'request-error-list': '["ssl_ca_crt is present, so ssl_crt and '
+        'ssl_key are required"]'}
+    )
+
+  def test_ssl_ca_crt_garbage(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'ssl_ca_crt_garbage']
+    self.assertLogAccessUrlWithPop(
+      parameter_dict, 'ssl_ca_crt_garbage')
+    self.assertEqual(
+      {
+        'domain': 'sslcacrtgarbage.example.com',
+        'replication_number': '1',
+        'url': 'http://sslcacrtgarbage.example.com',
+        'site_url': 'http://sslcacrtgarbage.example.com',
+        'secure_access':
+        'https://sslcacrtgarbage.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('CA.wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
+
+  def test_ssl_ca_crt_does_not_match(self):
+    parameter_dict = self.slave_connection_parameter_dict_dict[
+      'ssl_ca_crt_does_not_match']
+    self.assertLogAccessUrlWithPop(
+      parameter_dict, 'ssl_ca_crt_does_not_match')
+    self.assertEqual(
+      {
+        'domain': 'sslcacrtdoesnotmatch.example.com',
+        'replication_number': '1',
+        'url': 'http://sslcacrtdoesnotmatch.example.com',
+        'site_url': 'http://sslcacrtdoesnotmatch.example.com',
+        'secure_access':
+        'https://sslcacrtdoesnotmatch.example.com',
+        'public-ipv4': LOCAL_IPV4,
+      },
+      parameter_dict
+    )
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
+
+    self.assertEqual(
+      open('wildcard.example.com.crt').read(),
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(result, 'Path', '/test-path')
 
   def test_https_only(self):
     parameter_dict = self.slave_connection_parameter_dict_dict[

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -9,6 +9,8 @@ TestSlave-1/var/log/httpd/_custom_domain_access_log
 TestSlave-1/var/log/httpd/_custom_domain_error_log
 TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_access_log
 TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_error_log
+TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_access_log
+TestSlave-1/var/log/httpd/_custom_domain_ssl_crt_ssl_key_ssl_ca_crt_error_log
 TestSlave-1/var/log/httpd/_custom_domain_wildcard_access_log
 TestSlave-1/var/log/httpd/_custom_domain_wildcard_error_log
 TestSlave-1/var/log/httpd/_disabled-cookie-list_access_log
@@ -51,6 +53,10 @@ TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_access_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify-unverified_error_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_access_log
 TestSlave-1/var/log/httpd/_ssl-proxy-verify_ssl_proxy_ca_crt_error_log
+TestSlave-1/var/log/httpd/_ssl_ca_crt_does_not_match_access_log
+TestSlave-1/var/log/httpd/_ssl_ca_crt_does_not_match_error_log
+TestSlave-1/var/log/httpd/_ssl_ca_crt_garbage_access_log
+TestSlave-1/var/log/httpd/_ssl_ca_crt_garbage_error_log
 TestSlave-1/var/log/httpd/_type-notebook_access_log
 TestSlave-1/var/log/httpd/_type-notebook_error_log
 TestSlave-1/var/log/httpd/_type-redirect_access_log

software/caddy-frontend/test/test_data/test.TestSlave.test_monitor_promise_list-CADDY.txt

@@ -6,6 +6,8 @@ TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_custom_domain-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key_ssl_ca_crt-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_custom_domain_ssl_crt_ssl_key_ssl_ca_crt-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_custom_domain_wildcard-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_custom_domain_wildcard-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_disabled-cookie-list-error-log-last-day
@@ -51,6 +53,10 @@ TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-las
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify-unverified-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify_ssl_proxy_ca_crt-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_ssl-proxy-verify_ssl_proxy_ca_crt-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_ssl_ca_crt_does_not_match-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_ssl_ca_crt_does_not_match-error-log-last-hour
+TestSlave-1/etc/monitor-promise/check-_ssl_ca_crt_garbage-error-log-last-day
+TestSlave-1/etc/monitor-promise/check-_ssl_ca_crt_garbage-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_type-eventsource-error-log-last-day
 TestSlave-1/etc/monitor-promise/check-_type-eventsource-error-log-last-hour
 TestSlave-1/etc/monitor-promise/check-_type-notebook-error-log-last-day

software/caddy-frontend/test/utils.py

@@ -260,7 +260,7 @@ class SlapOSInstanceTestCase(unittest.TestCase):
 
 
   @classmethod
-  def runComputerPartition(cls):
+  def runComputerPartition(cls, max_quantity=None):
     """Instanciate the software.
 
     This is the equivalent of doing:
@@ -272,6 +272,9 @@ class SlapOSInstanceTestCase(unittest.TestCase):
 
     This can be called by tests to simulate re-request with different parameters.
     """
+    run_cp_kw = {}
+    if max_quantity is not None:
+      run_cp_kw['max_quantity'] = max_quantity
     logger = logging.getLogger()
     logger.level = logging.DEBUG
     stream = StringIO.StringIO()
@@ -286,7 +289,8 @@ class SlapOSInstanceTestCase(unittest.TestCase):
       cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
         cls.config,
         cluster_configuration=instance_parameter_dict,
-        environment=os.environ)
+        environment=os.environ,
+        **run_cp_kw)
       stream.seek(0)
       stream.flush()
       message = ''.join(stream.readlines()[-100:])

software/erp5/instance-erp5-input-schema.json

@@ -297,7 +297,6 @@
                 "description": "Instantiate a server. If missing, 'storage-dict' must contain the necessary properties to mount the ZODB. The partition reference is 'zodb'.",
                 "$ref": "./instance-zeo-schema.json"
               }
-
             }
           },
           {
@@ -363,7 +362,6 @@
         "$ref": "../caucase/instance-caucase-input-schema.json"
       },
       "type": "object"
-    }
     },
     "test-runner": {
       "description": "Test runner parameters.",
@@ -386,4 +384,5 @@
       },
       "type": "object"
     }
+  }
 }

software/erp5testnode/testsuite/caddy-frontend/software.cfg

@@ -54,8 +54,7 @@ branch = master
 [erp5.util-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/erp5.git
-revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
-branch = master
+revision = 55d7381c54a2e90aff2c4a88943883c35ea111de
 
 [configuration]
 test_software_release = ${:_profile_base_location_}/../../../caddy-frontend/software.cfg

software/erp5testnode/testsuite/plantuml/README.md

+# PlantUML test
+
+This software release is simply to run the test suite from `../../plantuml/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-PLANTUML-TEST` in test result module.
+

software/erp5testnode/testsuite/plantuml/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg.in
+md5sum = 4ab7207a0440a904b4374add9b744312

software/erp5testnode/testsuite/plantuml/instance.cfg.in

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+working-dir = $${buildout:directory}/tmp/
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/plantuml/test
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}
+  SLAPOS_TEST_WORKING_DIR=$${create-directory:working-dir}

software/erp5testnode/testsuite/plantuml/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../component/pillow/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.plantuml-setup]
+<= setup-develop-egg
+egg = slapos.test.plantuml
+setup = ${slapos-repository:location}/software/plantuml/test/
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${pillow-python:egg}
+  ${slapos.test.plantuml-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+erp5.util = 0.4.56
+slapos.recipe.template = 4.3
+plantuml = 0.1.1
+httplib2 = 0.11.3
+image = 1.5.25
+Pillow = 5.3.0
\ No newline at end of file

software/jstestnode/buildout.hash.cfg

@@ -27,4 +27,4 @@ md5sum = 9f22db89a2679534aa8fd37dbca86782
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = bd9ff3543f0dfaf2702624e3ed74d334
+md5sum = b44268d46a41042a879f47babb66c922

software/jstestnode/instance-jstestnode-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate JSTestNode",
   "additionalProperties": false,
+  "required": [
+    "test-suite",
+    "test-runner"
+  ],
   "properties": {
     "test-suite": {
       "description": "The test suite to run",
@@ -18,18 +22,28 @@
       "default": "(the web server started by this instance)",
       "example": "https://softinst1234.host.vifib.net/"
     },
+    "test-runner": {
       "oneOf": [
         {
-        "title": "selenium server",
-        "description": "Configuration for running tests on selenium server",
+          "type": "object",
+          "title": "Selenium Server",
+          "description": "Configuration for Selenium server",
+          "additionalProperties": false,
+          "required": [
+            "desired-capabilities",
+            "server-url",
+            "target"
+          ],
           "properties": {
             "target": {
               "description": "Target system",
+              "type": "string",
               "const": "selenium-server"
             },
             "server-url": {
               "description": "URL of the selenium server",
-            "type": "string"
+              "type": "string",
+              "format": "uri"
             },
             "verify-server-certificate": {
               "description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
@@ -43,43 +57,51 @@
             },
             "desired-capabilities": {
               "description": "Desired browser capabilities",
+              "required": [
+                "browserName"
+              ],
               "type": "object",
               "properties": {
                 "browserName": {
                   "description": "Name of the browser being used, for example firefox, chrome",
-                "type": "string",
-                "required": true
+                  "type": "string"
                 },
                 "version": {
                   "description": "The browser version",
                   "type": "string"
                 }
-            },
-            "additionalProperties": true
+              }
             }
           }
         },
         {
-        "title": "firefox",
-        "description": "Configuration for running tests on local firefox process",
+          "type": "object",
+          "title": "Firefox",
+          "description": "Configuration for Firefox",
+          "additionalProperties": false,
           "properties": {
             "target": {
               "description": "Target system",
               "const": "firefox",
+              "type": "string",
               "default": "firefox"
             }
           }
         },
         {
-        "title": "node",
-        "description": "Configuration for running tests on local nodejs",
+          "type": "object",
+          "title": "NodeJS",
+          "description": "Configuration for NodeJS",
+          "additionalProperties": false,
           "properties": {
             "target": {
               "description": "Target system",
-            "const": "node"
+              "const": "node",
+              "type": "string"
             }
           }
         }
       ]
     }
+  }
 }

software/jstestnode/runTestSuite.in

@@ -25,8 +25,14 @@ BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
 ETC_DIRECTORY = '$${directory:etc}'
 
 def main():
+  parsed_parameters = json.load(
+      open('$${runTestSuite-config-file:rendered}', 'rb'))
+  test_runner = parsed_parameters.get('test-runner', {})
+
   parser = argparse.ArgumentParser(description='Run a test suite.')
-  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite', help='The test suite name',
+                      default=parsed_parameters.get('test-suite', ''),
+                      required=not parsed_parameters.has_key('test-suite'))
   parser.add_argument('--test_suite_title', help='The test suite title')
   parser.add_argument('--test_node_title', help='The test node title')
   parser.add_argument('--project_title', help='The project title')
@@ -40,9 +46,6 @@ def main():
 
   args = parser.parse_args()
 
-  parsed_parameters = json.load(
-      open('$${runTestSuite-config-file:rendered}', 'rb'))
-
   is_browser_running = False
   try:
     test_suite_title = args.test_suite_title or args.test_suite
@@ -62,7 +65,7 @@ def main():
     ##########################
     # Run all tests
     ##########################
-    target = parsed_parameters.get('target', 'firefox')
+    target = test_runner.get('target', 'firefox')
     if target == 'node':
       # Execute NodeJS tests
       result_string = check_output(['${nodejs-output:node}', '${jio-repository.git:location}/test/node.js'],
@@ -92,24 +95,24 @@ def main():
             firefox_binary='${firefox-wrapper:location}',
             executable_path='${geckodriver:location}')
       else:
-        assert target == 'selenium-server', "Unsupported target {}".format(parsed_parameters['target'])
+        assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
         # use a remote connection which verifies TLS certificate
         # workaround for https://github.com/SeleniumHQ/selenium/issues/6534
-        executor = RemoteConnection(parsed_parameters['server-url'], keep_alive=True)
+        executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
         cert_reqs = 'CERT_REQUIRED'
         ca_certs = certifi.where()
-        if not parsed_parameters.get('verify-server-certificate', True):
+        if not test_runner.get('verify-server-certificate', True):
           cert_reqs = 'CERT_NONE'
           ca_certs = None
-        if parsed_parameters.get('server-ca-certificate'):
+        if test_runner.get('server-ca-certificate'):
           ca_certs = os.path.join(ETC_DIRECTORY, "cacerts.pem")
           with open(ca_certs, 'w') as f:
-            f.write(parsed_parameters.get('server-ca-certificate'))
+            f.write(test_runner.get('server-ca-certificate'))
         executor._conn = urllib3.PoolManager(cert_reqs=cert_reqs, ca_certs=ca_certs)
 
         browser = webdriver.Remote(
             command_executor=executor,
-            desired_capabilities=parsed_parameters['desired-capabilities'],
+            desired_capabilities=test_runner['desired-capabilities'],
         )
 
       # adjust path for remote test url

software/kvm/instance-kvm-cluster-input-schema.json

@@ -368,11 +368,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": [
-                "host-name",
-                "ip-address",
-                "ipv6"
-              ],
+              "format": "internet-address",
               "default": "debian.nbd.vifib.net"
             },
             "nbd-port": {

software/kvm/instance-kvm-cluster-simplified-input-schema.json

@@ -64,11 +64,7 @@
               "title": "NBD hostname or IP",
               "description": "hostname (or IP) of the NBD server containing the boot image.",
               "type": "string",
-              "format": [
-                "host-name",
-                "ip-address",
-                "ipv6"
-              ],
+              "format": "internet-address",
               "default": "debian.nbd.vifib.net"
             },
             "nbd-port": {

software/kvm/instance-kvm-input-schema.json

@@ -172,11 +172,7 @@
       "title": "NBD hostname",
       "description": "hostname (or IP) of the NBD server containing the boot image.",
       "type": "string",
-      "format": [
-        "host-name",
-        "ip-address",
-        "ipv6"
-      ],
+      "format": "internet-address",
       "default": "debian.nbd.vifib.net"
     },
     "nbd-port": {
@@ -191,11 +187,7 @@
       "title": "Second NBD hostname",
       "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
       "type": "string",
-      "format": [
-        "host-name",
-        "ip-address",
-        "ipv6"
-      ]
+      "format": "internet-address"
     },
     "nbd2-port": {
       "title": "Second NBD port",

software/plantuml/README.md

+# PlantUML
+
+http://plantuml.com/
+
+PlantUML is a service rendering UML diagrams defined in a simple and intuitive
+language.
+
+Each diagram has a unique URL which is made of an encoded version of the
+diagram code.
+
+Diagrams can be rendered as png, svg or ascii art text.
+
+See http://plantuml.com/PlantUML_Language_Reference_Guide.pdf for a full
+reference on the diagram language.

software/plantuml/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance]
+filename = instance.cfg.in
+md5sum = 8915151103355dd59da31979a14e59fd
+
+[tomcat-server-xml]
+filename = server.xml.in
+md5sum = fdfa7eb249082855039ca98f310324e9
+
+[font.conf]
+filename = font.conf.in
+md5sum = caa3463c9c3766ac5f2396a517d6f926

software/plantuml/font.conf.in

+<?xml version="1.0"?>
+<!DOCTYPE fontconfig SYSTEM "fonts.dtd">
+<fontconfig>
+  <cachedir>$${:fontcache}</cachedir>
+  <!-- installed fonts: $${:installed-fonts} -->
+  <dir>${fonts:location}</dir>
+  <dir>$${:fonts}</dir>
+  <include>${fontconfig:location}/etc/fonts/conf.d</include>
+</fontconfig>
\ No newline at end of file

software/plantuml/instance.cfg.in

+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[fontconfig-conf]
+recipe = slapos.recipe.template
+url = ${font.conf:output}
+output = $${directory:etc}/font.conf
+
+fonts = $${directory:fonts}
+fontcache = $${directory:fontcache}
+installed-fonts =
+  ${liberation-fonts:location}
+  ${ipaex-fonts:location}
+  ${ipa-fonts:location}
+  ${ocrb-fonts:location}
+  ${android-fonts:location}
+  ${dejavu-fonts:location}
+
+[tomcat-server-xml]
+recipe = slapos.recipe.template
+url = ${tomcat-server-xml:output}
+output = $${directory:catalina_conf}/server.xml
+ip = $${instance-parameter:ipv6-random}
+port = 8899
+scheme = https
+
+[tomcat-web-xml]
+recipe = plone.recipe.command
+command = [ -f $${:location} ] || cp ${tomcat9:location}/conf/web.xml $${:location}
+location = $${directory:catalina_conf}/web.xml
+
+[tomcat-keystore]
+recipe = plone.recipe.command
+command =
+  ${java-re-8-output:keytool} \
+    -genkeypair \
+    -alias "tomcat" \
+    -keyalg RSA \
+    -keypass "$${:pass}" \
+    -dname "CN=Web Server,OU=Unit,O=Organization,L=City,S=State,C=Country" \
+    -keystore "$${:file}" \
+    -storepass "$${:pass}"
+file = $${directory:catalina_base}/.keystore
+pass = insecure
+
+[tomcat-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/$${:_buildout_section_name_}
+command-line = ${tomcat9:location}/bin/catalina.sh run
+environment =
+  JRE_HOME=${java-re-8:location}
+  CATALINA_BASE=$${directory:catalina_base}
+  GRAPHVIZ_DOT=${graphviz:location}/bin/dot
+  FONTCONFIG_FILE=$${fontconfig-conf:output}
+  LD_LIBRARY_PATH=${fontconfig:location}/lib:${freetype:location}/lib
+# XXX java is still loading system fonts ... ( even with $JAVA_FONTS or -Djava.awt.fonts )
+# related links:
+#   https://docs.oracle.com/javase/8/docs/technotes/guides/intl/fontconfig.html
+#   https://bugs.openjdk.java.net/browse/JDK-7175487
+hash-files =
+  $${buildout:directory}/software_release/buildout.cfg
+  $${tomcat-server-xml:output}
+
+ip =  $${tomcat-server-xml:ip}
+port =  $${tomcat-server-xml:port}
+scheme = $${tomcat-server-xml:scheme}
+hostname =  [$${:ip}]
+url = $${:scheme}://$${:hostname}:$${:port}
+needs = $${tomcat-web-xml:location}
+
+[promises]
+recipe =
+instance-promises =
+  $${tomcat-listen-promise:path}
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promises}/$${:_buildout_section_name_}
+
+[tomcat-listen-promise]
+<= check-port-listening-promise
+hostname= $${tomcat-instance:ip}
+port = $${tomcat-instance:port}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = $${tomcat-instance:url}
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+services = $${:etc}/service
+promises = $${:etc}/promise
+
+fonts = $${:srv}/fonts/
+fontcache = $${buildout:directory}/.fontcache/
+
+# tomcat directories
+catalina_base = $${:var}/tomcat
+catalina_logs = $${:catalina_base}/logs
+catalina_temp = $${:catalina_base}/temp
+catalina_webapps = $${:catalina_base}/webapps
+catalina_work = $${:catalina_base}/work
+catalina_conf = $${:catalina_base}/conf
\ No newline at end of file

software/plantuml/server.xml.in

+<?xml version='1.0' encoding='utf-8'?>
+<Server port="-1" shutdown="SHUTDOWN">
+  <Service name="Catalina">
+    <Connector
+      protocol="org.apache.coyote.http11.Http11Nio2Protocol"
+      sslImplementationName="org.apache.tomcat.util.net.jsse.JSSEImplementation"
+      address="$${tomcat-server-xml:ip}"
+      port="$${tomcat-server-xml:port}"
+      maxThreads="10"
+      scheme="$${tomcat-server-xml:scheme}"
+      secure="true"
+      clientAuth="false"
+      SSLEnabled="true"
+      keystorePass="$${tomcat-keystore:pass}"
+      keystoreFile="$${tomcat-keystore:file}"
+    />
+
+    <Engine name="Catalina" defaultHost="localhost">
+      <Valve className="org.apache.catalina.valves.AccessLogValve"
+             directory="logs" prefix="localhost_access_log." suffix=".log"
+             pattern="common"/>
+      <Host name="localhost"  appBase="webapps"
+            unpackWARs="true" autoDeploy="true">
+        <Context path="" docBase="${plantuml.war:location}/plantuml.war"
+           privileged="true">
+        </Context>
+      </Host>
+    </Engine>
+  </Service>
+</Server>
\ No newline at end of file

software/plantuml/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../stack/nodejs.cfg
+  ../../component/fontconfig/buildout.cfg
+  ../../component/freetype/buildout.cfg
+  ../../component/graphviz/buildout.cfg
+  ../../component/java/buildout.cfg
+  ../../component/tomcat/buildout.cfg
+  ../../component/fonts/buildout.cfg
+  buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  instance
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+
+[tomcat-server-xml]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:_buildout_section_name_}
+
+[font.conf]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:_buildout_section_name_}
+
+[plantuml.war]
+recipe = slapos.recipe.build:download
+# XXX the war from sourceforge has no version in URL
+url = https://netcologne.dl.sourceforge.net/project/plantuml/plantuml.war#2018-10-21
+md5sum = f956cd28b18ec34740bb1757276f9641
+
+[versions]
+slapos.recipe.template = 4.3

software/plantuml/test/README.md

+Tests for PlantUML software release

software/plantuml/test/data/test_ascii_art.txt

+     ┌───┐          ┌─────┐
+     │Bob│          │Alice│
+     └─┬─┘          └──┬──┘
+       │    hello      │   
+       │──────────────>│   
+       │               │   
+       │   Go Away     │   
+       │<──────────────│   
+     ┌─┴─┐          ┌──┴──┐
+     │Bob│          │Alice│
+     └───┘          └─────┘

software/plantuml/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.plantuml'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' PlantUML",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'psutil',
+        'plantuml',
+        'requests'
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/plantuml/test/test.py

+##############################################################################
+# coding: utf-8
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import textwrap
+import hashlib
+from io import BytesIO
+
+from PIL import Image
+import requests
+import plantuml
+
+import utils
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class PlantUMLTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+
+class TestSimpleDiagram(PlantUMLTestCase):
+  def setUp(self):
+    self.url = self.computer_partition.getConnectionParameterDict()["url"]
+    self.plantuml = plantuml.PlantUML(
+      url='{}/png/'.format(self.url),
+      http_opts={"disable_ssl_certificate_validation": True}
+    )
+
+  def assertImagesSimilar(self, i1, i2, tolerance=5):
+    """Assert images difference between images is less than `tolerance` %.
+   taken from https://rosettacode.org/wiki/Percentage_difference_between_images
+    """
+    pairs = zip(i1.getdata(), i2.getdata())
+    if len(i1.getbands()) == 1:
+      # for gray-scale jpegs
+      dif = sum(abs(p1-p2) for p1,p2 in pairs)
+    else:
+      dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
+
+    ncomponents = i1.size[0] * i1.size[1] * 3
+    self.assertLessEqual((dif / 255.0 * 100) / ncomponents, tolerance)
+
+  def assertImagesSame(self, i1, i2):
+    """Assert images are exactly same."""
+    self.assertImagesSimilar(i1, i2, 0)
+
+  def test_sequence_diagram(self):
+    png = self.plantuml.processes(textwrap.dedent("""\
+    @startuml
+    Bob -> Alice : hello
+    Alice -> Bob : Go Away
+    @enduml
+    """))
+    # we cannot just compare the hash of the image against a reference that can be found with
+    # http://www.plantuml.com/plantuml/png/SoWkIImgAStDuNBAJrBGjLDmpCbCJbMmKiX8pSd9vuBmWC8WMIi5ztm5n_B4IYw7rBmKe1u0
+    # because plantuml include information about the server in the output image metadata ( you can
+    # use http://exif.regex.info/exif.cgi to see metadata )
+    # So we process the image to remove metadata.
+    reference = Image.open(os.path.join(os.path.dirname(__file__), "data", "test_sequence_diagram.png"))
+    self.assertImagesSame(Image.open(BytesIO(png)), reference)
+
+  def test_class_diagram(self):
+    """Class diagram require a working graphviz installation"""
+    png = self.plantuml.processes(textwrap.dedent("""\
+    @startuml
+    class Car
+
+    Driver - Car : drives >
+    Car *- Wheel : have 4 >
+    Car -- Person : < owns
+
+    @enduml
+    """))
+    # rendering is not exactly same on class diagrams, because of fonts and maybe also something in graphviz.
+    # We just compare that image are similar.
+
+    # http://www.plantuml.com/plantuml/png/SoWkIImgAStDuKhEIImkLd1EBEBYSYdAB4ijKj05yHIi5590t685EouGLqjN8JmZDJK7A9wHM9QgO08LrzLL24WjAixF0qhOAEINvnLpSJcavgK0ZGO0
+    reference = Image.open(os.path.join(os.path.dirname(__file__), "data", "test_class_diagram.png"))
+    self.assertImagesSimilar(Image.open(BytesIO(png)), reference)
+
+  def test_fonts(self):
+    """Test slapos provided fonts are used"""
+    png = self.plantuml.processes(textwrap.dedent("""\
+    @startuml
+    listfonts 私は申し訳ありません：私は日本語を話さない。Je ne parle pas japonais.
+    @enduml
+    """))
+    # URL on the reference implementation would be
+    # http://www.plantuml.com/plantuml/png/SoWkIImgAStDuSh9B2v9oyyhALPulhpnSUFwvrCsFswS_c85a6nwtDJrk77VuyRPZviclzyp2wBWsVIbp-QiUR5gtkEcIIzMRdpSEFLnuwh7ZIsF6vgyKXNoKXKA4ejoG6InGbPYGNvUOcQn7fT3QbuAq3O0
+    # but we don't have same fonts, so we compare against the fonts of a slapos instance.
+    reference = Image.open(os.path.join(os.path.dirname(__file__), "data", "test_fonts.png"))
+    self.assertImagesSimilar(Image.open(BytesIO(png)), reference)
+
+  def test_editor(self):
+    """Test the embedded editor"""
+    r = requests.get('{}/uml/'.format(self.url), verify=False)
+    self.assertEqual(r.status_code, requests.codes.ok)
+
+  def test_svg(self):
+    """Test svg rendering"""
+    image_key = plantuml.deflate_and_encode(textwrap.dedent("""\
+    @startuml
+    Bob -> Alice : hello
+    Alice -> Bob : Go Away
+    @enduml
+    """))
+    svg = requests.get('{}/svg/{}'.format(self.url, image_key), verify=False).text
+    self.assertIn('<?xml version="1.0" encoding="UTF-8"', svg)
+
+  def test_ascii_art(self):
+    """Test ascii art rendering"""
+    image_key = plantuml.deflate_and_encode(textwrap.dedent("""\
+    @startuml
+    Bob -> Alice : hello
+    Alice -> Bob : Go Away
+    @enduml
+    """))
+    aa = requests.get('{}/txt/{}'.format(self.url, image_key), verify=False).content
+    with open(os.path.join(os.path.dirname(__file__), "data", "test_ascii_art.txt"), 'rb') as reference:
+      self.assertEqual(aa, reference.read())
+
+
+class ServicesTestCase(PlantUMLTestCase):
+  @staticmethod
+  def generateHashFromFiles(file_list):
+    hasher = hashlib.md5()
+    for path in file_list:
+      with open(path, 'r') as afile:
+        buf = afile.read()
+      hasher.update("%s\n" % len(buf))
+      hasher.update(buf)
+    hash = hasher.hexdigest()
+    return hash
+
+  def test_hashes(self):
+    hash_files = [
+      'software_release/buildout.cfg',
+      'var/tomcat/conf/server.xml'
+    ]
+    expected_process_names = [
+      'tomcat-instance-{hash}-on-watch',
+    ]
+
+    supervisor = self.getSupervisorRPCServer().supervisor
+    process_names = [process['name']
+                     for process in supervisor.getAllProcessInfo()]
+
+    hash_files = [os.path.join(self.computer_partition_root_path, path)
+                  for path in hash_files]
+
+    for name in expected_process_names:
+      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      expected_process_name = name.format(hash=h)
+
+      self.assertIn(expected_process_name, process_names)

software/slapos-testing/buildout.hash.cfg

@@ -15,5 +15,5 @@
 
 [template]
 filename = instance.cfg
-md5sum = d361db5f94e8c568e2aa44014d0ba91b
+md5sum = f27dd22ce5b925dd81f2fcd988b10691
 

software/slapos-testing/instance.cfg

@@ -26,6 +26,10 @@ srv = $${buildout:directory}/srv
 recipe = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
 
+[kedifa]
+<= download-source
+repository = ${kedifa-repository:location}
+
 [caucase]
 <= download-source
 repository = ${caucase-repository:location}
@@ -69,7 +73,7 @@ wrapper-path = $${create-directory:bin}/runTestSuite
 command-line =
   ${buildout:bin-directory}/runTestSuite
   --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
-  --source_code_path_list=$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location}
+  --source_code_path_list=$${kedifa:location},$${caucase:location},$${erp5.util:location},$${slapos.cookbook:location},$${slapos.core:location},$${slapos.recipe.build:location},$${slapos.recipe.cmmi:location},$${slapos.recipe.template:location},$${slapos.toolbox:location},$${slapos.libnetworkcache:location}
 
 # Notes about environment:
 # * slapos.cookbook:wrapper does not seem to allow "extending" PATH. Tests

software/slapos-testing/software.cfg

@@ -34,6 +34,11 @@ recipe = zc.recipe.egg:develop
 egg = caucase
 setup = ${caucase-repository:location}
 
+[kedifa-setup]
+<= setup-develop-egg
+egg = kedifa
+setup = ${kedifa-repository:location}
+
 [slapos.libnetworkcache-setup]
 <= setup-develop-egg
 egg = slapos.libnetworkcache
@@ -93,6 +98,7 @@ eggs =
   ${bcrypt:egg}
   dnspython
   Jinja2
+  ${kedifa-setup:egg}
   ${caucase-setup:egg}
   ${erp5.util-setup:egg}
   ${slapos.cookbook-setup:egg}
@@ -119,6 +125,10 @@ git-executable = ${git:location}/bin/git
 forbid-download-cache = true
 branch = master
 
+[kedifa-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/kedifa.git
+
 [caucase-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/caucase.git
@@ -189,7 +199,7 @@ pycurl = 7.43.0.2
 pyflakes = 2.0.0
 smmap2 = 2.0.4
 zope.testing = 4.6.2
-
+urllib3 = 1.24.1 
 # Required by:
 # caucase
 PyJWT = 1.6.4

software/wendelin/software.cfg

@@ -3,12 +3,14 @@ versions = versions
 extends =
   ../../component/wendelin.core/buildout.cfg
   ../../component/msgpack-python/buildout.cfg
+  ../../component/msgpack-numpy/buildout.cfg
   ../../component/scipy/buildout.cfg
   ../../software/erp5/software.cfg
 parts +=
   wendelin
   scipy
   msgpack-python
+  msgpack-numpy
   ipython
   wendelin.core
   jupyter
@@ -25,6 +27,7 @@ eggs +=
   astor
   ${scipy:egg}
   ${msgpack-python:egg}
+  ${msgpack-numpy:egg}
   ${wendelin.core:egg}
   ${ipython:egg}
   
@@ -88,5 +91,6 @@ revision = 262f87ea056ceba6f90dff0ba76b2c70867e0f9a
 revision = 318efce0bfe8a0682d316a52051661277035a17d
 
 [versions]
-msgpack-python = 0.5.6
+msgpack = 0.5.6
+msgpack-numpy = 0.4.4.2
 wendelin.core = 0.12

stack/erp5/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
-md5sum = 1af531c51f575a1d1362f2ca2d61620d
+md5sum = e405227118a70fbf949a6414469989db
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in

stack/erp5/instance-mariadb-start-clone-from-backup.sh.in

@@ -88,6 +88,7 @@ zcat "$BACKUP" | "$CLIENT" -u root
 
 echo "Configuring server as slave..."
 if [ "$MASTER_USE_GTID" -eq 1 ]; then
+  "$CLIENT" -u root -e "$SQL_SET_GTID"
   MASTER_USE_GTID_SQL="current_pos"
 else
   MASTER_USE_GTID_SQL="NO"
@@ -104,8 +105,14 @@ fi
   MASTER_SSL_VERIFY_SERVER_CERT=1,
   MASTER_USE_GTID=$MASTER_USE_GTID_SQL;
 "
-"$CLIENT" -u root -e "$SQL_CHANGE_MASTER"
-test "$MASTER_USE_GTID" -eq 1 && "$CLIENT" -u root -e "$SQL_SET_GTID"
+if [ "$MASTER_USE_GTID" -eq 0 ]; then
+  # No GTID, use binlog name & offset as provided by backup file.
+  # Example: CHANGE MASTER TO MASTER_LOG_FILE='binlog.003447', MASTER_LOG_POS=360;
+  # Notes:
+  # - Must happen after setting MASTER_HOST & MASTER_PORT.
+  # - Implicitly sets MASTER_USE_GTID=NO if it was set before.
+  "$CLIENT" -u root -e "$SQL_CHANGE_MASTER"
+fi
 "$CLIENT" -u root -e "START SLAVE;"
 
 echo "Stopping mariadb..."

stack/slapos.cfg

@@ -98,7 +98,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
-setuptools = 33.1.1
+setuptools = 40.4.3
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.5.2+slapos013
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
@@ -133,12 +133,12 @@ pyOpenSSL = 18.0.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
-six = 1.10.0
+six = 1.11.0
 slapos.cookbook = 1.0.75
-slapos.core = 1.4.11
+slapos.core = 1.4.13
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
-slapos.libnetworkcache = 0.15
+slapos.libnetworkcache = 0.16
 slapos.rebootstrap = 4.1
 slapos.recipe.build = 0.39
 slapos.recipe.cmmi = 0.8
@@ -149,7 +149,7 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.8
+# slapos.core==1.4.13
 Flask = 0.12
 
 # Required by:
@@ -193,23 +193,31 @@ erp5.util = 0.4.51
 feedparser = 5.2.1
 
 # Required by:
-# jsonschema==2.6.0
+# jsonschema==3.0.0a3
 functools32 = 3.2.3.post2
 
+# Required by:
+# jsonschema==3.0.0a3
+attrs = 18.2.0
+
+# Required by:
+# jsonschema==3.0.0a3
+pyrsistent = 0.14.5
+
 # Required by:
 # cryptography==1.8.1
 ipaddress = 1.0.18
 
 # Required by:
 # slapos.cookbook==1.0.62
-jsonschema = 2.6.0
+jsonschema = 3.0.0a3
 
 # Required by:
 # slapos.toolbox==0.81
 lockfile = 0.12.2
 
 # Required by:
-# slapos.core==1.4.8
+# slapos.core==1.4.13
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -238,15 +246,15 @@ python-dateutil = 2.7.3
 rpdb = 0.1.5
 
 # Required by:
-# slapos.core==1.4.8
+# slapos.core==1.4.13
 supervisor = 3.3.3
 
 # Required by:
-# slapos.core==1.4.8
+# slapos.core==1.4.13
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.8
+# slapos.core==1.4.13
 zope.interface = 4.3.3
 
 [networkcache]