Update Release candidate

Conflicts:
	software/erp5/software.cfg

README.rst

@@ -2,3 +2,14 @@ slapos.cookbook
 ===============
 
 Cookbook of SlapOS recipes.
+
+
+testing
+=======
+
+Unit tests for recipes can be found under ``slapos/test/recipe``. To run the
+tests use provided unittest.defaultTestLoader inside ``slapos/test/test_recipe``
+by invoking 
+
+    python setup.py test --test-suite slapos.test.test_recipe.additional_tests
+

component/alsa/buildout.cfg

@@ -5,15 +5,12 @@ parts =
 [alsa]
 # Contains libasound
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.0.24.1.tar.bz2
-md5sum = 7cc05f25e1d5b65da8fb3fdcd540f226
+url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.3.tar.bz2
+md5sum = eefe5992567ba00d6110a540657aaf5c
 configure-options =
   --disable-static
   --disable-aload
-  --disable-mixer
   --disable-rawmidi
-  --disable-hwdep
-  --disable-seq
   --disable-ucm
   --disable-alisp
   --disable-old-symbols

component/apache/apache-backend.conf.in

@@ -108,12 +108,16 @@ SSLProxyEngine On
 
 # As backend is trusting REMOTE_USER header unset it always
 RequestHeader unset REMOTE_USER
+RequestHeader unset SSL_CLIENT_SERIAL
 {% if parameter_dict['ca-cert'] -%}
-SSLVerifyClient require
+SSLVerifyClient optional
 RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
 SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+{%   if parameter_dict['crl'] -%}
 SSLCARevocationCheck chain
 SSLCARevocationFile {{ parameter_dict['crl'] }}
+{%-   endif %}
 {%- endif %}
 
 ErrorLog "{{ parameter_dict['error-log'] }}"
@@ -128,12 +132,24 @@ CustomLog "{{ parameter_dict['access-log'] }}" combined
 </Directory>
 
 RewriteEngine On
-{% for port, _, backend in parameter_dict['backend-list'] -%}
+{% for port, _, backend, enable_authentication in parameter_dict['backend-list'] -%}
 {%   for ip in parameter_dict['ip-list'] -%}
 Listen {{ ip }}:{{ port }}
 {%   endfor -%}
 <VirtualHost *:{{ port }}>
   SSLEngine on
+{% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
+  SSLVerifyClient require
+  SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+  SSLCARevocationCheck chain
+  SSLCARevocationFile {{ parameter_dict['crl'] }}
+
+  LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+
+  # We would like to separate the the authentificated logs.
+  ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log"
+  CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined
+{% endif -%}
   RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
 </VirtualHost>
 {% endfor -%}

component/apache/buildout.cfg

@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-version = 2.4.25
+version = 2.4.26
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 2826f49619112ad5813c0be5afcc7ddb
+md5sum = d4d47749a44461cb2e6c9d78a22b522b
 pre-configure =
   cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
   cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
@@ -190,5 +190,5 @@ make-targets =
 [template-apache-backend-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/apache-backend.conf.in
-md5sum = feef079241bda3407b7ceed5876cb61f
+md5sum = 8f1f92ad308ab6bad973c790ee583428
 mode = 640

component/automake/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 
 [automake]
 recipe = slapos.recipe.cmmi
-md5sum = 9a1ddb0e053474d9d1105cfe39b0c48d
-url = http://ftp.gnu.org/gnu/automake/automake-1.15.tar.xz
+md5sum = 24cd3501b6ad8cd4d7e2546f07e8b4d4
+url = http://ftp.gnu.org/gnu/automake/automake-1.15.1.tar.xz
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/automake-1.15-shebang_workaround.patch#203f9199b0e629de3630b5959f8cf73e

component/bcrypt/buildout.cfg

+[buildout]
+extends =
+  ../libffi/buildout.cfg
+
+parts =
+  bcrypt
+
+[bcrypt-env]
+CPPFLAGS=-I${libffi:location}/include
+CFLAGS=-I${libffi:location}/include
+
+[bcrypt]
+recipe = zc.recipe.egg:custom
+egg = bcrypt
+
+rpath =
+  ${libffi:location}/lib/
+environment = bcrypt-env

component/ca-certificates/buildout.cfg

@@ -11,9 +11,9 @@ parts =
 
 [ca-certificates]
 recipe = slapos.recipe.cmmi
-url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130.tar.xz
+url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130+nmu1.tar.xz
 patch-binary = ${patch:location}/bin/patch
-md5sum = 1a0a3a1b3390dc83affed4b0c2ae1c05
+md5sum = a09e8b63126188fd0ed77f6fbaf5d35f
 patches =
   ${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
   ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b

component/consul/buildout.cfg

+[buildout]
+
+
+parts =
+  consul
+
+[consul]
+recipe = slapos.recipe.build
+
+# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
+url_x86-64 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_amd64.zip
+url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
+
+# supported architectures md5sums
+md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
+md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
+
+# script to install.
+script =
+  location = %(location)r
+  self.failIfPathExists(location)
+  import sys
+  ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
+  WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
+  platform = guessPlatform()
+  url = self.options['url_' + platform]
+  md5sum = self.options['md5sum_' + platform]
+  extract_dir = self.extract(self.download(url, md5sum))
+  shutil.move(extract_dir, location)

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.54.0.tar.bz2
-md5sum = 89bb7ba87384dfbf4f1a3f953da42458
+url = http://curl.haxx.se/download/curl-7.54.1.tar.bz2
+md5sum = 6b6eb722f512e7a24855ff084f54fe55
 configure-options =
   --disable-static
   --disable-ldap

component/firewalld/buildout.cfg

@@ -68,7 +68,7 @@ configure-options =
   --disable-static
 
 environment =
-  PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
   LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi

component/fonts/buildout.cfg

@@ -20,10 +20,9 @@ strip-top-level-dir = true
 
 # Liberation(tm) Fonts - a font family which aims at metric
 # compatibility with Arial, Times New Roman, and Courier New.
-# https://fedorahosted.org/liberation-fonts/
 [liberation-fonts]
 <= fonts-base
-url = https://fedorahosted.org/releases/l/i/liberation-fonts/liberation-fonts-ttf-2.00.1.tar.gz
+url = https://releases.pagure.org/liberation-fonts/liberation-fonts-ttf-2.00.1.tar.gz
 md5sum = 5c781723a0d9ed6188960defba8e91cf
 
 # IPAex and IPA Font - Japanese fonts provided by IPA

component/glib/buildout.cfg

@@ -14,7 +14,7 @@ recipe = slapos.recipe.cmmi
 url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/glib-2.50.2.tar.xz
 md5sum = 5eeb2bfaf78a07be59585e8b6e80b1d6
 configure-options =
-  --with-python=${python2.7:location}/bin/python2.7
+  --with-python=${buildout:executable}
   --disable-libmount
   --disable-static
   --disable-selinux

component/gnutls/buildout.cfg

 [buildout]
 extends =
+  ../gettext/buildout.cfg
   ../gmp/buildout.cfg
   ../nettle/buildout.cfg
   ../p11-kit/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../lunzip/buildout.cfg
-  ../zlib/buildout.cfg
 
 parts = gnutls
 
-[gpg-error]
+[gpg-common]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.26.tar.bz2
-md5sum = 97456709dbbcbb69354317ffe3e347cd
+url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_section_name_}-${:version}.tar.bz2
+# we'd like to combine <= and +=
+configure-options = ${:configure-options-extra}
+  --enable-option-checking=fatal
+  --disable-dependency-tracking
+environment = ${:environment-extra}
+  PATH=${gettext:location}/bin:%(PATH)s
+environment-extra =
 
-[gcrypt]
-recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.6.tar.bz2
-md5sum = 944cf6595021d0c33478148a315b335b
-configure-options =
-  --with-gpg-error-prefix=${gpg-error:location}
-environment =
-  CPPFLAGS=-I${gpg-error:location}/include
-  LDFLAGS=-L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
+[libgpg-error]
+<= gpg-common
+version = 1.27
+md5sum = 5217ef3e76a7275a2a3b569a12ddc989
+configure-options-extra =
+  --disable-doc
+  --disable-tests
+
+[libgcrypt]
+<= gpg-common
+version = 1.7.7
+md5sum = d1769481b1b506a632fd66c5e5f62e41
+configure-options-extra =
+  --with-gpg-error-prefix=${libgpg-error:location}
+  --disable-doc
+environment-extra =
+  LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib
 
 [gnutls]
-recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.26.tar.xz
-md5sum = 9840c06019bfa7db07ed8fd5e63106d5
-configure-options =
+<= gpg-common
+url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.13.tar.xz
+md5sum = 4fd41ad86572933c2379b4cc321a0959
+configure-options-extra =
+  --disable-doc
   --disable-static
   --disable-libdane
   --disable-guile
-  --disable-crywrap
   --without-tpm
+  --with-included-unistring
+pkg-config-path = ${libtasn1:location}/lib/pkgconfig:${nettle:location}/lib/pkgconfig:${p11-kit:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${nettle:location}/lib/pkgconfig:${p11-kit:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  CPPFLAGS=-I${gmp:location}/include -I${zlib:location}/include
-  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  PKG_CONFIG_PATH=${:pkg-config-path}
+  CPPFLAGS=-I${gmp:location}/include -I${libtasn1:location}/include
+  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${nettle:location}/lib -Wl,-rpath=${zlib:location}/lib

component/groonga/buildout.cfg

@@ -14,8 +14,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-7.0.0.tar.gz
-md5sum = 1b5383359ba1901e66b9e0910712569f
+url = http://packages.groonga.org/source/groonga/groonga-7.0.3.tar.gz
+md5sum = 4bddc049d43e8bbe40fae636ce03ac50
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/haproxy/buildout.cfg

@@ -12,8 +12,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.2.tar.gz
-md5sum = 7330b36f3764ebe409e9305803dc30e2
+url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz
+md5sum = ed84c80cb97852d2aa3161ed16c48a1c
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/headless-chromium/buildout.cfg

+[buildout]
+extends =
+  ../binutils/buildout.cfg
+  ../bison/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../gperf/buildout.cfg
+  ../ninja/buildout.cfg
+  ../freetype/buildout.cfg
+  ../cmake/buildout.cfg
+  ../git/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../cups/buildout.cfg
+  ../coreutils/buildout.cfg
+  ../depot_tools/buildout.cfg
+  ../findutils/buildout.cfg
+  ../fontconfig/buildout.cfg
+  ../gettext/buildout.cfg
+  ../glib/buildout.cfg
+  ../gtk-2/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../libffi/buildout.cfg
+  ../libpng/buildout.cfg
+  ../libxml2/buildout.cfg
+  ../mesa/buildout.cfg
+  ../nspr/buildout.cfg
+  ../nss/buildout.cfg
+  ../pcre/buildout.cfg
+  ../sqlite3/buildout.cfg
+  ../xorg/buildout.cfg
+  ../zlib/buildout.cfg
+
+parts =
+  chromium
+
+[gconf]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
+md5sum = 2b16996d0e4b112856ee5c59130e822c
+configure-options = --disable-orbit --disable-static
+environment =
+  PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
+
+[chromedriver]
+recipe = hexagonit.recipe.download
+url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
+md5sum = a72088c0a6b018ded2c0fff616da8f65
+
+[chromium-download]
+recipe = plone.recipe.command
+# This revision is 56.0.2924.122. Because the chromedriver only support some certain
+# version. Which version 56 is in the middle of 55-57.
+revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
+path = ${buildout:parts-directory}/${:_buildout_section_name_}
+command =
+  set -e
+  PATH=${depot_tools:location}:${git:location}/bin:$PATH
+  [ -d ${:path} ] && rm -r ${:path}
+  mkdir -p ${:path}
+  cd ${:path}
+  # Do never use `fetch` with the `--no-history` option unless you find an
+  # option to fetch directly at the wanted revision. `--no-history` could
+  # reduce the download size significantly but even if it retrieves enough
+  # commits at the time you test this section, development continues upstream
+  # and at some point the next command (gclient) would break.
+  # ...
+  # This command only could work in an empty dir.
+  fetch --nohooks chromium
+  gclient sync --revision ${:revision} --with_branch_heads
+stop-on-error = true
+
+[chromium]
+recipe = slapos.recipe.cmmi
+path = ${chromium-download:path}/src
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+configure-command =
+  gclient runhooks
+  mkdir -p ${:location}
+  echo 'use_udev = true
+  is_debug = false
+  enable_nacl = false' > ${:location}/args.gn
+  gn gen ${:location}
+
+# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
+make-binary = ninja -C ${:location} chrome
+environment =
+  PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
+  PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:path}/:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
+  CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
+  LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH

component/intltool/buildout.cfg

 [buildout]
 extends =
   ../gettext/buildout.cfg
+  ../patch/buildout.cfg
   ../perl/buildout.cfg
   ../perl-XML-Parser/buildout.cfg
 
@@ -9,10 +10,13 @@ parts =
 
 [intltool]
 recipe = slapos.recipe.cmmi
-url = http://edge.launchpad.net/intltool/trunk/0.41.1/+download/intltool-0.41.1.tar.gz
-md5sum = d6c91bf06681919ccfdf3624035b75dc
+url = http://edge.launchpad.net/intltool/trunk/0.51.0/+download/intltool-0.51.0.tar.gz
+md5sum = 12e517cac2b57a0121cda351570f1e63
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/intltool-0.51.0.regexp.patch#3e7736d5ce7876e9a8d078c35e7f327b
 depends =
   ${perl-XML-Parser:location}
 environment =
-  PATH=${perl:location}/bin:${gettext:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${perl:location}/bin:${gettext:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${perl:location}/lib/pkgconfig

component/intltool/intltool-0.51.0.regexp.patch

+--- intltool-0.51.0.orig/intltool-update.in	2015-03-09 02:39:54.000000000 +0100
++++ intltool-0.51.0.orig/intltool-update.in	2015-06-19 01:52:07.171228154 +0200
+@@ -1062,7 +1062,7 @@
+ 	}
+     }
+ 
+-    if ($str =~ /^(.*)\${?([A-Z_]+)}?(.*)$/)
++    if ($str =~ /^(.*)\$\{?([A-Z_]+)}?(.*)$/)
+     {
+ 	my $rest = $3;
+ 	my $untouched = $1;
+@@ -1190,10 +1190,10 @@
+ 	$name    =~ s/\(+$//g;
+ 	$version =~ s/\(+$//g;
+ 
+-	$varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\${?AC_PACKAGE_NAME}?/);
+-	$varhash{"PACKAGE"} = $name if (not $name =~ /\${?PACKAGE}?/);
+-	$varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\${?AC_PACKAGE_VERSION}?/);
+-	$varhash{"VERSION"} = $version if (not $name =~ /\${?VERSION}?/);
++	$varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\$\{?AC_PACKAGE_NAME}?/);
++	$varhash{"PACKAGE"} = $name if (not $name =~ /\$\{?PACKAGE}?/);
++	$varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\$\{?AC_PACKAGE_VERSION}?/);
++	$varhash{"VERSION"} = $version if (not $name =~ /\$\{?VERSION}?/);
+     }
+ 
+     if ($conf_source =~ /^AC_INIT\(([^,\)]+),([^,\)]+)[,]?([^,\)]+)?/m)
+@@ -1219,11 +1219,11 @@
+ 	$version =~ s/\(+$//g;
+         $bugurl  =~ s/\(+$//g if (defined $bugurl);
+ 
+-	$varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\${?AC_PACKAGE_NAME}?/);
+-	$varhash{"PACKAGE"} = $name if (not $name =~ /\${?PACKAGE}?/);
+-	$varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\${?AC_PACKAGE_VERSION}?/);
+-	$varhash{"VERSION"} = $version if (not $name =~ /\${?VERSION}?/);
+-        $varhash{"PACKAGE_BUGREPORT"} = $bugurl if (defined $bugurl and not $bugurl =~ /\${?\w+}?/);
++	$varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\$\{?AC_PACKAGE_NAME}?/);
++	$varhash{"PACKAGE"} = $name if (not $name =~ /\$\{?PACKAGE}?/);
++	$varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\$\{?AC_PACKAGE_VERSION}?/);
++	$varhash{"VERSION"} = $version if (not $name =~ /\$\{?VERSION}?/);
++        $varhash{"PACKAGE_BUGREPORT"} = $bugurl if (defined $bugurl and not $bugurl =~ /\$\{?\w+}?/);
+     }
+ 
+     # \s makes this not work, why?

component/libexpat/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [libexpat]
 recipe = slapos.recipe.cmmi
-url = http://downloads.sourceforge.net/project/expat/expat/2.2.0/expat-2.2.0.tar.bz2
-md5sum = 2f47841c829facb346eb6e3fab5212e2
+url = http://downloads.sourceforge.net/project/expat/expat/2.2.1/expat-2.2.1.tar.bz2
+md5sum = d9c3baeab58774cefc2f04faf29f2cf8
 configure-options =
   --disable-static

component/libtasn1/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [libtasn1]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.8.tar.gz
-md5sum = 9a6767705725544f2b86670dcfb34107
+url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.12.tar.gz
+md5sum = 5c724bd1f73aaf4a311833e1cd297b21
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/libtiff/buildout.cfg

@@ -18,7 +18,7 @@ configure-options =
   --disable-lzma
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/tiff_4.0.7-5.debian.patch#f132ec0dc5cde03f9bbbe90b7a79fed4
+  ${:_profile_base_location_}/tiff_4.0.7-6.debian.patch#b9249bb352e693b7975f9d343018994d
 environment =
   CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
   LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/libtiff/tiff_4.0.7-5.debian.patch → component/libtiff/tiff_4.0.7-6.debian.patch

@@ -1156,3 +1156,765 @@ index bdf754c..8bbcd52 100644
  	if (shortv != config && bitspersample != 8 && samplesperpixel > 1) {
  		fprintf(stderr,
  		    "%s: Cannot handle different planar configuration w/ bits/sample != 8\n",
+From 47f2fb61a3a64667bce1a8398a8fcb1b348ff122 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 12:15:01 +0000
+Subject: [PATCH] * libtiff/tif_jpeg.c: avoid integer division by zero in
+ JPEGSetupEncode() when horizontal or vertical sampling is set to 0. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2653
+
+---
+ ChangeLog          | 6 ++++++
+ libtiff/tif_jpeg.c | 7 +++++++
+ 2 files changed, 13 insertions(+)
+
+diff --git a/ChangeLog b/ChangeLog
+index c82bc76..a7208f5 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,11 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_jpeg.c: avoid integer division by zero in
++	JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* tools/tiffcp.c: error out cleanly in cpContig2SeparateByRow and
+ 	cpSeparate2ContigByRow if BitsPerSample != 8 to avoid heap based overflow.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2656 and
+diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
+index 38595f9..6c17c38 100644
+--- a/libtiff/tif_jpeg.c
++++ b/libtiff/tif_jpeg.c
+@@ -1626,6 +1626,13 @@ JPEGSetupEncode(TIFF* tif)
+ 	case PHOTOMETRIC_YCBCR:
+ 		sp->h_sampling = td->td_ycbcrsubsampling[0];
+ 		sp->v_sampling = td->td_ycbcrsubsampling[1];
++                if( sp->h_sampling == 0 || sp->v_sampling == 0 )
++                {
++                    TIFFErrorExt(tif->tif_clientdata, module,
++                            "Invalig horizontal/vertical sampling value");
++                    return (0);
++                }
++
+ 		/*
+ 		 * A ReferenceBlackWhite field *must* be present since the
+ 		 * default value is inappropriate for YCbCr.  Fill in the
+From 3cfd62d77c2a7e147a05bd678524c345fa9c2bb8 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 13:28:01 +0000
+Subject: [PATCH] * libtiff/tif_dirread.c: avoid division by floating point 0
+ in TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
+ and return 0 in that case (instead of infinity as before presumably)
+ Apparently some sanitizers do not like those divisions by zero. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2644
+
+---
+ ChangeLog             |  8 ++++++++
+ libtiff/tif_dirread.c | 10 ++++++++--
+ 2 files changed, 16 insertions(+), 2 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 6a752cd..722a405 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,13 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_dirread.c: avoid division by floating point 0 in
++	TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
++	and return 0 in that case (instead of infinity as before presumably)
++	Apparently some sanitizers do not like those divisions by zero.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2644
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_jpeg.c: avoid integer division by zero in
+ 	JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 570d0c3..8a1e42a 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -2872,7 +2872,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedRational(TIFF* tif, TIFFD
+ 		m.l = direntry->tdir_offset.toff_long8;
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabArrayOfLong(m.i,2);
+-	if (m.i[0]==0)
++        /* Not completely sure what we should do when m.i[1]==0, but some */
++        /* sanitizers do not like division by 0.0: */
++        /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
++	if (m.i[0]==0 || m.i[1]==0)
+ 		*value=0.0;
+ 	else
+ 		*value=(double)m.i[0]/(double)m.i[1];
+@@ -2900,7 +2903,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedSrational(TIFF* tif, TIFF
+ 		m.l=direntry->tdir_offset.toff_long8;
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabArrayOfLong(m.i,2);
+-	if ((int32)m.i[0]==0)
++        /* Not completely sure what we should do when m.i[1]==0, but some */
++        /* sanitizers do not like division by 0.0: */
++        /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
++	if ((int32)m.i[0]==0 || m.i[1]==0)
+ 		*value=0.0;
+ 	else
+ 		*value=(double)((int32)m.i[0])/(double)m.i[1];
+From 3144e57770c1e4d26520d8abee750f8ac8b75490 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 16:09:02 +0000
+Subject: [PATCH] * libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement
+ various clampings of double to other data types to avoid undefined behaviour
+ if the output range isn't big enough to hold the input value. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2643
+ http://bugzilla.maptools.org/show_bug.cgi?id=2642
+ http://bugzilla.maptools.org/show_bug.cgi?id=2646
+ http://bugzilla.maptools.org/show_bug.cgi?id=2647
+
+---
+ ChangeLog              | 10 ++++++
+ libtiff/tif_dir.c      | 18 +++++++---
+ libtiff/tif_dirread.c  | 10 +++++-
+ libtiff/tif_dirwrite.c | 90 ++++++++++++++++++++++++++++++++++++++++++++------
+ 4 files changed, 113 insertions(+), 15 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 722a405..6517640 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,15 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
++	of double to other data types to avoid undefined behaviour if the output range
++	isn't big enough to hold the input value.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2643
++	http://bugzilla.maptools.org/show_bug.cgi?id=2642
++	http://bugzilla.maptools.org/show_bug.cgi?id=2646
++	http://bugzilla.maptools.org/show_bug.cgi?id=2647
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_dirread.c: avoid division by floating point 0 in
+ 	TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
+ 	and return 0 in that case (instead of infinity as before presumably)
+diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
+index 68a55af..a04d28f 100644
+--- a/libtiff/tif_dir.c
++++ b/libtiff/tif_dir.c
+@@ -31,6 +31,7 @@
+  * (and also some miscellaneous stuff)
+  */
+ #include "tiffiop.h"
++#include <float.h>
+ 
+ /*
+  * These are used in the backwards compatibility code...
+@@ -154,6 +155,15 @@ checkInkNamesString(TIFF* tif, uint32 slen, const char* s)
+ 	return (0);
+ }
+ 
++static float TIFFClampDoubleToFloat( double val )
++{
++    if( val > FLT_MAX )
++        return FLT_MAX;
++    if( val < -FLT_MAX )
++        return -FLT_MAX;
++    return (float)val;
++}
++
+ static int
+ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ {
+@@ -312,13 +322,13 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+         dblval = va_arg(ap, double);
+         if( dblval < 0 )
+             goto badvaluedouble;
+-		td->td_xresolution = (float) dblval;
++		td->td_xresolution = TIFFClampDoubleToFloat( dblval );
+ 		break;
+ 	case TIFFTAG_YRESOLUTION:
+         dblval = va_arg(ap, double);
+         if( dblval < 0 )
+             goto badvaluedouble;
+-		td->td_yresolution = (float) dblval;
++		td->td_yresolution = TIFFClampDoubleToFloat( dblval );
+ 		break;
+ 	case TIFFTAG_PLANARCONFIG:
+ 		v = (uint16) va_arg(ap, uint16_vap);
+@@ -327,10 +337,10 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 		td->td_planarconfig = (uint16) v;
+ 		break;
+ 	case TIFFTAG_XPOSITION:
+-		td->td_xposition = (float) va_arg(ap, double);
++		td->td_xposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
+ 		break;
+ 	case TIFFTAG_YPOSITION:
+-		td->td_yposition = (float) va_arg(ap, double);
++		td->td_yposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
+ 		break;
+ 	case TIFFTAG_RESOLUTIONUNIT:
+ 		v = (uint16) va_arg(ap, uint16_vap);
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 8a1e42a..77b0f37 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -40,6 +40,7 @@
+  */
+ 
+ #include "tiffiop.h"
++#include <float.h>
+ 
+ #define IGNORE 0          /* tag placeholder used below */
+ #define FAILED_FII    ((uint32) -1)
+@@ -2406,7 +2407,14 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryFloatArray(TIFF* tif, TIFFDirEnt
+ 				ma=(double*)origdata;
+ 				mb=data;
+ 				for (n=0; n<count; n++)
+-					*mb++=(float)(*ma++);
++                                {
++                                    double val = *ma++;
++                                    if( val > FLT_MAX )
++                                        val = FLT_MAX;
++                                    else if( val < -FLT_MAX )
++                                        val = -FLT_MAX;
++                                    *mb++=(float)val;
++                                }
+ 			}
+ 			break;
+ 	}
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index c9e871b..2967da5 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -30,6 +30,7 @@
+  * Directory Write Support Routines.
+  */
+ #include "tiffiop.h"
++#include <float.h>
+ 
+ #ifdef HAVE_IEEEFP
+ #define TIFFCvtNativeToIEEEFloat(tif, n, fp)
+@@ -939,6 +940,69 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
+ 	return(0);
+ }
+ 
++static float TIFFClampDoubleToFloat( double val )
++{
++    if( val > FLT_MAX )
++        return FLT_MAX;
++    if( val < -FLT_MAX )
++        return -FLT_MAX;
++    return (float)val;
++}
++
++static int8 TIFFClampDoubleToInt8( double val )
++{
++    if( val > 127 )
++        return 127;
++    if( val < -128 || val != val )
++        return -128;
++    return (int8)val;
++}
++
++static int16 TIFFClampDoubleToInt16( double val )
++{
++    if( val > 32767 )
++        return 32767;
++    if( val < -32768 || val != val )
++        return -32768;
++    return (int16)val;
++}
++
++static int32 TIFFClampDoubleToInt32( double val )
++{
++    if( val > 0x7FFFFFFF )
++        return 0x7FFFFFFF;
++    if( val < -0x7FFFFFFF-1 || val != val )
++        return -0x7FFFFFFF-1;
++    return (int32)val;
++}
++
++static uint8 TIFFClampDoubleToUInt8( double val )
++{
++    if( val < 0 )
++        return 0;
++    if( val > 255 || val != val )
++        return 255;
++    return (uint8)val;
++}
++
++static uint16 TIFFClampDoubleToUInt16( double val )
++{
++    if( val < 0 )
++        return 0;
++    if( val > 65535 || val != val )
++        return 65535;
++    return (uint16)val;
++}
++
++static uint32 TIFFClampDoubleToUInt32( double val )
++{
++    if( val < 0 )
++        return 0;
++    if( val > 0xFFFFFFFFU || val != val )
++        return 0xFFFFFFFFU;
++    return (uint32)val;
++}
++
+ static int
+ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, uint32 count, double* value)
+ {
+@@ -959,7 +1023,7 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
+ 			if (tif->tif_dir.td_bitspersample<=32)
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((float*)conv)[i] = (float)value[i];
++					((float*)conv)[i] = TIFFClampDoubleToFloat(value[i]);
+ 				ok = TIFFWriteDirectoryTagFloatArray(tif,ndir,dir,tag,count,(float*)conv);
+ 			}
+ 			else
+@@ -971,19 +1035,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
+ 			if (tif->tif_dir.td_bitspersample<=8)
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((int8*)conv)[i] = (int8)value[i];
++					((int8*)conv)[i] = TIFFClampDoubleToInt8(value[i]);
+ 				ok = TIFFWriteDirectoryTagSbyteArray(tif,ndir,dir,tag,count,(int8*)conv);
+ 			}
+ 			else if (tif->tif_dir.td_bitspersample<=16)
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((int16*)conv)[i] = (int16)value[i];
++					((int16*)conv)[i] = TIFFClampDoubleToInt16(value[i]);
+ 				ok = TIFFWriteDirectoryTagSshortArray(tif,ndir,dir,tag,count,(int16*)conv);
+ 			}
+ 			else
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((int32*)conv)[i] = (int32)value[i];
++					((int32*)conv)[i] = TIFFClampDoubleToInt32(value[i]);
+ 				ok = TIFFWriteDirectoryTagSlongArray(tif,ndir,dir,tag,count,(int32*)conv);
+ 			}
+ 			break;
+@@ -991,19 +1055,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
+ 			if (tif->tif_dir.td_bitspersample<=8)
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((uint8*)conv)[i] = (uint8)value[i];
++					((uint8*)conv)[i] = TIFFClampDoubleToUInt8(value[i]);
+ 				ok = TIFFWriteDirectoryTagByteArray(tif,ndir,dir,tag,count,(uint8*)conv);
+ 			}
+ 			else if (tif->tif_dir.td_bitspersample<=16)
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((uint16*)conv)[i] = (uint16)value[i];
++					((uint16*)conv)[i] = TIFFClampDoubleToUInt16(value[i]);
+ 				ok = TIFFWriteDirectoryTagShortArray(tif,ndir,dir,tag,count,(uint16*)conv);
+ 			}
+ 			else
+ 			{
+ 				for (i = 0; i < count; ++i)
+-					((uint32*)conv)[i] = (uint32)value[i];
++					((uint32*)conv)[i] = TIFFClampDoubleToUInt32(value[i]);
+ 				ok = TIFFWriteDirectoryTagLongArray(tif,ndir,dir,tag,count,(uint32*)conv);
+ 			}
+ 			break;
+@@ -2094,6 +2094,7 @@ TIFFWriteDirectoryTagCheckedSlong8Array(
+ static int
+ TIFFWriteDirectoryTagCheckedRational(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, double value)
+ {
++	static const char module[] = "TIFFWriteDirectoryTagCheckedRational";
+ 	uint32 m[2];
+ 	assert(value>=0.0);
+ 	assert(sizeof(uint32)==4);
+@@ -2102,7 +2102,12 @@ TIFFWriteDirectoryTagCheckedRational(TIF
+ 		m[0]=0;
+ 		m[1]=1;
+ 	}
+-	else if (value==(double)(uint32)value)
++	else if( value != value )
++	{
++		TIFFErrorExt(tif->tif_clientdata,module,"Not-a-number value is illegal");
++		return 0;
++	}
++	else if (value <= 0xFFFFFFFFU && value==(double)(uint32)value)
+ 	{
+ 		m[0]=(uint32)value;
+ 		m[1]=1;
+@@ -2143,12 +2212,13 @@ TIFFWriteDirectoryTagCheckedRationalArray(TIFF* tif, uint32* ndir, TIFFDirEntry*
+ 	}
+ 	for (na=value, nb=m, nc=0; nc<count; na++, nb+=2, nc++)
+ 	{
+-		if (*na<=0.0)
++		if (*na<=0.0 || *na != *na)
+ 		{
+ 			nb[0]=0;
+ 			nb[1]=1;
+ 		}
+-		else if (*na==(float)(uint32)(*na))
++		else if (*na >= 0 && *na <= (float)0xFFFFFFFFU &&
++                         *na==(float)(uint32)(*na))
+ 		{
+ 			nb[0]=(uint32)(*na);
+ 			nb[1]=1;
+From 0a76a8c765c7b8327c59646284fa78c3c27e5490 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 16:13:50 +0000
+Subject: [PATCH] * libtiff/tif_jpeg.c: validate BitsPerSample in
+ JPEGSetupEncode() to avoid undefined behaviour caused by invalid shift
+ exponent. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
+
+---
+ ChangeLog          | 6 ++++++
+ libtiff/tif_jpeg.c | 7 +++++++
+ 2 files changed, 13 insertions(+)
+
+diff --git a/ChangeLog b/ChangeLog
+index 6517640..8e202a2 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,11 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
++	undefined behaviour caused by invalid shift exponent.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
+ 	of double to other data types to avoid undefined behaviour if the output range
+ 	isn't big enough to hold the input value.
+diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
+index 6c17c38..192989a 100644
+--- a/libtiff/tif_jpeg.c
++++ b/libtiff/tif_jpeg.c
+@@ -1632,6 +1632,13 @@ JPEGSetupEncode(TIFF* tif)
+                             "Invalig horizontal/vertical sampling value");
+                     return (0);
+                 }
++                if( td->td_bitspersample > 16 )
++                {
++                    TIFFErrorExt(tif->tif_clientdata, module,
++                                 "BitsPerSample %d not allowed for JPEG",
++                                 td->td_bitspersample);
++                    return (0);
++                }
+ 
+ 		/*
+ 		 * A ReferenceBlackWhite field *must* be present since the
+From 66e7bd59520996740e4df5495a830b42fae48bc4 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 16:33:34 +0000
+Subject: [PATCH] * libtiff/tif_read.c: avoid potential undefined behaviour on
+ signed integer addition in TIFFReadRawStrip1() in isMapped() case. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2650
+
+---
+ ChangeLog          |  6 ++++++
+ libtiff/tif_read.c | 27 ++++++++++++++++++---------
+ 2 files changed, 24 insertions(+), 9 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 8e202a2..3e31464 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,11 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
++	addition in TIFFReadRawStrip1() in isMapped() case.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
+ 	undefined behaviour caused by invalid shift exponent.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
+diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
+index 52bbf50..b7aacbd 100644
+--- a/libtiff/tif_read.c
++++ b/libtiff/tif_read.c
+@@ -420,16 +420,25 @@ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
+ 			return ((tmsize_t)(-1));
+ 		}
+ 	} else {
+-		tmsize_t ma,mb;
++		tmsize_t ma;
+ 		tmsize_t n;
+-		ma=(tmsize_t)td->td_stripoffset[strip];
+-		mb=ma+size;
+-		if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||(ma>tif->tif_size))
+-			n=0;
+-		else if ((mb<ma)||(mb<size)||(mb>tif->tif_size))
+-			n=tif->tif_size-ma;
+-		else
+-			n=size;
++		if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||
++                    ((ma=(tmsize_t)td->td_stripoffset[strip])>tif->tif_size))
++                {
++                    n=0;
++                }
++                else if( ma > TIFF_TMSIZE_T_MAX - size )
++                {
++                    n=0;
++                }
++                else
++                {
++                    tmsize_t mb=ma+size;
++                    if (mb>tif->tif_size)
++                            n=tif->tif_size-ma;
++                    else
++                            n=size;
++                }
+ 		if (n!=size) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			TIFFErrorExt(tif->tif_clientdata, module,
+From 48780b4fcc425cddc4ef8ffdf536f96a0d1b313b Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 16:38:26 +0000
+Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fgetimage.c:=20add=20explicit?=
+ =?UTF-8?q?=20uint32=20cast=20in=20putagreytile=20to=20avoid=20UndefinedBe?=
+ =?UTF-8?q?haviorSanitizer=20warning.=20Patch=20by=20Nicol=C3=A1s=20Pe?=
+ =?UTF-8?q?=C3=B1a.=20Fixes=20http://bugzilla.maptools.org/show=5Fbug.cgi?=
+ =?UTF-8?q?=3Fid=3D2658?=
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+---
+ ChangeLog              | 7 +++++++
+ libtiff/tif_getimage.c | 2 +-
+ 2 files changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 3e31464..6a342e5 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,12 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
++	avoid UndefinedBehaviorSanitizer warning.
++	Patch by Nicolás Peña.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2658
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
+ 	addition in TIFFReadRawStrip1() in isMapped() case.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index fed31f1..2fa1775 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -1305,7 +1305,7 @@ DECLAREContigPutFunc(putagreytile)
+     while (h-- > 0) {
+ 	for (x = w; x-- > 0;)
+         {
+-            *cp++ = BWmap[*pp][0] & (*(pp+1) << 24 | ~A1);
++            *cp++ = BWmap[*pp][0] & ((uint32)*(pp+1) << 24 | ~A1);
+             pp += samplesperpixel;
+         }
+ 	cp += toskew;
+From d60332057b9575ada4f264489582b13e30137be1 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Wed, 11 Jan 2017 19:02:49 +0000
+Subject: [PATCH] * libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add
+ _TIFFcalloc()
+
+* libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
+initialize tif_rawdata.
+Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
+---
+ ChangeLog           | 8 ++++++++
+ libtiff/tif_read.c  | 4 +++-
+ libtiff/tif_unix.c  | 8 ++++++++
+ libtiff/tif_vms.c   | 8 ++++++++
+ libtiff/tif_win32.c | 8 ++++++++
+ libtiff/tiffio.h    | 1 +
+ 6 files changed, 36 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 6a342e5..abd75d7 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,13 @@
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
++
++	* libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
++	initialize tif_rawdata.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
++
++2017-01-11 Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
+ 	avoid UndefinedBehaviorSanitizer warning.
+ 	Patch by Nicolás Peña.
+diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
+index 277fdd6..4535ccb 100644
+--- a/libtiff/tif_read.c
++++ b/libtiff/tif_read.c
+@@ -985,7 +985,9 @@ TIFFReadBufferSetup(TIFF* tif, void* bp, tmsize_t size)
+ 				 "Invalid buffer size");
+ 		    return (0);
+ 		}
+-		tif->tif_rawdata = (uint8*) _TIFFmalloc(tif->tif_rawdatasize);
++		/* Initialize to zero to avoid uninitialized buffers in case of */
++                /* short reads (http://bugzilla.maptools.org/show_bug.cgi?id=2651) */
++		tif->tif_rawdata = (uint8*) _TIFFcalloc(1, tif->tif_rawdatasize);
+ 		tif->tif_flags |= TIFF_MYBUFFER;
+ 	}
+ 	if (tif->tif_rawdata == NULL) {
+diff --git a/libtiff/tif_unix.c b/libtiff/tif_unix.c
+index 7c7bc96..89dd32e 100644
+--- a/libtiff/tif_unix.c
++++ b/libtiff/tif_unix.c
+@@ -316,6 +316,14 @@ _TIFFmalloc(tmsize_t s)
+ 	return (malloc((size_t) s));
+ }
+ 
++void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
++{
++    if( nmemb == 0 || siz == 0 )
++        return ((void *) NULL);
++
++    return calloc((size_t) nmemb, (size_t)siz);
++}
++
+ void
+ _TIFFfree(void* p)
+ {
+diff --git a/libtiff/tif_win32.c b/libtiff/tif_win32.c
+index d730b3a..3e9001b 100644
+--- a/libtiff/tif_win32.c
++++ b/libtiff/tif_win32.c
+@@ -360,6 +360,14 @@ _TIFFmalloc(tmsize_t s)
+ 	return (malloc((size_t) s));
+ }
+ 
++void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
++{
++    if( nmemb == 0 || siz == 0 )
++        return ((void *) NULL);
++
++    return calloc((size_t) nmemb, (size_t)siz);
++}
++
+ void
+ _TIFFfree(void* p)
+ {
+diff --git a/libtiff/tiffio.h b/libtiff/tiffio.h
+index 732da17..fbd9171 100644
+--- a/libtiff/tiffio.h
++++ b/libtiff/tiffio.h
+@@ -293,6 +293,7 @@ extern TIFFCodec* TIFFGetConfiguredCODECs(void);
+  */
+ 
+ extern void* _TIFFmalloc(tmsize_t s);
++extern void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz);
+ extern void* _TIFFrealloc(void* p, tmsize_t s);
+ extern void _TIFFmemset(void* p, int v, tmsize_t c);
+ extern void _TIFFmemcpy(void* d, const void* s, tmsize_t c);
+From 8283e4d1b7e53340684d12932880cbcbaf23a8c1 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Thu, 12 Jan 2017 17:43:25 +0000
+Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fojpeg.c:=20fix=20leak=20in=20?=
+ =?UTF-8?q?OJPEGReadHeaderInfoSecTablesAcTable=20when=20read=20fails.=20Pa?=
+ =?UTF-8?q?tch=20by=20Nicol=C3=A1s=20Pe=C3=B1a.=20Fixes=20http://bugzilla.?=
+ =?UTF-8?q?maptools.org/show=5Fbug.cgi=3Fid=3D2659?=
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+---
+ ChangeLog           | 7 +++++++
+ libtiff/tif_ojpeg.c | 3 +++
+ 2 files changed, 10 insertions(+)
+
+diff --git a/ChangeLog b/ChangeLog
+index 6e6f3b0..12e0370 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,10 @@
++2017-01-12 Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
++	when read fails.
++	Patch by Nicolás Peña.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
++
+ 2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
+diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
+index f19e8fd..b92f0eb 100644
+--- a/libtiff/tif_ojpeg.c
++++ b/libtiff/tif_ojpeg.c
+@@ -1918,7 +1918,10 @@ OJPEGReadHeaderInfoSecTablesAcTable(TIFF* tif)
+ 				rb[sizeof(uint32)+5+n]=o[n];
+ 			p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
+ 			if (p!=q)
++                        {
++                                _TIFFfree(rb);
+ 				return(0);
++                        }
+ 			sp->actable[m]=rb;
+ 			sp->sos_tda[m]=(sp->sos_tda[m]|m);
+ 		}
+From 2ea32f7372b65c24b2816f11c04bf59b5090d05b Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Thu, 12 Jan 2017 19:23:20 +0000
+Subject: [PATCH] * libtiff/tif_ojpeg.c: fix leak in
+ OJPEGReadHeaderInfoSecTablesQTable, OJPEGReadHeaderInfoSecTablesDcTable and
+ OJPEGReadHeaderInfoSecTablesAcTable
+
+---
+ ChangeLog           | 3 ++-
+ libtiff/tif_ojpeg.c | 6 ++++++
+ 2 files changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 12e0370..cd2fa17 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,6 +1,7 @@
+ 2017-01-12 Even Rouault <even.rouault at spatialys.com>
+ 
+-	* libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
++	* libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesQTable,
++	OJPEGReadHeaderInfoSecTablesDcTable and OJPEGReadHeaderInfoSecTablesAcTable
+ 	when read fails.
+ 	Patch by Nicolás Peña.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
+diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
+index b92f0eb..5f6c684 100644
+--- a/libtiff/tif_ojpeg.c
++++ b/libtiff/tif_ojpeg.c
+@@ -1790,7 +1790,10 @@ OJPEGReadHeaderInfoSecTablesQTable(TIFF* tif)
+ 			TIFFSeekFile(tif,sp->qtable_offset[m],SEEK_SET); 
+ 			p=(uint32)TIFFReadFile(tif,&ob[sizeof(uint32)+5],64);
+ 			if (p!=64)
++                        {
++                                _TIFFfree(ob);
+ 				return(0);
++                        }
+ 			sp->qtable[m]=ob;
+ 			sp->sof_tq[m]=m;
+ 		}
+@@ -1854,7 +1857,10 @@ OJPEGReadHeaderInfoSecTablesDcTable(TIFF* tif)
+ 				rb[sizeof(uint32)+5+n]=o[n];
+ 			p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
+ 			if (p!=q)
++                        {
++                                _TIFFfree(rb);
+ 				return(0);
++                        }
+ 			sp->dctable[m]=rb;
+ 			sp->sos_tda[m]=(m<<4);
+ 		}

component/mariadb/buildout.cfg

@@ -3,6 +3,7 @@
 
 [buildout]
 extends =
+  ../bzip2/buildout.cfg
   ../cmake/buildout.cfg
   ../groonga/buildout.cfg
   ../jemalloc/buildout.cfg
@@ -22,11 +23,13 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-url = https://downloads.mariadb.com/MariaDB/mariadb-10.1.21/source/mariadb-10.1.21.tar.gz
-md5sum = ff8b96edeeb22425117309a249055310
+url = https://downloads.mariadb.com/MariaDB/mariadb-10.1.24/source/mariadb-10.1.24.tar.gz
+md5sum = e1af4c62e3345b65975f2fa43a0040d2
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
+  ${:stable-patches}
+stable-patches =
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-command = ${cmake:location}/bin/cmake
 configure-options =
@@ -43,7 +46,7 @@ configure-options =
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
   -DWITH_JEMALLOC=yes
-  -DWITH_INNODB_BZIP2=OFF
+  -DWITH_INNODB_BZIP2=ON
   -DWITH_INNODB_LZ4=OFF
   -DWITH_INNODB_LZMA=ON
   -DWITH_INNODB_LZO=OFF
@@ -52,15 +55,15 @@ configure-options =
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
   -DWITHOUT_MROONGA_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
-  -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-  -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
+  -DCMAKE_C_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
+  -DCMAKE_CXX_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
   -DCMAKE_INSTALL_RPATH=${:CMAKE_INSTALL_RPATH}
-CMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
+CMAKE_INSTALL_RPATH=${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
-  CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
+  CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
+  CMAKE_LIBRARY_PATH=${bzip2:location}/lib:${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
   PATH=${patch:location}/bin:%(PATH)s
 post-install =
   mkdir -p ${:location}/include/wsrep &&
@@ -70,11 +73,15 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-7.00.tar.gz
-md5sum = befcbb7de8f5dd27366ea513dc092f3f
-pre-configure =
-  mkdir fake_mariadb_source &&
+url = http://packages.groonga.org/source/mroonga/mroonga-7.03.tar.gz
+md5sum = 045be7c932e80cedaa756ea94ae24883
+pre-configure = set -e
+  rm -rf fake_mariadb_source
+  mkdir -p fake_mariadb_source
   ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
+  cp -ar ${mariadb:location}/include fake_mariadb_source/include
+  ! test -f fake_mariadb_source/include/mysql/mariadb_stmt.h ||
+  sed -i -E -e '/^enum enum_(cursor|indicator)_type$/,/};/d' fake_mariadb_source/include/mysql/mariadb_stmt.h
 configure-options =
   --with-mysql-source=fake_mariadb_source
   --with-mysql-config=${mariadb:location}/bin/mysql_config
@@ -83,6 +90,8 @@ configure-options =
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/mroonga_boolean.patch#29f85fc98e85d39b31e53b1e1683d288
+pre-build =
+  sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
 environment =
   PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include

component/mariadb/mariarocks.cfg

 [mariadb]
-url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.5/source/mariadb-10.2.5.tar.gz
-md5sum = 16fb01bb375c8d8a91a9ecf491397d51
-pre-configure =
-  sed -i -e '/(rc == -1 || errno == EINTR)/s/||/\&\&/' libmariadb/plugins/pvio/pvio_socket.c
-  sed -i -e 's,set(SYSTEM_LIBS $${CMAKE_THREAD_LIBS_INIT}),set(SYSTEM_LIBS $${CMAKE_THREAD_LIBS_INIT} $${LIBRT}),' storage/rocksdb/build_rocksdb.cmake
+url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.6/source/mariadb-10.2.6.tar.gz
+md5sum = 3d454cdadbd3208e1c9c23e5338a62b0
+stable-patches =
 configure-options +=
   -DPLUGIN_DAEMON_EXAMPLE=NO
   -DPLUGIN_EXAMPLE=NO

component/matplotlib/buildout.cfg

@@ -29,7 +29,10 @@ CFLAGS = -I${libpng:location}/include -I${freetype:location}/include
 recipe = zc.recipe.egg:custom
 egg = matplotlib
 environment = matplotlib-env
-setup-eggs = ${numpy:egg}
+setup-eggs =
+  cycler
+  python-dateutil
+  ${numpy:egg}
 rpath =
   ${libpng:location}/lib
   ${freetype:location}/lib

component/mysql-5.5/buildout.cfg

-[buildout]
-extends =
-  ../autoconf/buildout.cfg
-  ../automake/buildout.cfg
-  ../bison/buildout.cfg
-  ../cmake/buildout.cfg
-  ../groonga/buildout.cfg
-  ../libaio/buildout.cfg
-  ../libtool/buildout.cfg
-  ../ncurses/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../zlib/buildout.cfg
-
-parts =
-    mysql-5.5
-
-[mysql-5.5-sphinx-patch]
-# this patch comes from sphinx-2.0.1-beta including changes for
-# MySQL-5.5 in
-# http://code.google.com/p/sphinxsearch/source/detail?r=2921
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 04549822ebfa91b5f84025ff49ef24c2
-filename = mysql-5.5-sphinx-2.0.1-beta.diff
-download-only = true
-
-[mysql-5.5-no_test-patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/${:filename}
-md5sum = bb7ee34b9a98da1f913566c63ffbc6f5
-filename = mysql_create_system_tables__no_test.patch
-download-only = true
-
-[mysql-5.5]
-recipe = slapos.recipe.cmmi
-version = 5.5.16
-url = http://mysql.he.net/Downloads/MySQL-5.5/mysql-${:version}.tar.gz
-md5sum = 462ab3752dd666ec06ca32f92673b59e
-# compile directory is required to build mysql plugins.
-keep-compile-dir = true
-patch-options = -p0
-patches =
-  ${mysql-5.5-sphinx-patch:location}/${mysql-5.5-sphinx-patch:filename}
-  ${mysql-5.5-no_test-patch:location}/${mysql-5.5-no_test-patch:filename}
-configure-command = ${cmake:location}/bin/cmake
-# we use embeded yassl instead of openssl to avoid compilation errors on sphinx search engine.
-configure-options =
-  -DCMAKE_INSTALL_PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
-  -DBUILD_CONFIG=mysql_release
-  -DDEFAULT_CHARSET=utf8
-  -DDEFAULT_COLLATION=utf8_unicode_ci
-  -DWITH_SSL=bundled
-  -DWITH_ZLIB=system
-  -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-  -DWITHOUT_DAEMON_EXAMPLE=1
-  -DWITH_SPHINX_STORAGE_ENGINE=1
-  -DCMAKE_C_FLAGS="-I${libaio:location}/include -I${ncurses:location}/include -I${zlib:location}/include"
-  -DCMAKE_INSTALL_RPATH=${libaio:location}/lib:${ncurses:location}/lib:${zlib:location}/lib
-environment =
-  CMAKE_PROGRAM_PATH=${autoconf:location}/bin:${automake:location}/bin:${cmake:location}/bin:${libtool:location}/bin:${bison:location}/bin:${flex:location}/bin
-  CMAKE_INCLUDE_PATH=${libaio:location}/include:${ncurses:location}/include:${zlib:location}/include
-  CMAKE_LIBRARY_PATH=${libaio:location}/lib:${ncurses:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${libaio:location}/lib
-
-[groonga-storage-engine-mysql-5.5]
-recipe = slapos.recipe.cmmi
-url = http://github.com/downloads/mroonga/mroonga/groonga-storage-engine-1.0.0.tar.gz
-md5sum = 289b8b7919e790599ea79b6fe9270e04
-configure-options =
-  --with-mysql-source=${mysql-5.5:location}__compile__/mysql-${mysql-5.5:version}
-  --with-mysql-config=${mysql-5.5:location}/bin/mysql_config
-environment =
-  PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  CPPFLAGS=-I${groonga:location}/include/groonga
-  LDFLAGS=-L${groonga:location}/lib
-  PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig

component/mysql-5.5/mysql_create_system_tables__no_test.patch

-# 33_scripts__mysql_create_system_tables__no_test.dpatch by  <ch@debian.org>
-
-A user with no password prevents a normal user from login under certain
-circumstances as it is checked first.
-See http://bugs.debian.org/301741
-and http://bugs.mysql.com/bug.php?id=6901
-
---- scripts/mysql_system_tables_data.sql	2008-12-04 22:59:44.000000000 +0100
-+++ scripts/mysql_system_tables_data.sql	2008-12-04 23:00:07.000000000 +0100
-@@ -26,8 +26,6 @@
- -- Fill "db" table with default grants for anyone to
- -- access database 'test' and 'test_%' if "db" table didn't exist
- CREATE TEMPORARY TABLE tmp_db LIKE db;
--INSERT INTO tmp_db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
--INSERT INTO tmp_db VALUES ('%','test\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
- INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0;
- DROP TABLE tmp_db;
- 
-@@ -40,8 +38,6 @@
- REPLACE INTO tmp_user SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','' FROM dual WHERE LOWER( @current_hostname) != 'localhost';
- REPLACE INTO tmp_user VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','');
- REPLACE INTO tmp_user VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','');
--INSERT INTO tmp_user (host,user) VALUES ('localhost','');
--INSERT INTO tmp_user (host,user) SELECT @current_hostname,'' FROM dual WHERE LOWER(@current_hostname ) != 'localhost';
- INSERT INTO user SELECT * FROM tmp_user WHERE @had_user_table=0;
- DROP TABLE tmp_user;
- 

component/mysql-python/buildout.cfg

-[buildout]
-extends =
-  ../mariadb/buildout.cfg
-  ../openssl/buildout.cfg
-  ../zlib/buildout.cfg
-
-parts =
-  mysql-python
-
-[mysql-python-env]
-PATH =${mariadb:location}/bin:%(PATH)s
-
-[mysql-python]
-recipe = zc.recipe.egg:custom
-egg = MySQL-python
-environment = mysql-python-env
-library-dirs =
-  ${zlib:location}/lib/
-  ${openssl:location}/lib/
-rpath =
-  ${mariadb:location}/lib/
-  ${zlib:location}/lib/
-  ${openssl:location}/lib/

component/nano/buildout.cfg

 [buildout]
 parts =
   nano
-  
+
 extends = 
+  ../file/buildout.cfg
   ../ncurses/buildout.cfg
 
 [nano]
 recipe = slapos.recipe.cmmi
-version = 2.2.6
-url = http://www.nano-editor.org/dist/v2.2/nano-2.2.6.tar.gz
-md5sum = 03233ae480689a008eb98feb1b599807
+url = http://www.nano-editor.org/dist/v2.8/nano-2.8.4.tar.xz
+md5sum = 02ff28870194178595b287fc16fa611b
+# The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
+# are installed on the system.
 environment=
-    CFLAGS=-I${ncurses:location}/include
-    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
+  PKG_CONFIG=false
+  NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
\ No newline at end of file
+  CPPFLAGS=-I${file:location}/include
+  LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+post-install =
+  cd ${:location} && mkdir etc &&
+  echo include "${:location}/share/nano/*.nanorc" > etc/nanorc

component/nettle/buildout.cfg

@@ -6,10 +6,10 @@ extends =
 
 [nettle]
 recipe = slapos.recipe.cmmi
-url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
-md5sum = 003d5147911317931dd453520eb234a5
+url = http://ftp.gnu.org/gnu/nettle/nettle-3.3.tar.gz
+md5sum = 10f969f78a463704ae73529978148dbe
 patches =
-  ${:_profile_base_location_}/nettle-lib-location.patch#41dd0ce2a73487929bdc637b75dd62c9
+  ${:_profile_base_location_}/nettle-lib-location.patch#3c5f5b285ffd5bc30436ee0f4c662084
 configure-option =
   --disable-static
   --disable-assembler

component/nettle/nettle-lib-location.patch

---- configure.orig	2013-07-05 15:37:28.000000000 +0200
-+++ configure	2013-07-05 15:47:48.000000000 +0200
-@@ -4680,52 +4680,6 @@
+--- configure.orig	2016-10-01 09:28:38.000000000 +0200
++++ configure	2017-06-22 11:22:13.865295091 +0200
+@@ -6779,58 +6779,6 @@
  if test "x$ABI" != xstandard ; then
    { $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
  $as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
@@ -43,6 +43,12 @@
 -      *:freebsd*:64)
 -	libdir='${exec_prefix}/lib'
 -	;;
+-      *:irix*:32)
+-	libdir='${exec_prefix}/lib32'
+-	;;
+-      *:irix*:64)
+-	libdir='${exec_prefix}/lib64'
+-	;;
 -      *)
 -        { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
 -$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
@@ -52,4 +58,4 @@
 -  fi
  fi
  
- # Select assembler code
+ OPT_NETTLE_SOURCES=""

component/nghttp2/buildout.cfg

@@ -3,6 +3,8 @@ extends =
   ../automake/buildout.cfg
   ../libtool/buildout.cfg
   ../pkgconfig/buildout.cfg
+  ../python-2.7/buildout.cfg
+  ../python-setuptools/buildout.cfg
 
 parts =
   nghttp2
@@ -18,4 +20,4 @@ pre-configure =
   automake
   autoconf
 environment =
-  PATH=${autoconf:location}/bin:${automake:location}/bin:${git:location}/bin:${libtool:location}/bin:${m4:location}/bin:%(PATH)s
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${git:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python2.7:location}/bin:%(PATH)s

component/ninja/buildout.cfg

+[buildout]
+parts =
+  ninja
+
+[ninja]
+recipe = slapos.recipe.cmmi
+version = v1.7.1
+url = https://github.com/ninja-build/ninja/archive/${:version}.tar.gz
+configure-command = ./configure.py
+configure-options = --bootstrap
+make-binary = true
+post-install = ls && cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_}/

component/ocropy/buildout.cfg

+[buildout]
+extends =
+  ../git/buildout.cfg
+  ../gzip/buildout.cfg
+  ../scipy/buildout.cfg
+  ../lxml-python/buildout.cfg
+  ../matplotlib/buildout.cfg
+  ../numpy/buildout.cfg
+  ../patch/buildout.cfg
+  ../pillow/buildout.cfg
+  ../numpy/buildout.cfg
+
+parts = ocropy
+
+[ocropy-eng-traineddata]
+recipe = hexagonit.recipe.download
+filename = en-default.pyrnn.gz
+md5sum = cedd140c7d7650e910f0550ad0f04727
+download-only = true
+url = http://www.tmbdev.net/en-default.pyrnn.gz
+
+[ocropy-env]
+OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:location}/${ocropy-eng-traineddata:filename}
+HOME = ${ocropy:egg}
+
+[ocropy]
+recipe = zc.recipe.egg:custom
+egg = ocropy
+setup-eggs =
+  ${numpy:egg}
+  ${scipy:egg}
+  ${matplotlib:egg}
+  ${pillow-python:egg}
+patches =
+  ${:_profile_base_location_}/ocropy.patch
+patch-options = -p0
+patch-binary = ${patch:location}/bin/patch
+environment = ocropy-env
+find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
+md5sum = 240b8866dd7248816e01af469a328c09

component/ocropy/ocropy.patch

+diff --git ocrolib/__init__.py ocrolib/__init__.py
+index 1e0d627..81e85fb 100644
+--- ocrolib/__init__.py
++++ ocrolib/__init__.py
+@@ -1,7 +1,7 @@
+ __all__ = [
+     "binnednn","cairoextras","common","components","dbtables",
+     "fgen","gmmtree","gtkyield","hocr","lang","native",
+-    "mlp","multiclass","default","lineest"
++    "mlp","multiclass","default","lineest", "psegutils"
+ ]
+ 
+ ################################################################
+@@ -9,5 +9,6 @@ __all__ = [
+ ################################################################
+ 
+ import default
++from psegutils import *
+ from common import *
+ from default import traceback as trace
+diff --git ocrolib/common.py ocrolib/common.py
+index 27c0f26..14f088f 100644
+--- ocrolib/common.py
++++ ocrolib/common.py
+@@ -14,6 +14,7 @@ import unicodedata
+ import inspect
+ import glob
+ import cPickle
++import gzip
+ from ocrolib.exceptions import (BadClassLabel, BadInput, FileNotFound,
+                                 OcropusException)
+ 
+@@ -428,6 +429,7 @@ def unpickle_find_global(mname,cname):
+         exec "import "+mname
+     return getattr(sys.modules[mname],cname)
+ 
++
+ def load_object(fname,zip=0,nofind=0,verbose=0):
+     """Loads an object from disk. By default, this handles zipped files
+     and searches in the usual places for OCRopus. It also handles some
+@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
+     if zip==0 and fname.endswith(".gz"):
+         zip = 1
+     if zip>0:
+-        # with gzip.GzipFile(fname,"rb") as stream:
+-        with os.popen("gunzip < '%s'"%fname,"rb") as stream:
++        with gzip.GzipFile(fname,"rb") as stream:
+             unpickler = cPickle.Unpickler(stream)
+             unpickler.find_global = unpickle_find_global
+             return unpickler.load()
+@@ -618,7 +619,7 @@ def ocropus_find_file(fname, gz=True):
+ 
+     possible_prefixes.append(os.path.normpath(os.path.join(
+         os.path.dirname(inspect.getfile(inspect.currentframe())),
+-        os.pardir, os.pardir, os.pardir, os.pardir, "share", "ocropus")))
++        os.pardir, "share", "ocropus")))
+ 
+     possible_prefixes.append("/usr/local/share/ocropus")
+ 
+diff --git ocrolib/native.py ocrolib/native.py
+index b7a207f..240450b 100644
+--- ocrolib/native.py
++++ ocrolib/native.py
+@@ -44,6 +44,7 @@ class CompileError(Exception):
+ 
+ def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
+                      options="-shared -fopenmp -std=c99 -fPIC",verbose=0):
++    prefix = os.path.join(os.path.dirname(__file__), prefix)
+     if not os.path.exists(prefix):
+         os.mkdir(prefix)
+     m = hashlib.md5()
+diff --git setup.py setup.py
+index 2ec5832..6697b12 100644
+--- setup.py
++++ setup.py
+@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
+ from distutils.core import setup #, Extension, Command
+ #from distutils.command.install_data import install_data
+ 
+-if not os.path.exists("models/en-default.pyrnn.gz"):
++models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
++    [c for c in glob.glob("models/*pyrnn.gz")]
++if not models:
+     print()
+     print("You should download the default model 'en-default.pyrnn.gz'")
+     print("and put it into ./models.")
+@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
+     print("Check https://github.com/tmbdev/ocropy for the location")
+     print("of model files.")
+     print()
++    sys.exit(1)
+ 
+-models = [c for c in glob.glob("models/*pyrnn.gz")]
+ scripts = [c for c in glob.glob("ocropus-*") if "." not in c and "~" not in c]
+ 
++# compile pynative files now and include them in the build
++sys.path.insert(0, os.curdir)
++import ocrolib.nutils
++pynative_files = [os.path.join(*c.split(os.path.sep)[1:]) \
++    for c in glob.glob('ocrolib/.pynative/*')]
++
+ setup(
+     name = 'ocropy',
+     version = 'v1.0',
+     author = "Thomas Breuel",
+     description = "The OCRopy RNN-based Text Line Recognizer",
+     packages = ["ocrolib"],
+-    data_files= [('share/ocropus', models)],
++    package_data = {'ocrolib': pynative_files},
++    data_files= [('share/ocropus', models), ("", ["LICENSE"])],
+     scripts = scripts,
+     )

component/p11-kit/buildout.cfg

@@ -9,8 +9,9 @@ extends =
 
 [p11-kit]
 recipe = slapos.recipe.cmmi
-url = http://p11-glue.freedesktop.org/releases/p11-kit-0.23.1.tar.gz
-md5sum = 96f073270c489c9a594e1c9413f42db8
+url = https://github.com/p11-glue/p11-kit/releases/download/${:version}/p11-kit-${:version}.tar.gz
+version = 0.23.7
+md5sum = ebbefd123210594231adb4bde21b8560
 configure-options =
   --disable-static
   --disable-doc-html

component/perl-CGI/buildout.cfg

+[buildout]
+extends =
+  ../perl/buildout.cfg
+parts =
+  perl-CGI
+
+[perl-CGI]
+recipe = slapos.recipe.cmmi
+url = http://www.cpan.org/modules/by-module/CGI/CGI-4.36.tar.gz
+md5sum = 48566d81f430a7f755bb4d8f426ce35d
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL

component/perl-DBD-MySQL/DBD-mysql-4.042.mariadb.patch

+diff -u DBD-mysql-4.042/dbdimp.c DBD-mysql-4.042/dbdimp.c
+--- DBD-mysql-4.042/dbdimp.c	2017-02-28 14:36:40.000000000 +0100
++++ DBD-mysql-4.042/dbdimp.c	2017-06-08 12:43:39.704228470 +0200
+@@ -2139,6 +2139,7 @@
+ 
+     if (result)
+     {
++      my_bool reconnect= 0;
+ #if MYSQL_VERSION_ID >=SERVER_PREPARE_VERSION
+       /* connection succeeded. */
+       /* imp_dbh == NULL when mysql_dr_connect() is called from mysql.xs
+@@ -2155,7 +2156,7 @@
+         we turn off Mysql's auto reconnect and handle re-connecting ourselves
+         so that we can keep track of when this happens.
+       */
+-      result->reconnect=0;
++      mysql_options(result, MYSQL_OPT_RECONNECT, &reconnect);
+     }
+     else {
+       /* 
+diff -u DBD-mysql-4.042/dbdimp.h DBD-mysql-4.042/dbdimp.h
+--- DBD-mysql-4.042/dbdimp.h	2017-02-28 14:36:40.000000000 +0100
++++ DBD-mysql-4.042/dbdimp.h	2017-06-08 12:44:21.233948264 +0200
+@@ -67,6 +67,12 @@
+ #define SSL_LAST_VERIFY_VERSION 50799
+ #define MYSQL_VERSION_5_0 50001
+ /* This is to avoid the ugly #ifdef mess in dbdimp.c */
++
++/* MariaDB Connector/C has MARIADB_VERSION_ID instead of MYSQL_VERSION_ID */
++#if !defined(MYSQL_VERSION_ID) && defined(MARIADB_VERSION_ID)
++#define MYSQL_VERSION_ID MARIADB_VERSION_ID
++#endif
++
+ #if MYSQL_VERSION_ID < SQL_STATE_VERSION
+ #define mysql_sqlstate(svsock) (NULL)
+ #endif

component/perl-DBD-MySQL/buildout.cfg

@@ -3,6 +3,7 @@ extends =
   ../mysql-tritonn-5.0/buildout.cfg
   ../mariadb/buildout.cfg
   ../patch/buildout.cfg
+  ../perl-Devel-CheckLib/buildout.cfg
 
 parts =
   perl-DBD-MySQL
@@ -11,24 +12,24 @@ parts =
 recipe = slapos.recipe.cmmi
 depends =
   ${perl-DBI:location}
-url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.027.tar.gz
-md5sum = 11fa6449715ba9d3fa5f4ee5ae688d6f
+  ${perl-Devel-CheckLib:location}
+url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.042.tar.gz
+md5sum = a144bd950b55af68835d44bc4ea6e5aa
 patches =
   ${:_profile_base_location_}/DBD-mysql-4.027.rpathsupport.patch#a932982b7725e6621cfce3a3d7917e03
+  ${:_profile_base_location_}/DBD-mysql-4.042.mariadb.patch#5864d36d19c4a05034b3a4873f7c659a
 patch-options = -p1
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL --libs="-L${zlib:location}/lib -L${openssl:location}/lib $(mysql_config --libs)"
 
 [perl-DBD-MySQL]
 <= perl-DBD-MySQL-common
-configure-command =
-  ${perl:location}/bin/perl Makefile.PL --mysql_config=${mysql-tritonn-5.0:location}/bin/mysql_config
 environment =
   OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mysql-tritonn-5.0:location}/lib/mysql -Wl,-rpath=${openssl:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${mysql-tritonn-5.0:location}/bin:${patch:location}/bin:%(PATH)s
 
 [perl-DBD-mariadb]
 <= perl-DBD-MySQL-common
-configure-command =
-  ${perl:location}/bin/perl Makefile.PL --mysql_config=${mariadb:location}/bin/mysql_config
 environment =
-  OTHERLDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mariadb:location}/lib -Wl,-rpath=${openssl:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mariadb:location}/lib -Wl,-rpath=${openssl:location}/lib
+  PATH=${mariadb:location}/bin:${patch:location}/bin:%(PATH)s

component/perl-Devel-CheckLib/buildout.cfg

+[buildout]
+extends =
+  ../perl/buildout.cfg
+
+[perl-Devel-CheckLib]
+recipe = slapos.recipe.cmmi
+url = http://www.cpan.org/modules/by-module/Devel/Devel-CheckLib-1.11.tar.gz
+md5sum = c20873798591176283f51af4ad8088cd
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL

component/perl-Encode-HanExtra/buildout.cfg

@@ -10,7 +10,7 @@ recipe = slapos.recipe.cmmi
 url = http://www.cpan.org/modules/by-module/Encode/Encode-HanExtra-0.23.tar.gz
 md5sum = e1d3bc32c1c8ee304235a06fbcd5d5a4
 configure-command =
-  ${perl:location}/bin/perl Makefile.PL
+  ${perl:location}/bin/perl -I. Makefile.PL
 patch-options = -p1
 patches =
   ${:_profile_base_location_}/Encode-HanExtra-0.23_use_shebang.patch#4ac804013a68a3e6b7c3bc4115d25a5e

component/perl-URI/buildout.cfg

@@ -6,7 +6,7 @@ parts =
 
 [perl-URI]
 recipe = slapos.recipe.cmmi
-url = http://www.cpan.org/modules/by-module/URI/URI-1.60.tar.gz
-md5sum = 70f739be8ce28b8baba7c5920ffee4dc
+url = http://www.cpan.org/modules/by-module/URI/URI-1.71.tar.gz
+md5sum = 247c3da29a794f72730e01aa5a715daf
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/perl-XML-NamespaceSupport/buildout.cfg

@@ -9,4 +9,4 @@ recipe = slapos.recipe.cmmi
 url = http://www.cpan.org/modules/by-module/XML/XML-NamespaceSupport-1.11.tar.gz
 md5sum = 222cca76161cd956d724286d36b607da
 configure-command =
-  ${perl:location}/bin/perl Makefile.PL
+  ${perl:location}/bin/perl -I. Makefile.PL

component/perl/buildout.cfg

@@ -7,9 +7,9 @@ parts =
 
 [perl]
 recipe = slapos.recipe.cmmi
-version = 5.20.2
+version = 5.26.0
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
-md5sum = 21062666f1c627aeb6dbff3c6952738b
+md5sum = 8d34cb5a4eccd66e7a6a80e62b7b4aec
 siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =
@@ -25,4 +25,4 @@ configure-command =
     -Dusethreads
 environment =
   PATH=${patch:location}/bin:%(PATH)s
-post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#28d12de5c9c7985e65233d56817e3bec:post_make_hook
+post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#943453b7d3ff8d49ed12d44a7f7076ee:post_make_hook

component/perl/perl-create-libs-symlink.py

-
+from zc.buildout import UserError
 import fnmatch
 import os
-import pprint
-
 
 def post_make_hook(options, buildout):
   location = options['location']
-  matches = []
-  for root, dirnames, filenames in os.walk(location):
-    for filename in fnmatch.filter(filenames, 'libperl.a'):
-      matches.append(os.path.join(root, filename))
-  nr_matches = len(matches)
-  if nr_matches == 0:
-    print "ERROR - no libperl.* found!"
-    exit
-  elif nr_matches > 1:
-    print "WARNING - several libperl.a found, taking only the first one:", "\n".join(matches)
-  
+  matches = [os.path.join(root, filename)
+    for root, dirnames, filenames in os.walk(location)
+    for filename in fnmatch.filter(filenames, 'libperl.a')]
+  if not matches:
+    raise UserError("ERROR - no libperl.* found!")
+  if len(matches) > 1:
+    print("WARNING - several libperl.a found, taking only the first one:",
+          "\n".join(matches))
+
   # matches[0] is a prefix of "location"
   # For the symlink, we want the relative path.
   rel_link = os.path.relpath(os.path.dirname(matches[0]), location)
@@ -24,4 +20,4 @@ def post_make_hook(options, buildout):
   if os.path.islink(simlink_location):
     os.unlink(simlink_location)
   os.symlink(rel_link, simlink_location)
-  print "Created symlink \"libs-c\" to", rel_link
+  print("Created symlink \"libs-c\" to", rel_link)

component/powerdns/buildout.cfg

@@ -23,12 +23,13 @@ url = http://downloads.powerdns.com/releases/pdns-3.3.1.tar.gz
 md5sum = 074e2ff211fd12ecad25b5c1cc190dd4
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-boost=${boost-lib:location}
   --with-modules="geo"
   --with-dynmodules=""
   --without-lua
 environment =
-  PATH=${make:location}/bin:${libtool:location}/bin:${pkgconfig:location}/bin:${bison:location}/bin:${flex:location}/bin:${git:location}/bin:${ragel:location}/bin:%(PATH)s
-  LDFLAGS = -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lz
+  PATH=${gcc:location}/bin:${make:location}/bin:${libtool:location}/bin:${pkgconfig:location}/bin:${bison:location}/bin:${flex:location}/bin:${git:location}/bin:${ragel:location}/bin:%(PATH)s
+  LDFLAGS=-L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lz
   CPPFLAGS=-I${boost-lib:location}/include
 make-target =
   install

component/python-2.4/buildout.cfg

-[buildout]
-extends =
-  ../bzip2/buildout.cfg
-  ../gdbm/buildout.cfg
-  ../gettext/buildout.cfg
-  ../ncurses/buildout.cfg
-  ../openssl/buildout.cfg
-  ../readline/buildout.cfg
-  ../sqlite3/buildout.cfg
-  ../zlib/buildout.cfg
-
-parts =
-    pythonbin2.4
-
-[python2.4-dbm-patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-# original patch from http://bugs.gentoo.org/attachment.cgi?id=109117
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 8611020af1463b42f253ac73a91b09a1
-download-only = true
-filename = python2.4-dbm.patch
-
-[python2.4-no_system_inc_dirs-patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 65c1193ac42c15109be0e6f9b7a671b8
-download-only = true
-filename = python2.4-no_system_inc_dirs.patch
-
-[python2.4-CPPFLAGS-patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 912b6d4b107e3f382995a4d3afcd3eca
-download-only = true
-filename = python2.4-backport-CPPFLAGS-setup-from-python2.6.patch
-
-[python2.4]
-recipe = slapos.recipe.cmmi
-# This is actually the default setting for prefix, but we can't use it in
-# other settings in this part if we don't set it explicitly here.
-prefix = ${buildout:parts-directory}/${:_buildout_section_name_}
-version = 2.4
-package_version = ${:version}.6
-executable = ${:prefix}/bin/python${:version}
-
-python_version_major = 2.4
-python_version_minor = 6
-python_version_full = ${:python_version_major}.${:python_version_minor}
-url =
-  http://python.org/ftp/python/${:python_version_full}/Python-${:python_version_full}.tgz
-patches =
-	${python2.4-dbm-patch:location}/${python2.4-dbm-patch:filename}
-        ${python2.4-no_system_inc_dirs-patch:location}/${python2.4-no_system_inc_dirs-patch:filename}
-        ${python2.4-CPPFLAGS-patch:location}/${python2.4-CPPFLAGS-patch:filename}
-
-configure-options =
-  --enable-unicode=ucs4
-  --with-threads
-
-environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include/ -I${ncurses:location}/include/ncursesw/ -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
-
-[pythonbin2.4]
-# XXX/Note: This is hackish way to have fully featured python interpreter
-recipe = zc.recipe.egg:scripts
-python = python2.4
-eggs = ${eggs:eggs}
-  invokepython>=0.4
-extra-paths =
-  ${zope-2.8:location}/lib/python
-scripts =
-  invokepython=python${python2.4:python_version_major}
-  ipython=ipython${python2.4:python_version_major}
-
-[bootstrap2.4]
-recipe = zc.recipe.egg
-eggs = zc.buildout
-suffix =
-scripts =
-    buildout=bootstrap2.4
-arguments = sys.argv[1:] + ["bootstrap"]
-suffix = 2.4
-python = python2.4

component/python-2.4/python2.4-backport-CPPFLAGS-setup-from-python2.6.patch

-From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
-Date: Mon, 7 Mar 2011 13:02:05 +0900
-Subject: [PATCH] Pass CPPFLAGS when building modules in Python 2.4
-
-In contrary to Python 2.6, when building Python 2.4 modules, CPPFLAGS 
-specified to configure script is not passed at all. This patch backports
-the fix from Python 2.6.
-
---- Makefile.pre.in	2006-10-09 02:41:25.000000000 +0900
-+++ Makefile.pre.in	2011-03-07 14:58:34.368000777 +0900
-@@ -56,7 +56,10 @@
- OPT=		@OPT@
- BASECFLAGS=	@BASECFLAGS@
- CFLAGS=		$(BASECFLAGS) $(OPT)
--CPPFLAGS=	-I. -I$(srcdir)/Include
-+# Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to
-+# be able to build extension modules using the directories specified in the
-+# environment variables
-+CPPFLAGS=	-I. -I$(srcdir)/Include @CPPFLAGS@
- LDFLAGS=	@LDFLAGS@
- LDLAST=		@LDLAST@
- SGI_ABI=	@SGI_ABI@
---- setup.py	2006-10-09 02:41:25.000000000 +0900
-+++ setup.py	2011-03-07 14:53:36.208000779 +0900
-@@ -3,7 +3,7 @@
- 
- __version__ = "$Revision: 52231 $"
- 
--import sys, os, getopt, imp, re
-+import sys, os, getopt, imp, re, optparse
- 
- from distutils import log
- from distutils import sysconfig
-@@ -243,6 +243,39 @@
-         add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib')
-         add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
- 
-+        # Add paths specified in the environment variables LDFLAGS and
-+        # CPPFLAGS for header and library files.
-+        # We must get the values from the Makefile and not the environment
-+        # directly since an inconsistently reproducible issue comes up where
-+        # the environment variable is not set even though the value were passed
-+        # into configure and stored in the Makefile (issue found on OS X 10.3).
-+        for env_var, arg_name, dir_list in (
-+                ('LDFLAGS', '-L', self.compiler.library_dirs),
-+                ('CPPFLAGS', '-I', self.compiler.include_dirs)):
-+            env_val = sysconfig.get_config_var(env_var)
-+            if env_val:
-+                # To prevent optparse from raising an exception about any
-+                # options in env_val that is doesn't know about we strip out
-+                # all double dashes and any dashes followed by a character
-+                # that is not for the option we are dealing with.
-+                #
-+                # Please note that order of the regex is important!  We must
-+                # strip out double-dashes first so that we don't end up with
-+                # substituting "--Long" to "-Long" and thus lead to "ong" being
-+                # used for a library directory.
-+                env_val = re.sub(r'(^|\s+)-(-|(?!%s))' % arg_name[1],
-+                                 ' ', env_val)
-+                parser = optparse.OptionParser()
-+                # Make sure that allowing args interspersed with options is
-+                # allowed
-+                parser.allow_interspersed_args = True
-+                parser.error = lambda msg: None
-+                parser.add_option(arg_name, dest="dirs", action="append")
-+                options = parser.parse_args(env_val.split())[0]
-+                if options.dirs:
-+                    for directory in reversed(options.dirs):
-+                        add_dir_to_list(dir_list, directory)
-+
-         # Add paths to popular package managers on OS X/darwin
-         if sys.platform == "darwin":
-             # Fink installs into /sw by default
---- Lib/optparse.py	2006-05-29 03:15:43.000000000 +0900
-+++ Lib/optparse.py	2011-03-07 21:20:17.192000789 +0900
-@@ -69,7 +69,13 @@
- import sys, os
- import types
- import textwrap
--from gettext import gettext as _
-+
-+try:
-+    from gettext import gettext
-+except ImportError:
-+    def gettext(message):
-+        return message
-+_ = gettext
- 
- def _repr(self):
-     return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self)
-

component/python-2.4/python2.4-no_system_inc_dirs.patch

-From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
-Date: Mon, 7 Mar 2011 14:10:31 +0900
-Subject: [PATCH] Do not use system include directory
-
-Backport python-2.6.6-no_system_inc_dirs.patch to Python 2.4.
-
---- setup.py	2006-10-09 02:41:25.000000000 +0900
-+++ setup.py	2011-03-07 15:15:00.724000778 +0900
-@@ -270,7 +270,7 @@
-             '/lib64', '/usr/lib64',
-             '/lib', '/usr/lib',
-             ]
--        inc_dirs = self.compiler.include_dirs + ['/usr/include']
-+        inc_dirs = self.compiler.include_dirs
-         exts = []
- 
-         platform = self.get_platform()
-@@ -528,31 +528,7 @@
- 
-         # construct a list of paths to look for the header file in on
-         # top of the normal inc_dirs.
--        db_inc_paths = [
--            '/usr/include/db4',
--            '/usr/local/include/db4',
--            '/opt/sfw/include/db4',
--            '/sw/include/db4',
--            '/usr/include/db3',
--            '/usr/local/include/db3',
--            '/opt/sfw/include/db3',
--            '/sw/include/db3',
--        ]
--        # 4.x minor number specific paths
--        for x in (0,1,2,3,4):
--            db_inc_paths.append('/usr/include/db4%d' % x)
--            db_inc_paths.append('/usr/include/db4.%d' % x)
--            db_inc_paths.append('/usr/local/BerkeleyDB.4.%d/include' % x)
--            db_inc_paths.append('/usr/local/include/db4%d' % x)
--            db_inc_paths.append('/pkg/db-4.%d/include' % x)
--            db_inc_paths.append('/opt/db-4.%d/include' % x)
--        # 3.x minor number specific paths
--        for x in (2,3):
--            db_inc_paths.append('/usr/include/db3%d' % x)
--            db_inc_paths.append('/usr/local/BerkeleyDB.3.%d/include' % x)
--            db_inc_paths.append('/usr/local/include/db3%d' % x)
--            db_inc_paths.append('/pkg/db-3.%d/include' % x)
--            db_inc_paths.append('/opt/db-3.%d/include' % x)
-+        db_inc_paths = []
- 
-         # Add some common subdirectories for Sleepycat DB to the list,
-         # based on the standard include directories. This way DB3/4 gets

component/python-2.6/python-2.6.6-no_system_inc_dirs.patch

---- Python-2.6.6/setup.py~	2010-12-11 12:05:13.000000000 +0100
-+++ Python-2.6.6/setup.py	2011-03-05 13:33:04.556662701 +0100
-@@ -411,7 +411,7 @@
-             '/lib64', '/usr/lib64',
-             '/lib', '/usr/lib',
-             ]
--        inc_dirs = self.compiler.include_dirs + ['/usr/include']
-+        inc_dirs = self.compiler.include_dirs
-         exts = []
-         missing = []
- 
-@@ -818,15 +818,6 @@
-         # construct a list of paths to look for the header file in on
-         # top of the normal inc_dirs.
-         db_inc_paths = [
--            '/usr/include/db4',
--            '/usr/local/include/db4',
--            '/opt/sfw/include/db4',
--            '/usr/include/db3',
--            '/usr/local/include/db3',
--            '/opt/sfw/include/db3',
--            # Fink defaults (http://fink.sourceforge.net/)
--            '/sw/include/db4',
--            '/sw/include/db3',
-         ]
-         # 4.x minor number specific paths
-         for x in gen_db_minor_ver_nums(4):

component/python-2.7/buildout.cfg

@@ -56,9 +56,10 @@ configure-options =
 make-binary =
 make-targets = make profile-opt && make install
 
+extra-ldflags =
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes
 environment =
   PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib ${:extra-ldflags}

component/python-mysqlclient/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../mariadb/buildout.cfg
   ../openssl/buildout.cfg
+  ../patch/buildout.cfg
   ../zlib/buildout.cfg
 
 parts =
@@ -13,6 +14,10 @@ PATH =${mariadb:location}/bin:%(PATH)s
 [python-mysqlclient]
 recipe = zc.recipe.egg:custom
 egg = mysqlclient
+patches =
+  https://github.com/PyMySQL/mysqlclient-python/commit/1693848c9f6ca863868d94d63499830f7f4f3a1f.diff#a493a91f9263243eb331fcab9901b8b0
+patch-options = -p1
+patch-binary = ${patch:location}/bin/patch
 environment = python-mysqlclient-env
 library-dirs =
   ${zlib:location}/lib/

component/python-slip/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 
 [python-slip]
 recipe = slapos.recipe.cmmi
-url = https://fedorahosted.org/released/python-slip/python-slip-0.6.1.tar.bz2
+url = https://github.com/nphilipp/python-slip/releases/download/python-slip-0.6.1/python-slip-0.6.1.tar.bz2
 md5sum = a6d8ee96245fc21785d4c1c062c85f2f
 configure-command = true
 make-targets = 

component/python-2.6/buildout.cfg → component/python3/buildout.cfg

-
 [buildout]
-# XXX: Extends shall not jump out of software
 extends =
   ../bzip2/buildout.cfg
+  ../file/buildout.cfg
   ../gdbm/buildout.cfg
   ../gettext/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../libffi/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
+  ../patch/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
-  ../file/buildout.cfg
 
 parts =
-    python2.6
+    python3.5
 
-[python-2.6.6-no_system_inc_dirs.patch]
-recipe = hexagonit.recipe.download
-ignore-existing = true
-url = ${:_profile_base_location_}/${:filename}
-path = ${:filename}
-md5sum = ed7f8e11a97e45e5e09649c8b242e917
-download-only = true
-filename = python-2.6.6-no_system_inc_dirs.patch
-
-[python2.6]
+[python3-common]
 recipe = slapos.recipe.cmmi
+package_version_suffix =
+
 # This is actually the default setting for prefix, but we can't use it in
 # other settings in this part if we don't set it explicitly here.
 prefix = ${buildout:parts-directory}/${:_buildout_section_name_}
-version = 2.6
-package_version = ${:version}.8
-package_version_suffix =
-executable = ${:prefix}/bin/python${:version}
+executable = ${:prefix}/bin/${:_buildout_section_name_}
 
-url =
-  http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.bz2
-md5sum = c6e0420a21d8b23dee8b0195c9b9a125
 patch-options = -p1
 patches =
-  ${python-2.6.6-no_system_inc_dirs.patch:location}/${python-2.6.6-no_system_inc_dirs.patch:filename}
+  ${:_profile_base_location_}/default_encoding.patch#4ad9664e622d5556b4c32b1d9cb587ff
+url =
+  http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =
   --enable-ipv6
-  --enable-unicode=ucs4
+  --without-ensurepip
+  --with-system-expat
+  --with-system-ffi
   --with-threads
+# Profiled build:
+  --enable-optimizations
+pre-install = mkdir profile-opt
 
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include/ -I${ncurses:location}/include/ncursesw/ -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
+  PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
+  LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
 
-[bootstrap2.6]
-recipe = zc.recipe.egg
-eggs = zc.buildout
-suffix =
-scripts =
-    buildout=bootstrap2.6
-arguments = sys.argv[1:] + ["bootstrap"]
-python = python2.6
+[python3.5]
+<= python3-common
+package_version = 3.5.3
+md5sum = 57d1f8bfbabf4f2500273fb0706e6f21

component/python3/default_encoding.patch

+--- Python-3.5.3/Programs/python.c.orig	2017-01-17 08:57:53.000000000 +0100
++++ Python-3.5.3/Programs/python.c	2017-06-08 19:30:42.927077437 +0200
+@@ -24,6 +24,9 @@
+     int i, res;
+     char *oldloc;
+ 
++    if (!getenv("LC_CTYPE") && !getenv("LC_ALL"))
++        setenv("LANG", "C.UTF-8", 0);
++
+     argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+     argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+     if (!argv_copy || !argv_copy2) {

component/qemu-kvm/buildout.cfg

@@ -37,7 +37,7 @@ configure-options =
   --enable-vnc-png
   --disable-vnc-jpeg
   --extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${ncurses:location}/include -I${zlib:location}/include -I${libpng:location}/include -I${libaio:location}/include -I${attr:location}/include -I${libcap:location}/include"
-  --extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${gpg-error:location}/lib -L${gpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
+  --extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${libgpg-error:location}/lib -L${libgpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
   --disable-werror
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s

component/screen/buildout.cfg

 [buildout]
-extends =
-  ../patch/buildout.cfg
-
 parts =
   screen
-
 extends =
   ../ncurses/buildout.cfg
+  ../texinfo/buildout.cfg
 
 [screen]
 recipe = slapos.recipe.cmmi
-version = 4.0.3
-url = http://ftp.gnu.org/gnu/screen/screen-4.0.3.tar.gz
-md5sum = 8506fd205028a96c741e4037de6e3c42
-# Compatibility with CentOS
-patch-options = -p1 -b
-patches =
-  ${:_profile_base_location_}/screen-4.0.3-stropts.patch#5677917353067ffbcb08a23a59605aa4
+url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
+version = 4.5.1
+md5sum = a8c5da2f42f8a18fa4dada2419d1549b
 environment =
   CFLAGS=-I${ncurses:location}/include
   LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
-  PATH=${patch:location}/bin:%(PATH)s
+  PATH=${texinfo:location}/bin:%(PATH)s

component/screen/screen-4.0.3-stropts.patch

-diff -up screen-4.0.3/tty.sh.stropts screen-4.0.3/tty.sh
---- screen-4.0.3/tty.sh.stropts	2003-09-08 16:24:25.000000000 +0200
-+++ screen-4.0.3/tty.sh	2008-04-07 11:28:34.000000000 +0200
-@@ -76,7 +76,7 @@ exit 0
- #endif
- 
- #include "config.h"
--#ifdef SVR4
-+#if 0
- #include <sys/stropts.h>	/* for I_POP */
- #endif
- 
-diff -up screen-4.0.3/screen.c.stropts screen-4.0.3/screen.c
---- screen-4.0.3/screen.c.stropts	2008-04-07 11:25:21.000000000 +0200
-+++ screen-4.0.3/screen.c	2008-04-07 11:29:14.000000000 +0200
-@@ -50,7 +50,7 @@
- 
- #include "config.h"
- 
--#ifdef SVR4
-+#if 0
- # include <sys/stropts.h>
- #endif
- 
-diff -up screen-4.0.3/process.c.stropts screen-4.0.3/process.c
---- screen-4.0.3/process.c.stropts	2003-09-18 14:53:54.000000000 +0200
-+++ screen-4.0.3/process.c	2008-04-07 11:29:47.000000000 +0200
-@@ -37,7 +37,7 @@
- #include "config.h"
- 
- /* for solaris 2.1, Unixware (SVR4.2) and possibly others: */
--#ifdef SVR4
-+#if 0 
- # include <sys/stropts.h>
- #endif
- 
-diff -up screen-4.0.3/pty.c.stropts screen-4.0.3/pty.c
---- screen-4.0.3/pty.c.stropts	2003-09-08 16:26:18.000000000 +0200
-+++ screen-4.0.3/pty.c	2008-04-07 11:30:07.000000000 +0200
-@@ -34,7 +34,7 @@
- #endif
- 
- /* for solaris 2.1, Unixware (SVR4.2) and possibly others */
--#ifdef HAVE_SVR4_PTYS
-+#if 0 
- # include <sys/stropts.h>
- #endif
- 

component/slapos/buildout.cfg

@@ -28,8 +28,6 @@ parts =
   py
   firewalld-patch
 
-rebootstrap-directory = rebootstrap
-
 [environment]
 # Note: For now original PATH is appended to the end, as not all tools are
 #       provided by SlapOS

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://sqlite.org/2015/sqlite-autoconf-3081101.tar.gz
-md5sum = 298c8d6af7ca314f68de92bc7a356cbe
+url = http://sqlite.org/2017/sqlite-autoconf-3190000.tar.gz
+md5sum = 2426883b4dd3a9fd6aeb28a16b81a72b
 configure-options =
   --disable-static
   --enable-readline

component/w3-validator/buildout.cfg

 [buildout]
 extends =
   ../perl/buildout.cfg
+  ../perl-CGI/buildout.cfg
   ../perl-Config-General/buildout.cfg
   ../perl-Encode-HanExtra/buildout.cfg
   ../perl-HTML-Encoding/buildout.cfg
@@ -23,6 +24,7 @@ erp5.recipe.w3validator = 1.0.2
 [w3-validator]
 recipe = erp5.recipe.w3validator
 depends =
+  ${perl-CGI:location}
   ${perl-Config-General:location}
   ${perl-Encode-HanExtra:location}
   ${perl-HTML-Encoding:location}

component/xorg/buildout.cfg

@@ -525,9 +525,9 @@ configure-options =
   --with-sha1=libgcrypt
 environment =
   PKG_CONFIG_PATH=${pixman:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${fixesproto:location}/lib/pkgconfig:${damageproto:location}/lib/pkgconfig:${xcmiscproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig:${bigreqsproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${fontsproto:location}/lib/pkgconfig:${videoproto:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig:${resourceproto:location}/lib/pkgconfig:${xineramaproto:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${libXfont:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  CPPFLAGS=-I${bzip2:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${zlib:location}/include
+  CPPFLAGS=-I${bzip2:location}/include -I${libgcrypt:location}/include -I${libgpg-error:location}/include -I${zlib:location}/include
   PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${libgcrypt:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -L${libgpg-error:location}/lib -Wl,-rpath=${libgpg-error:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
 [libXi]
 recipe = slapos.recipe.cmmi
@@ -557,3 +557,20 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libXi:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig
+
+[scrnsaverproto]
+recipe = slapos.recipe.cmmi
+url = https://www.x.org/releases/X11R7.7/src/everything/scrnsaverproto-1.2.2.tar.bz2
+md5sum = edd8a73775e8ece1d69515dd17767bfb
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
+
+[libXScrnSaver]
+recipe = slapos.recipe.cmmi
+url = https://www.x.org/releases/X11R7.7/src/everything/libXScrnSaver-1.2.2.tar.bz2
+md5sum = 7a773b16165e39e938650bcc9027c1d5
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${libX11:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${scrnsaverproto:location}/lib/pkgconfig
+

component/zope-2.12/buildout.cfg

-[buildout]
-# adapt https://svn.erp5.org/repos/public/experimental/erp5.buildout-zope-2.12/

component/zope-2.8/buildout.cfg

-# based on https://svn.erp5.org/repos/public/experimental/erp5.buildout
-[buildout]
-extends=
-  ../python-2.4/buildout.cfg
-
-[zope-2.8]
-recipe = erp5.recipe.zope2install
-url = http://www.zope.org/Products/Zope/2.8.11/Zope-2.8.11-final.tgz
-md5sum = eff3e52d6ecde2d3669ea81e445b5a3a
-python = python2.4
-skip-fake-eggs =
-  ClientForm
-  mechanize
-  pytz

schema.json

@@ -2,49 +2,56 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "description": "Slapos Software Release instantiation descriptor",
   "additionalProperties": false,
+  "required": [
+    "software-type"
+  ],
   "properties": {
     "name": {
       "description": "A short human-friendly name for the sofware release",
+      "default": "",
       "type": "string"
     },
     "description": {
       "description": "A short description of the sofware release",
+      "default": "",
       "type": "string"
     },
     "serialisation": {
       "description": "How the parameters and results are serialised",
-      "require": true,
       "enum": ["xml", "json-in-xml"],
       "type": "string"
     },
     "software-type": {
       "description": "Existing software types",
-      "require": true,
       "patternProperties": {
         ".*": {
           "description": "Software type declaration",
           "additionalProperties": false,
+          "required": [
+            "request",
+            "response"
+          ],
           "properties": {
             "title": {
               "description": "A human-friendly title of the software type",
+              "default": "",
               "type": "string"
             },
             "description": {
               "description": "A human-friendly description of the software type",
+              "default": "",
               "type": "string"
             },
             "serialisation": {
-              "description": "How the parameters and results are serialised, if different from global setting",
+              "description": "How the parameters and results are serialised, if different from global setting, required if global setting is not provided",
               "enum": ["xml", "json-in-xml"],
               "type": "string"
             },
             "request": {
-              "require": true,
               "description": "URL, relative to Software Release base path, of a json schema for values expected by instance of current software type",
               "type": "string"
             },
             "response": {
-              "require": true,
               "description": "URL, relative to Software Release base path, of a json schema for values published by instance of current software type",
               "type": "string"
             },
@@ -54,10 +61,12 @@
             },
             "shared" : {
               "description": "Define if the request will request a Slave or Software Instance.",
+              "default": "false",
               "type": "boolean"
             },
             "index": {
               "description": "Value to use instead of software type id to sort them (in order to display most relevant software types earlier in a list, for example)",
+              "default": 0,
               "type": "number"
             }
           },

setup.py

@@ -163,6 +163,7 @@ setup(name=name,
           'publishsection = slapos.recipe.publish:PublishSection',
           'publishurl = slapos.recipe.publishurl:Recipe',
           'random.time = slapos.recipe.random:Time',
+          'random.integer = slapos.recipe.random:Integer',
           'readline = slapos.recipe.readline:Recipe',
           'redis.server = slapos.recipe.redis:Recipe',
           'request = slapos.recipe.request:Recipe',

slapos/recipe/check_url_available/__init__.py

@@ -42,6 +42,9 @@ class Recipe(GenericBaseRecipe):
       'check_secure': self.options.get('check-secure', 0),
       'http_code': self.options.get('http_code', '200'),
       'time_out': self.options.get('timeout-file-path', timeout_file),
+      'ca-cert-file': self.options.get('ca-cert-file', ''),
+      'cert-file': self.options.get('cert-file', ''),
+      'key-file': self.options.get('key-file', ''),
     }
 
     # XXX-Cedric in this script, curl won't check certificate

slapos/recipe/check_url_available/template/check_url.in

@@ -5,6 +5,9 @@
 URL="%(url)s"
 F_TIMEOUT="%(time_out)s"
 TIMEOUT=20
+CA_CERT="%(ca-cert-file)s"
+CERT="%(cert-file)s"
+KEY="%(key-file)s"
 
 if [ -z "$URL" ]; then
   echo "No URL specified." >&2
@@ -20,7 +23,11 @@ if [ -s "$F_TIMEOUT" ]; then
   fi
 fi
 
-CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null)
+if [ -z "$CA_CERT" ] || [ -z "$CERT" ] || [ -z "$KEY" ]; then
+  CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null)
+else
+  CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null --cacert $CA_CERT --cert $CERT --key $KEY)
+fi
 
 if [ $? -eq 3 ]; then
   echo "URL malformed: $URL." >&2

slapos/recipe/librecipe/execute.py

@@ -22,8 +22,8 @@ def _wait_files_creation(file_list):
     watchdescriptors = dict()
     for dirname in directories.keys():
       wd = inotifyx.add_watch(fd,
-                              dirname,
-                              inotifyx.IN_CREATE | inotifyx.IN_DELETE)
+                      dirname,
+                      inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
       watchdescriptors[wd] = dirname
 
     # Set to True the file wich exists

slapos/recipe/librecipe/generic.py

@@ -131,7 +131,7 @@ class GenericBaseRecipe(object):
 
   def createWrapper(self, name, command, parameters, comments=[],
       parameters_extra=False, environment=None,
-      pidfile=None
+      pidfile=None, reserve_cpu=False
   ):
     """
     Creates a shell script for process replacement.
@@ -139,6 +139,8 @@ class GenericBaseRecipe(object):
     Takes care of #! line limitation when the wrapped command is a script.
     if pidfile parameter is specified, then it will make the wrapper a singleton,
     accepting to run only if no other instance is running.
+
+    :param reserve_cpu: bool, try to reserve one core for the `command`
     """
 
     lines = [ '#!/bin/sh' ]
@@ -163,6 +165,14 @@ class GenericBaseRecipe(object):
           fi
           echo $$ > $pidfile""" % shlex.quote(pidfile)))
 
+    if reserve_cpu:
+      # if the CGROUPS cpuset is available (and prepared by slap format)
+      # request an exclusive CPU core for this process
+      lines.append(dedent("""
+        # put own PID into waiting list for exclusive CPU-core access
+        echo $$ >> ~/.slapos-cpu-exclusive
+        """))
+
     lines.append(dedent('''
     # If the wrapped command uses a shebang, execute the referenced
     # executable passing the script path as first argument.

slapos/recipe/neoppod.py

@@ -43,6 +43,7 @@ class NeoBaseRecipe(GenericBaseRecipe):
       # useful, as per NEO deploying constraints.
       raise UserError('"masters" parameter is mandatory')
     option_list = [
+      # Keep the -l option first, as expected by logrotate snippets.
       '-l', options['logfile'],
       '-m', options['masters'],
       '-b', self._getBindingAddress(),
@@ -91,6 +92,8 @@ class Storage(NeoBaseRecipe):
     engine = self.options.get('engine')
     if engine: # old versions of NEO don't support -e
       r  += '-e', engine
+    if self.options.get('disable-drop-partitions'):
+      r.append('--disable-drop-partitions')
     return r
 
 class Admin(NeoBaseRecipe):

slapos/recipe/random.py

@@ -39,6 +39,28 @@ import string
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+class Integer(object):
+  """
+  Generate a random integer (see standard random.randint documentation).
+
+  Input options:
+  minimum (required)
+    integer(-ish) minmum bound, included.
+  maximum (required)
+    integer(-ish) maximum bound, included.
+
+  Output options:
+  value
+    Resulting integer.
+  """
+  def __init__(self, buildout, name, options):
+    options['value'] = random.randint(int(options['minimum']), int(options['maximum']))
+
+  def install(self):
+    pass
+
+  update = install
+
 class Time(object):
   """Generate a random time from a 24h time clock"""
 
@@ -101,7 +123,6 @@ class Password(object):
   """
 
   def __init__(self, buildout, name, options):
-    options_get = options.get
     self.create_once = options.get('create-once', 'True').lower() \
           in GenericBaseRecipe.TRUE_VALUES
     try:
@@ -118,13 +139,21 @@ class Password(object):
         if e.errno != errno.ENOENT:
           raise
     if not passwd:
-      passwd = self.generatePassword(int(options_get('bytes', '8')))
+      passwd = self.generatePassword(int(options.get('bytes', '8')))
       self.update = self.install
     self.passwd = passwd.strip('\n')
     # Password must not go into .installed file, for 2 reasons:
     # security of course but also to prevent buildout to always reinstall.
-    options.get = lambda option, *args, **kw: passwd \
-      if option == 'passwd' else options_get(option, *args, **kw)
+    def get(option, *args, **kw):
+      return passwd if option == 'passwd' else options_get(option, *args, **kw)
+
+    try:
+      options_get = options._get
+    except AttributeError:
+      options_get = options.get
+      options.get = get
+    else:
+      options._get = get
 
   generatePassword = staticmethod(generatePassword)
 

slapos/recipe/request.py

@@ -170,6 +170,20 @@ class Recipe(object):
           # Backward compatibility. Old SlapOS master and core don't know this.
           self.logger.warning("Impossible to fetch instance GUID nor state.")
     except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
+      self.logger.warning(
+        'Request for %(request_name)r with software release '
+        '%(software_release)r and software type %(software_type)r failed '
+        'with partition_parameter_kw=%(partition_parameter_kw)r, '
+        'filter_kw=%(filter_kw)r, shared=%(shared)r, state=%(state)r.', dict(
+          software_release=software_url,
+          software_type=software_type,
+          request_name=name,
+          partition_parameter_kw=partition_parameter_kw,
+          filter_kw=filter_kw,
+          shared=slave,
+          state=requested_state
+        )
+      )
       self._raise_request_exception = exc
       self._raise_request_exception_formatted = traceback.format_exc()
       return_parameter_dict = {}

slapos/recipe/slapconfiguration.py

@@ -26,6 +26,7 @@
 ##############################################################################
 
 import json
+import logging
 import os
 
 import slapos.slap
@@ -33,15 +34,21 @@ from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
 from slapos.util import mkdir_p
+from slapos import format as slapformat
+
+
+logger = logging.getLogger("slapos")
+
 
 class Recipe(object):
   """
-  Retrieves slap partition parameters, and makes them available to other
-  buildout section in various ways, and in various encodings.
-  Populates the buildout section it is used in with all slap partition
-  parameters.
-  Also provides access to partition properties: all IPv4, IPv6 and tap
-  interfaces it is allowed to use.
+  Retrieve slap partition parameters and make them available in buildout section.
+
+  There are two sources of parameters. First is configuration file slapos.cfg and
+  derived information.
+  Second is partition's resource_file which is made available in form of keys joined
+  with "-" and with all "_" replaced by "-".
+  For example {"tun": {"ipv4": <addr>}} would be available in buildout as ${instance:tun-ipv4}.
 
   Input:
     url
@@ -67,7 +74,7 @@ class Recipe(object):
       Example:
         ${storage-configuration:storage-home}
 
-  Output:
+  Output (keys derived from SlapOS configuration):
     root-instance-title
       Hosting subscription or root instance title
     instance-title
@@ -122,6 +129,12 @@ class Recipe(object):
           options['configuration.' + key] = value
 
   def fetch_parameter_dict(self, options, instance_root):
+      """Gather parameters about current computer and partition.
+
+      Use two sources of truth
+      1. SlapOS Master - for external computer/partition information
+      2. format.Partition.resource_file - for partition specific details
+      """
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -233,6 +246,27 @@ class Recipe(object):
       options['storage-dict'] = storage_dict
 
       options['tap'] = tap_set
+
+      # The external information transfered from Slap Master has been processed
+      # so we extend with information gathered from partition resource file
+      if hasattr(slapformat.Partition, "resource_file"):
+        resource_home = instance_root
+        while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
+          resource_home = os.path.normpath(os.path.join(resource_home, '..'))
+          if resource_home == "/":
+            break
+        else:
+          # no break happened - let's add partition resources into options
+          logger.debug("Using partition resource file {}".format(
+            os.path.join(resource_home, slapformat.Partition.resource_file)))
+          with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
+            partition_params = json.load(fi)
+          # be very careful with overriding master's information
+          for key, value in flatten_dict(partition_params).items():
+            if key not in options:
+              options[key] = value
+      # print out augmented options to see what we are passing
+      logger.debug(str(options))
       return self._expandParameterDict(options, parameter_dict)
 
   def _expandParameterDict(self, options, parameter_dict):
@@ -261,3 +295,14 @@ class JsonDump(Recipe):
 
     update = install
 
+
+def flatten_dict(data, key_prefix=''):
+  """Transform folded dict into one-level key-subkey-subsubkey dictionary."""
+  output = {}
+  for key, value in data.items():
+    prefixed_key = key_prefix + key.replace("_", "-")  # to be consistent with `fetch_parameter_dict`
+    if isinstance(value, dict):
+      output.update(flatten_dict(value, key_prefix=prefixed_key + "-"))
+      continue
+    output[prefixed_key] = value
+  return output

slapos/recipe/wrapper.py

@@ -31,6 +31,16 @@ import os
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
+    """Recipe to create a script from given command and options.
+
+    :param str command-line: shell command which launches the intended process
+    :param str wrapper-path: absolute path to file's destination
+
+    :param lines wait-for-files: list of files to wait for
+    :param str pidfile: path to pidfile ensure exclusivity for the process
+    :param bool parameters-extra: whether wrapper parameters are passed onto command
+    :param bool reserve-cpu: command will ask for an exclusive CPU core
+    """
     def install(self):
         command_line = shlex.split(self.options['command-line'])
         wrapper_path = self.options['wrapper-path']
@@ -38,6 +48,7 @@ class Recipe(GenericBaseRecipe):
         environment = self.options.get('environment')
         parameters_extra = self.options.get('parameters-extra')
         pidfile = self.options.get('pidfile')
+        reserve_cpu = self.options.get('reserve-cpu', False)
 
         if not wait_files and not environment:
           # Create a simple wrapper as shell script
@@ -47,6 +58,7 @@ class Recipe(GenericBaseRecipe):
              parameters=command_line[1:],
              parameters_extra=parameters_extra,
              pidfile=pidfile,
+             reserve_cpu=reserve_cpu
           )]
 
         # More complex needs: create a Python script as wrapper
@@ -76,5 +88,6 @@ class Recipe(GenericBaseRecipe):
              parameters=[],
              parameters_extra=parameters_extra,
              pidfile=pidfile,
+             reserve_cpu=reserve_cpu
         )]
 

slapos/recipe/zero_knowledge.py

@@ -36,32 +36,36 @@ class WriteRecipe(GenericBaseRecipe):
   """
   """
   def __init__(self, buildout, name, options):
-    if not "filename" in options:
-      raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
+    if not "filename" in options and not "file-path" in options:
+      raise zc.buildout.UserError("You have to provide the parameter either \"filename\" or \"file-path\"")
 
-    self.filename = options['filename'].strip()
-    self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+    self._options = options.copy()
+    if options.get('filename'):
+      self.filename = options['filename'].strip()
+      self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+      del self._options['filename']
+    else:
+      self.path = options['file-path'].strip()
+      del self._options['file-path']
+    del self._options['recipe']
     self.name = name
-    self.options = options.copy()
-    del self.options['filename']
-    del self.options['recipe']
+
+  def install(self):
 
     # Set up the parser, and write config file if needed
     self.parser = ConfigParser.ConfigParser()
     try:
       self.parser.read(self.path)
       #clean_options(options)
-      for key in self.options:
+      for key in self._options:
         if key not in self.parser.options(self.name):
-          self.parser.set(self.name, key, self.options[key])
+          self.parser.set(self.name, key, self._options[key])
       with open(self.path, 'w') as file:
         self.parser.write(file)
     # If the file or section do not exist
     except (ConfigParser.NoSectionError, IOError) as e:
       self.full_install()
 
-  install = update = lambda self: []
-
   def full_install(self):
     """XXX-Nicolas : when some parameter's value is changed in
     buildout profile, this will override custom user defined values"""
@@ -69,21 +73,25 @@ class WriteRecipe(GenericBaseRecipe):
     if self.parser.has_section(self.name):
       self.parser.remove_section(self.name)
     self.parser.add_section(self.name)
-    for key in self.options:
-      self.parser.set(self.name, key, self.options[key])
+    for key in self._options:
+      self.parser.set(self.name, key, self._options[key])
     with open(self.path, 'w') as file:
       self.parser.write(file)
 
+  update = install
 
 class ReadRecipe(GenericBaseRecipe):
   """
   """
   def __init__(self, buildout, name, options):
-    if not "filename" in options:
-      raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
+    if not "filename" in options and not "file-path" in options:
+      raise zc.buildout.UserError("You have to provide the parameter either \"filename\" or file-path")
 
-    self.filename = options['filename'].strip()
-    self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+    if options.get('filename'):
+      self.filename = options['filename'].strip()
+      self.path = os.path.join(buildout['buildout']['directory'], self.filename)
+    else:
+      self.path = options['file-path'].strip()
 
     # Set up the parser, and write config file if needed
     self.parser = ConfigParser.ConfigParser()

slapos/test/recipe/test_slaposconfiguration.py

+# coding: utf-8
+import json
+import mock
+import os
+import unittest
+from collections import defaultdict
+from slapos.recipe import slapconfiguration
+from slapos import format as slapformat
+
+
+class SlapConfigurationTest(unittest.TestCase):
+
+  def setUp(self):
+    """Prepare files on filesystem."""
+    self.instance_root = "/tmp/instance_test_resourcefile"
+    os.mkdir(self.instance_root)
+    # create testing resource file
+    self.resource_file = os.path.join(self.instance_root, slapformat.Partition.resource_file)
+    self.resource = {
+      "tun": {
+        "ipv4": "192.168.0.1"
+      },
+      "address_list": [
+        10, 20
+      ]
+    }
+    with open(self.resource_file, "wt") as fo:
+      json.dump(self.resource, fo)
+    # do your tests inside try block and clean up in finally
+    self.buildout = {
+      "buildout": {
+        "directory": self.instance_root
+      }
+    }
+
+  def tearDown(self):
+    os.unlink(self.resource_file)
+    os.rmdir(self.instance_root)
+
+  @mock.patch("slapos.slap.slap")
+  def test_correct_naming(self, MockClient):
+    """Test correct naming of variables from resource file."""
+    MockClient.initializeConnection.return_value = None
+    MockClient.getInstanceParameterDict.return_value = dict()
+  
+    options = defaultdict(str)
+    recipe = slapconfiguration.Recipe(self.buildout, "slapconfiguration", options)
+
+    self.assertEqual(options['tun-ipv4'], "192.168.0.1",
+      "Folded attrs should be separated by -")
+
+    self.assertEqual(options['address-list'], [10, 20],
+      "All underscores should be replaced with -")

software/agent/software.cfg

@@ -45,35 +45,35 @@ eggs =
 [versions]
 apache-libcloud = 0.18.0
 ecdsa = 0.13
-erp5.util = 0.4.46
+erp5.util = 0.4.49
 gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 2.8
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 rpdb = 0.1.5

software/apache-frontend/buildout.hash.cfg

@@ -19,15 +19,15 @@ md5sum = f686f765e55d1dce2e55a400f0714b3e
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg
-md5sum = e2f60121bc1116319b4d20b4c4badc1c
+md5sum = 979eecabeaad0114fb24f65bf52f7e6a
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = 9b17c835bcd927269cf510bf612f5985
+md5sum = c55506cfb92d87f915d50d32fb129a28
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 8a8db11499f0e3b00c0bd7d1e5afc99b
+md5sum = 4010f7366262d00c33f9e7f53f63d42e
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -39,11 +39,11 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e
 
 [template-apache-frontend-configuration]
 filename = templates/apache.conf.in
-md5sum = 51e0c8957e3cf30e09c7b836143c15c9
+md5sum = 82cdb4ab02fec36285b9c1ce502f82f0
 
 [template-custom-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 8a8db11499f0e3b00c0bd7d1e5afc99b
+md5sum = 4010f7366262d00c33f9e7f53f63d42e
 
 [template-not-found-html]
 filename = templates/notfound.html
@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 8fda2e88476ccb3652036302364f3506
+md5sum = 5344bff68a3f7dead633a35771ad1a19
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in
@@ -92,3 +92,11 @@ md5sum = a5186f666acb2f040ede04c91e60408f
 [template-nginx-notebook-slave-virtualhost]
 filename = templates/nginx-notebook-slave.conf.in
 md5sum = 82d74a7f2aceb2b4a7acc6259291b7f2
+
+[template-apache-lazy-script-call]
+filename = templates/apache-lazy-script-call.sh.in
+md5sum = ebe5d3d19923eb812a40019cb11276d8
+
+[template-apache-graceful-script]
+filename = templates/apache-graceful-script.sh.in
+md5sum = 41299cc64200e7b8217fb9dec20bb8b9

software/apache-frontend/common.cfg

@@ -141,6 +141,14 @@ url = ${:_profile_base_location_}/templates/nginx.cfg.in
 output = ${buildout:directory}/template-nginx.cfg.in
 mode = 0644
 
+[template-apache-lazy-script-call]
+<=download-template
+filename = apache-lazy-script-call.sh.in
+
+[template-apache-graceful-script]
+<=download-template
+filename = apache-graceful-script.sh.in
+
 [template-nginx-eventsource-slave-virtualhost]
 <=download-template
 filename = nginx-eventsource-slave.conf.in

software/apache-frontend/instance-apache-frontend.cfg

@@ -23,6 +23,7 @@ parts =
   promise-apache-frontend-cached
   promise-apache-frontend-ssl-cached
   promise-apache-is-process-older-than-dependency-set
+  promise-apache-mpm-watchdog
 
   promise-nginx-frontend-v4-https
   promise-nginx-frontend-v4-http
@@ -44,7 +45,6 @@ parts =
 ## Nginx
   nginx-frontend
 
-
 ## Monitor for apache
   monitor-base
   monitor-ats-cache-stats-wrapper
@@ -172,8 +172,6 @@ extra-context =
     key nginx_configuration_directory apache-directory:nginx-slave-configuration
     key apache_cached_configuration_directory apache-directory:slave-with-cache-configuration
     key slave_with_cache_configuration_directory apache-directory:slave-with-cache-configuration
-    key cached_port apache-configuration:cache-through-port
-    key ssl_cached_port apache-configuration:ssl-cache-through-port
     key http_port instance-parameter:configuration.plain_http_port
     key https_port instance-parameter:configuration.port
     key nginx_http_port instance-parameter:configuration.plain_nginx_port
@@ -185,7 +183,6 @@ extra-context =
     key apache_log_directory apache-directory:slave-log
     key local_ipv4 instance-parameter:ipv4-random
     key global_ipv6 slap-network-information:global-ipv6
-    key cache_port apache-configuration:cache-port
     key varnginx directory:varnginx
     key empty_template software-release-path:template-empty
     key template_custom_slave_configuration software-release-path:template-slave-configuration
@@ -194,6 +191,7 @@ extra-context =
     key template_eventsource_slave_configuration software-release-path:template-nginx-eventsource-slave-virtualhost
     key template_notebook_slave_configuration software-release-path:template-nginx-notebook-slave-virtualhost
     raw software_type single-custom-personal
+    key frontend_lazy_graceful_reload frontend-apache-lazy-graceful:rendered
     section logrotate_dict logrotate
     section frontend_configuration frontend-configuration
     section apache_configuration apache-configuration
@@ -372,7 +370,7 @@ name = apache
 log = $${apache-configuration:error-log} $${apache-configuration:access-log}
 frequency = daily
 rotatep-num = 30
-post = $${apache-configuration:frontend-graceful-command}
+post = $${frontend-apache-lazy-graceful:rendered} &
 sharedscripts = true
 notifempty = true
 create = true
@@ -500,14 +498,35 @@ mode = 700
 ### End of ATS sections
 
 ### Apaches Graceful and promises
-[frontend-apache-graceful]
+[frontend-apache-graceful-bin]
 < = jinja2-template-base
 template = ${template-wrapper:output}
-rendered = $${directory:etc-run}/frontend-apache-safe-graceful
+rendered = $${directory:bin}/frontend-apache-safe-graceful
 mode = 0700
 extra-context =
     key content apache-configuration:frontend-graceful-command
 
+[frontend-apache-graceful]
+< = jinja2-template-base
+template = ${template-apache-graceful-script:target}
+rendered = $${directory:etc-run}/frontend-apache-safe-graceful
+mode = 0700
+extra-context =
+    key directory_run directory:run
+    key directory_etc directory:etc
+    key apache_graceful_reload_command apache-configuration:frontend-graceful-command
+
+[frontend-apache-lazy-graceful]
+< = jinja2-template-base
+template = ${template-apache-lazy-script-call:target}
+rendered = $${directory:bin}/frontend-apache-lazy-graceful
+mode = 0700
+pid-file = $${directory:run}/lazy-graceful.pid
+extra-context =
+    key pid_file :pid-file
+    raw wait_time 60
+    key lazy_command apache-configuration:frontend-graceful-command
+
 # Promises checking configuration:
 [promise-frontend-apache-configuration]
 < = jinja2-template-base
@@ -553,13 +572,18 @@ path = $${directory:promise}/apache_ssl_cached
 hostname = $${instance-parameter:ipv4-random}
 port = $${apache-configuration:ssl-cache-through-port}
 
-
 [promise-apache-is-process-older-than-dependency-set]
 recipe = slapos.cookbook:wrapper
 command-line = ${buildout:bin-directory}/is-process-older-than-dependency-set $${apache-configuration:pid-file} 
 wrapper-path = $${directory:promise}/apache-frontend-is-running-actual-software-release
 parameters-extra = true
 
+[promise-apache-mpm-watchdog]
+recipe = slapos.cookbook:wrapper
+command-line = ${buildout:bin-directory}/apache-mpm-watchdog --url http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u admin -p $${monitor-htpasswd:passwd} -d $${directory:run}/apache-mpm-watchdog.json 
+wrapper-path = $${directory:promise}/apache-mpm-watchdog
+parameters-extra = true
+
 
 [slap_connection]
 # Kept for backward compatiblity
@@ -589,7 +613,7 @@ extra_slave_instance_list =
 frontend-name =
 monitor-cors-domains =
 monitor-httpd-port = 8072
-monitor-username = $${monitor-htpasswd:username}
+monitor-username = $${monitor-instance-parameter:username}
 monitor-password = $${monitor-htpasswd:passwd}
 
 #######
@@ -631,7 +655,7 @@ extra-context =
 template = ${template-wrapper:output}
 rendered = $${monitor-directory:reports}/monitor-apache-server-status-wrapper
 mode = 0700
-command = ${curl:location}/bin/curl -s http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u $${monitor-htpasswd:username}:$${monitor-htpasswd:passwd} 2>&1 
+command = ${curl:location}/bin/curl -s http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u $${monitor-instance-parameter:username}:$${monitor-htpasswd:passwd} 2>&1
 extra-context =
   key content monitor-apache-server-status-wrapper:command
 

software/apache-frontend/instance-apache-input-schema.json

@@ -14,7 +14,44 @@
 			"description": "Value used to set IPReadLimit Parameter for antiloris.",
 			"type": "integer",
 			"default": 10
-		},		
+		},
+                "mpm-server-limit": {
+                        "title": "ServerLimit",
+                        "description": "Value used to set ServerLimit on apache configuration.",
+                        "type": "integer",
+                        "default": 16
+                },
+                "mpm-max-clients": {
+                        "title": "MaxClients",
+                        "description": "Value used to set MaxClients on apache configuration.",
+                        "type": "integer",
+                        "default": 400
+                },
+                "mpm-start-servers": {
+                        "title": "StartServers",
+                        "description": "Value used to set StartServers on apache configuration.",
+                        "type": "integer",
+                        "default": 3
+                },
+                "mpm-thread-per-child": {
+                        "title": "ThreadsPerChild",
+                        "description": "Value used to set ThreadsPerChild on apache configuration.",
+                        "type": "integer",
+                        "default": 25
+                },
+                "mpm-graceful-shutdown-timeout": {
+                        "title": "ThreadsPerChild",
+                        "description": "Value used to set ThreadsPerChild on apache configuration.",
+                        "type": "integer",
+                        "default": 5
+                },
+                "enable-http2-by-default": {
+                        "title": "Enable HTTP2 by Default",
+                        "description": "Use HTTP2 as default Protocol",
+                        "type": "string",
+                        "default": "true",
+                        "enum": ["true", "false"]
+                },
                 "re6st-verification-url": {
 			"title": "Test Verification URL",
 			"description": "Url to verify if the internet and/or re6stnet is working.",

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -81,7 +81,7 @@ context =
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
 config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-config-monitor-username = ${monitor-htpasswd:username}
+config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 {% set frontend_software_url_key = "-frontend-software-release-url" %}

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -140,7 +140,13 @@
       "default": "false",
       "enum": ["false", "true"]
     },
-
+    "enable-http2": {
+      "title": "Enable HTTP2 Protocol",
+      "description": "Use HTTP2 Protocol for the site",
+      "type": "string",
+      "default": "true",
+      "enum": ["true", "false"]
+    },
     "prefer-gzip-encoding-to-backend": {
       "title": "Prefer gzip Encoding for Backend",
       "description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",

software/apache-frontend/software.cfg

@@ -10,8 +10,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 2.8
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3
@@ -19,41 +19,41 @@ pyasn1 = 0.2.3
 websockify = 0.8.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 dnspython = 1.15.0
 
 # Required by:
-# slapos.toolbox==0.61
-erp5.util = 0.4.46
+# slapos.toolbox==0.69
+erp5.util = 0.4.49
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 passlib = 1.6.5
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 pycurl = 7.43.0
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 rpdb = 0.1.5

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -2,6 +2,9 @@
 
 {% set cached_server_dict = {} -%}
 {% set part_list = [] -%}
+{% set cache_port = apache_configuration.get('cache-port') %}
+{% set cached_port = apache_configuration.get('cache-through-port') %}
+{% set ssl_cached_port = apache_configuration.get('ssl-cache-through-port') %}
 {% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
 {% set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) -%}
 {% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
@@ -27,7 +30,7 @@ context =
 {{ key }} = {{ value }}
 {% endfor %}
 
-post = {{ apache_configuration.get('frontend-graceful-command') }}
+post = {{ frontend_lazy_graceful_reload }} &
 frequency = daily
 rotatep-num = 30
 sharedscripts = true
@@ -106,7 +109,7 @@ crl = {{ custom_ssl_directory }}/crl/
 {%   endif -%}
 
 [slave-log-directories]
-{{slave_reference}}-log-folder = {{ slave_log_folder }}
+{{slave_reference}} = {{ slave_log_folder }}
 
 {# Set slave logrotate entry #}
 [{{slave_logrotate_section}}]

software/apache-frontend/templates/apache-graceful-script.sh.in

+#!/bin/sh
+
+RUN_DIR={{ directory_run }}
+ETC_DIR={{ directory_etc }}
+
+APACHE_SIGNATURE_FILE=$RUN_DIR/apache_configuration.signature
+NAPACHE_SIGNATURE_FILE=$RUN_DIR/napache_configuration.signature
+
+touch $APACHE_SIGNATURE_FILE
+sha256sum $ETC_DIR/apache*.conf $ETC_DIR/apache-*.d/*.conf $ETC_DIR/apache-*.d/ssl/*.*key $ETC_DIR/apache-*.d/ssl/*.*crt* | sort -k 66 > $NAPACHE_SIGNATURE_FILE 
+
+# If no diff, no restart for now
+if diff "$APACHE_SIGNATURE_FILE" "$NAPACHE_SIGNATURE_FILE"; then
+  echo "Nothing Changed, so nothing to reload"
+  exit 0
+fi
+echo "Reloading apache.."
+
+{{ apache_graceful_reload_command }}
+
+mv "$NAPACHE_SIGNATURE_FILE" "$APACHE_SIGNATURE_FILE" 

software/apache-frontend/templates/apache-lazy-script-call.sh.in

+#!/bin/bash
+
+PID=$$
+PIDFILE={{ pid_file }}
+
+sleep $((RANDOM % 10))
+
+if [ ! -f $PIDFILE ]; then
+  echo $PID > $PIDFILE
+  sleep {{ wait_time }}
+  {{ lazy_command }}
+else
+  ps --pid `cat $PIDFILE` &>/dev/null
+  if [ $? -eq 0 ]; then
+    echo "Skipped"
+  else
+    echo $PID > $PIDFILE
+    sleep {{ wait_time }}
+    {{ lazy_command }}
+  fi
+fi
+

software/apache-frontend/templates/apache.conf.in

@@ -16,7 +16,6 @@ ServerRoot {{ instance_home }}
 {% endfor -%}
 
 ServerAdmin {{ server_admin }}
-DefaultType text/plain
 TypesConfig {{ httpd_home }}/conf/mime.types
 AddType application/x-compress .Z
 AddType application/x-gzip .gz .tgz
@@ -31,7 +30,7 @@ TraceEnable off
 
 # Log configuration
 ErrorLog "{{ error_log }}"
-LogLevel info
+LogLevel notice
 # LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
 # LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b" common
 # CustomLog "{{ access_log }}" common
@@ -83,6 +82,7 @@ LoadModule filter_module       {{ httpd_home }}/modules/mod_filter.so
 LoadModule http2_module       {{ httpd_home }}/modules/mod_http2.so
 LoadModule info_module        {{ httpd_home }}/modules/mod_info.so
 LoadModule status_module      {{ httpd_home }}/modules/mod_status.so
+LoadModule reqtimeout_module  {{ httpd_home }}/modules/mod_reqtimeout.so
 
 # The following directives modify normal HTTP response behavior to
 # handle known problems with browser implementations.
@@ -105,6 +105,11 @@ BrowserMatch "^gnome-vfs" redirect-carefully
 BrowserMatch "^XML Spy" redirect-carefully
 BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
 
+
+{% if slapparameter_dict.get('enable-http2-by-default', 'true') == 'true' %}
+Protocols h2 http/1.1
+{% endif %}
+
 # Increase IPReadLimit to 10
 <IfModule antiloris_module>
    # IPReadLimit - Maximum simultaneous connections in READ state per IP address 
@@ -125,7 +130,12 @@ ExtendedStatus On
     Require valid-user
 </Location>
 
-
+ServerLimit {{ slapparameter_dict.get('mpm-server-limit', '16') }}
+MaxClients {{ slapparameter_dict.get('mpm-max-clients', '400') }}
+StartServers {{ slapparameter_dict.get('mpm-start-servers', '3') }}
+ThreadsPerChild {{ slapparameter_dict.get('mpm-thread-per-child', '25') }}
+#MaxRequestsPerChild 0 // Default value is 0
+GracefulShutdownTimeout {{ slapparameter_dict.get('mpm-graceful-shutdown-timeout', '5') }}
 
 # Deflate
 AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
@@ -156,13 +166,7 @@ SSLProxyCheckPeerCN off
 SSLProxyCheckPeerExpire off
 
 include {{frontend_configuration.get('log-access-configuration')}}
-
-NameVirtualHost *:{{ http_port }}
-NameVirtualHost *:{{ https_port }}
 include {{ slave_configuration_directory }}/*.conf
-
-NameVirtualHost *:{{ cached_port }}
-NameVirtualHost *:{{ ssl_cached_port }}
 include {{ slave_with_cache_configuration_directory }}/*.conf
 
 ErrorDocument 404 /notfound.html

software/apache-frontend/templates/default-virtualhost.conf.in

@@ -3,6 +3,7 @@
 {%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES -%}
 {%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES -%}
 {%- set server_alias_list =  slave_parameter.get('server-alias', '').split() -%}
+{%- set enable_h2 = ('' ~ slave_parameter.get('enable-http2', 'true')).lower() in TRUE_VALUES -%}
 {%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
 {%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() -%}
 {%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES -%}
@@ -33,8 +34,10 @@
   SSLProtocol all -SSLv2 -SSLv3
   SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
   SSLHonorCipherOrder on
-  Protocols h2 http/1.1
 
+{% if enable_h2 %}
+  Protocols h2 http/1.1
+{% endif -%}
 
 {% for key, value in ssl_configuration_list -%}
 {%   if value in slave_parameter -%}
@@ -44,7 +47,7 @@
 
   # One Slave two logs
   ErrorLog "{{ slave_parameter.get('error_log') }}"
-  LogLevel info
+  LogLevel notice
   LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
   CustomLog "{{ slave_parameter.get('access_log') }}" combined
 
@@ -116,14 +119,16 @@
 
   # One Slave two logs
   ErrorLog "{{ slave_parameter.get('error_log') }}"
-  LogLevel info
+  LogLevel notice
   LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
   CustomLog "{{ slave_parameter.get('access_log') }}" combined
 
   # Remove "Secure" from cookies, as backend may be https
   Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
 
+{% if enable_h2 %}
   Protocols h2 http/1.1
+{% endif -%}
 
 {% if disable_no_cache_header %}
   RequestHeader unset Cache-Control

software/caucase/instance-caucase-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "server-port": {
+      "title": "http port to use",
+      "description": "Caucase http port to use.",
+      "type": "integer",
+      "default": 8009,
+      "minimum": 1,
+      "maximum": 65535
+    },
+    "server-https-port": {
+      "title": "https port to use",
+      "description": "Caucase port to use for https connexion.",
+      "type": "integer",
+      "default": 8010,
+      "minimum": 1,
+      "maximum": 65535
+    },
+    "external-url": {
+      "title": "External http url",
+      "description": "External http url which point to caucase on http. This url will be added in signed certificate as CRL distribution point URI",
+      "type": "string",
+      "format": "uri"
+    },
+    "ca-subject": {
+      "title": "Subject of CA Certificate",
+      "description": "CA certificate subject as string. The format is: /C=XX/ST=State/L=City/OU=OUnit/O=Company/CN=CA Auth/emailAddress=xx@example.com. Only /CN is mandatory.",
+      "type": "string",
+      "default": "/C=FR/O=Company/CN=SlapOS Certificate Authority/emailAddress=xx@example.com"
+    },
+    "max-request-amount": {
+      "title": "Number of pending csr to accept",
+      "description": "Number of pending csr to accept. If this limit is reached, no more csr will be accepted by the CA.",
+      "type": "integer",
+      "default": 10
+    },
+    "crt-life-time": {
+      "title": "Signed Certificate life time",
+      "description": "The time in seconds before a generated certificate will expire. Default: 365*24*60*60 seconds (1 year)",
+      "type": "integer",
+      "default": 31536000,
+      "minimum": 86400
+    },
+    "crl-life-period": {
+      "title": "CRL life time period",
+      "description": "Number of individual certificate validity periods during which the CRL is valid. Default: 1/50.0",
+      "type": "float",
+      "default": 0.2
+    },
+    "ca-life-period": {
+      "title": "CA Certificate life period",
+      "description": "Number of individual certificate validity periods during which the CA certificate is valid. Default: 10",
+      "type": "float",
+      "default": 10
+    },
+    "crt-keep-time": {
+      "title": "Time before cleanup certificate content on CA",
+      "description": "The time in seconds before a generated certificate will be deleted on CA server. Set 0 to never delete. Default: 30*24*60*60 seconds (30 days)",
+      "default": 5184000,
+      "type": "integer"
+    }
+  }
+}
\ No newline at end of file

software/caucase/instance-caucase-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Caucase instantiation",
+  "properties": {
+    "http-url": {
+      "description": "Caucase URL on HTTP",
+      "type": "string"
+    },
+    "https-url": {
+      "description": "Caucase URL on HTTPS",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/caucase/software.cfg

+[buildout]
+extends =
+  ../../stack/caucase/buildout.cfg
+  ../../stack/slapos.cfg
+
+parts = 
+  slapos-cookbook
+  extra-eggs
+  instance-caucase
+

software/caucase/software.cfg.json

+{
+    "name": "Caucase",
+    "description": "Caucase certificate authority",
+    "serialisation": "json-in-xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Single caucase instance",
+            "request": "instance-caucase-input-schema.json",
+            "response": "instance-caucase-output-schema.json",
+            "index": 0
+        }
+    }
+}
\ No newline at end of file

software/cdn-me/software.cfg

@@ -38,6 +38,6 @@ eggs =
 [versions]
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
-erp5.util = 0.4.46
+erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0

software/cloudooo/instance-cloudooo.cfg.in

@@ -42,7 +42,7 @@
 {%   do apache_ip_list.append('[' ~ ipv6 ~ ']') -%}
 {% endif -%}
 {% set apache_dict = {} -%}
-{% do apache_dict.__setitem__(publish_url_name, (apache_port, "https", 'http://' ~ ipv4 ~ ':' ~ haproxy_port)) -%}
+{% do apache_dict.__setitem__(publish_url_name, (apache_port, "https", 'http://' ~ ipv4 ~ ':' ~ haproxy_port, False)) -%}
 
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
 {% set section_list = [] -%}

software/cloudooo/software-common.cfg

@@ -83,5 +83,5 @@ template-logrotate-base = ${template-logrotate-base:rendered}
 [template-cloudooo-instance]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudooo.cfg.in
-md5sum = e1a4a73aeb8b978e953a3f4f39b98a4d
+md5sum = 0a260754586e2a44010f3303c58a927a
 mode = 640

software/dream/software.cfg

@@ -48,12 +48,12 @@ scipy = 0.13.3
 simpy = 3.0.5
 zope.dottedname = 4.1.0
 tablib = 0.10.0
-mysqlclient = 1.3.10
+mysqlclient = 1.3.10+SlapOSPatched001
 
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/instance-erp5-input-schema.json

@@ -111,6 +111,12 @@
               "default": 5,
               "type": "integer"
             },
+            "ssl-authentication": {
+              "title": "Enable SSL Client authentication on this zope instance.",
+              "description": "If set to true, will set SSL Client verification to required on apache VirtualHost which allow to access this zope instance.",
+              "type": "boolean",
+              "default": false
+            },
             "webdav": {
               "description": "Serve webdav queries, implies timerserver-interval=0 (disabled). Mixing webdav and non-webdav nodes in a single family will give unspecified results.",
               "default": false,
@@ -263,6 +269,28 @@
       "description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
       "default": "",
       "type": "string"
+    },
+    "caucase": {
+      "description": "Caucase certificate authority parameters",
+      "properties": {
+        "url": {
+          "title": "Caucase URL",
+          "description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
+          "default": "",
+          "type": "string",
+          "format": "uri"
+        },
+        "crl-update-periodicity": {
+            "title": "Periodicity of CRL update",
+            "description": "Periodicity of CRL update, in cron format. The CRL will be downloaded from caucase URL and the new content will be saved if there was a change. Everytime a new CRL is writen, Apache reload will be called.",
+            "type": "string",
+            "default": "0 0 * * *"
+        }
+      },
+      "additionalProperties": {
+        "$ref": "../caucase/instance-caucase-input-schema.json#/properties"
+      },
+      "type": "object"
     }
   }
 }

software/erp5/instance-erp5-output-schema.json

@@ -72,6 +72,11 @@
       "description": "Jupyter notebook web UI access information",
       "pattern": "^https://",
       "type": "string"
+    },
+    "caucase-http-url": {
+      "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.",
+      "pattern": "^http://",
+      "type": "string"
     }
   },
   "patternProperties": {

software/erp5testnode/software.cfg

@@ -60,6 +60,6 @@ md5sum = 05519f3887a309d3ec069e0aa9f52ebc
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.48
-slapos.recipe.template = 2.7
+erp5.util = 0.4.49
+slapos.recipe.template = 3.0
 ipython = 5.3.0

software/erp5testnode/testsuite/deploy-test/README.rst

+deply-test
+==========
+
+Introduction
+------------
+
+This is software release to run tests on VMs, but without direct access (like
+ssh) to the VM.
+
+It is supposed to be used as backend for ERP5TestNode.
+
+Characteristics and limitations:
+
+ * partitions share the same user (as they access files directly)
+ * this SR will be installed and instantiated only from local file system
+   (like git clone)

software/erp5testnode/testsuite/deploy-test/deploy-script-controller

+#!/bin/bash
+
+# Script for controlling deploy script
+#
+# Assumption: deployment script returning with code 0
+#             run correctly and everything is done
+#
+# This script is run inside of KVM, by passing its
+# URL with bootstrap-script-url
+# The script configuration happens by passing
+# it in text form with data-to-vm
+#
+# Format of data-to-vm is shell script:
+#  URL=<url>\nWAITTIME=<seconds>\nTRIES=<amount>
+#
+# Expected values in configuration:
+#  * URL - the url of the script to download and test
+#  * WAITTIME - waiting time, before next try
+#  * TRIES - amount of tries
+
+# Possible TODOs:
+#  * post results on each try
+#  * use function + trap to assure posting on exit
+
+LOG_FILE=/var/log/test-script-deployment.log
+
+wget -O /tmp/test-script.cfg.$$ -q http://10.0.2.100/data
+
+source /tmp/test-script.cfg.$$
+
+if [ -z "$LOG_FILE" ] ; then
+  echo "Output log file is missing"
+  exit 1
+fi
+
+if [ -z "$URL" ] ; then
+  echo "URL is missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+if [ -z "$WAITTIME" ] ; then
+  echo "WAITTIME missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+if [ -z "$TRIES" ] ; then
+  echo "TRIES missing" >> $LOG_FILE 2>&1
+  exit 1
+fi
+
+DEPLOYMENT_SCRIPT=/tmp/test-script-deployment.bash.$$
+wget -O $DEPLOYMENT_SCRIPT -q $URL
+
+if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
+  echo "exit 1" > $DEPLOYMENT_SCRIPT
+fi
+
+function upload ()
+{
+  try=$1
+  LOG_FILE=$2
+  t=`date '+%Y%m%d%H%S'`
+  mv $LOG_FILE ${LOG_FILE}.$t
+  # just to be sure flush all disk operations before uploading
+  flush
+  curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
+}
+
+try=1
+while true; do
+  echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
+  export TEST_YML_PATH
+  /bin/bash $DEPLOYMENT_SCRIPT >> $LOG_FILE 2>&1
+  result=$?
+  if [ $result == 0 ] ; then
+    echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
+    break
+  fi
+  if (( try > TRIES )) ; then
+    echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
+    break
+  fi
+  # wait WAITTIME before checking the state
+  echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
+  upload $try $LOG_FILE
+  sleep $WAITTIME
+  ((try++))
+done
+
+exit $result

software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2

+{# Get test type, default to script-from-url, as defined in instance-input-schema.json -#}
+{% set test_type = slapparameter_dict.get('test-type', 'script-from-url') -%}
+{# Choose parameters according to test type -#}
+{% if test_type == 'script-from-url' -%}
+{% set script_url = slapparameter_dict.get('script-to-test-url') -%}
+{% set test_yml_path = '/not/required' -%}
+{% elif test_type == 'cloned-playbook' -%}
+{% set script_url = 'http://10.0.2.100/standalone-local-playbook' -%}
+{% set test_yml_path = slapparameter_dict.get('yml-path-to-test') -%}
+{% else -%}
+The test_type = "{{ test_type }}" is unsupported.
+{% endif -%}
+[buildout]
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+parts =
+  request-kvm
+
+[request-kvm]
+<= slap-connection
+
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = kvm
+name = test-kvm
+sla-computer_guid = ${slap-connection:computer-id}
+
+# Tested image
+# Passed by request
+config-virtual-hard-drive-url = {{ slapparameter_dict.get('image-to-test-url') }}
+config-virtual-hard-drive-md5sum = {{ slapparameter_dict.get('image-to-test-md5sum') }}
+
+# The test script
+config-bootstrap-script-url = {{ in_vm_test_script }}#{{ in_vm_test_script_md5 }}
+
+# Script configuration
+config-data-to-vm =
+  URL={{ script_url }}
+  WAITTIME={{ waittime }}
+  TRIES={{ tries }}
+  TEST_YML_PATH={{ test_yml_path }}
+
+# require HTTP server
+config-enable-http-server = true
+
+# VM options
+config-ram-size = 2048
+config-cpu-count = 2
+
+# await for system to be ready
+return =
+  url
+  backend-url
+  ipv6
+  nat-rule-port-443
+  nat-rule-port-80
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc/
+var = ${:home}/var/
+srv = ${:home}/srv/
+bin = ${:home}/bin/
+tmp = ${:home}/tmp/
+log = ${:var}/log/
+services = ${:etc}/service/
+scripts = ${:etc}/run/

software/erp5testnode/testsuite/deploy-test/instance-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+
+  "title": "Input Parameters",
+  "properties": {
+    "image-to-test-url": {
+      "title": "Image To Test (URL)",
+      "description": "Absolute URL to QCOW2 vm-bootstrap compatible image.",
+      "type": "string"
+    },
+    "image-to-test-md5sum": {
+      "title": "MD5 checksum of Image To Test.",
+      "description": "MD5 checksum of QCOW2 vm-bootstrap compatible image.",
+      "type": "string"
+    },
+    "test-relative-directory": {
+      "title": "Relative directory where tests are found.",
+      "description": "The base of this directory is [slapos-package].",
+      "type": "string"
+    },
+    "test-type": {
+      "title": "Type of the test.",
+      "type": "string",
+      "default": "script-from-url",
+      "enum" : ["script-from-url", "cloned-playbook"]
+    },
+    "script-to-test-url": {
+      "title": "Optional URL of script to test, used for test-type=script-from-url.",
+      "description": "Optional URL of script to test, which will be injected into the VM.\nThis parameter will NOT use local git clone of [slapos-package] part, it will test fully provided script.",
+      "type": "string"
+    },
+    "yml-path-to-test": {
+      "title": "Optional YML path to test from the playbook, used for test-type=cloned-playbook.",
+      "description": "Optional YML path to test from the playbook, provided as relative path to playbook root.\nThis parameter will use local git clone of [slapos-package] part, from which playbook will be constructed.",
+      "type": "string"
+    }
+  },
+  "required": ["image-to-test-url", "image-to-test-md5sum", "test-type"]
+}

software/erp5testnode/testsuite/deploy-test/instance.cfg.in

+[buildout]
+extends = ${template:output}
+parts +=
+  runTestSuite
+  copy-test-data
+
+[copy-test-data]
+# Note: This is a hack, as such while requesting KVM it is not possible
+#       to fill correctly http directory
+# Note: This will only work inside of non-user related environment
+#       dest shell variable will evaluate to requested partition's
+#       srv/public directory
+#       The "if" statement makes this script be run only in partition
+#       *without* srv/public - so according to current knowledge, the
+#       one which requests KVM backend for testing
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command =
+  if [ ! -d srv/public ] ; then
+    dest=`echo ../*/srv/public`
+    cp ${playbook:output} $dest/playbook.tar.gz &&
+    cp ${standalone-local-playbook:location} $dest/standalone-local-playbook
+  fi
+
+[runTestSuite]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
+template = inline:
+  #!/bin/sh
+  export PATH=${python-with-eggs:location}:$PATH
+  exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" "$@"
+mode = 0755
+context =
+    key slapparameter_dict slap-configuration:configuration
+    key partition_ipv4 slap-configuration:ipv4
+
+[switch_softwaretype]
+default = $${:deploy-test}
+deploy-test = $${dynamic-template-deploy-test:rendered}
+
+[dynamic-template-deploy-test]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
+rendered = $${buildout:directory}/template-deploy-test.cfg
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw bin_directory ${buildout:bin-directory}
+    raw in_vm_test_script ${deploy-script-controller-script:location}
+    raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
+    raw waittime ${deploy-script-controller-script:waittime}
+    raw tries ${deploy-script-controller-script:tries}
+mode = 0644

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

+from __future__ import print_function
+import argparse
+import os
+import glob
+from time import gmtime, strftime, time, sleep
+from erp5.util import taskdistribution, testsuite
+import logging
+import sys
+import tempfile
+import json
+
+
+SLEEP_TIME = 10
+TRY_AMOUNT = 3600
+
+
+def waitForSite(partition_path):
+  status_dict = {'command': 'file not found'}
+
+  # find test result, wait 10h
+  try_num = 1
+  start = time()
+  result_found = False
+  while 1:
+    finished = False
+    try_info = 'Try %s/%s: ' % (try_num, TRY_AMOUNT)
+    test_result_glob = os.path.join(
+        partition_path,
+        '..',
+        '*',
+        'srv',
+        'public',
+        'test-script-result',
+    )
+    print(try_info + 'Waiting for data in %r.' % (test_result_glob,))
+    result_list = glob.glob(test_result_glob)
+    if len(result_list) > 0:
+      result_path = result_list[0]
+      print(try_info + 'Data directory %r found, looking for results.' % (
+            result_path,))
+      result_file_list = list((
+          os.path.join(dirname, filename)
+          for dirname, dirnames, filenames in os.walk(result_path)
+          for filename in filenames
+      ))
+      if len(result_file_list):
+        print(try_info + 'No result posted, will check next try.')
+      for result_file in result_file_list:
+        print(try_info + 'Data found.')
+        result_found = True
+        result_file = os.path.abspath(result_file)
+        status_dict['command'] = result_file
+        result = open(result_file).read()
+        # remove result, as it is not required anymore
+        os.unlink(result_file)
+        print(try_info + 'Analysis of result %r:' % (result_file,))
+        print(try_info + result)
+        status_dict['stderr'] = 'Last result:\n%s' % (result,)
+        if 'FATAL: all hosts have already failed -- aborting' in result:
+          # failed
+          status_dict.update(
+              success=False
+          )
+          finished = False
+          status_dict['stdout'] = try_info + 'Build not yet successful.'
+          print(try_info + '%r: Found not yet finished run.' % (result_file,))
+        elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
+          # success
+          status_dict.update(
+              success=True
+          )
+          finished = True
+          print(try_info + '%r: Found finished successful run.' % (
+              result_file,))
+          status_dict['stdout'] = try_info + 'Build successful.'
+          break
+        else:
+          # unknown
+          status_dict.update(
+              success=False
+
+          )
+          status_dict['stdout'] = \
+              try_info + 'Cannot find success nor failure result in the output'
+          print(try_info + '%r: Found unknown run.' % (result_file,))
+          finished = False
+    if finished:
+      break
+    if try_num >= TRY_AMOUNT:
+      msg = try_info + 'Time exceeded, success not found.'
+      print(msg)
+      status_dict.setdefault('stdout', '')
+      status_dict['stdout'] = '\n'.join([status_dict['stdout'], msg])
+      break
+    try_num += 1
+    print(try_info + 'Sleeping for %ss.' % (SLEEP_TIME,))
+    sleep(SLEEP_TIME)
+  if not result_found:
+    status_dict['stdout'] = try_info + 'Test timed out and no result found.'
+    status_dict.update(
+        success=False
+
+    )
+  end = time()
+  status_dict.update(
+      date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
+      duration=end - start,
+  )
+  return status_dict
+
+
+def main():
+  logger = logging.getLogger()
+  logger.addHandler(logging.StreamHandler(sys.stdout))
+  logger.setLevel(logging.DEBUG)
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', type=int,
+                      help='Number of CPUs to use for the VM')
+  parser.add_argument('--master_url',
+                      help='The Url of Master controlling test suites')
+  # SlapOS and deploy test specific
+  parser.add_argument(
+      '--partition_path',
+      help="Path of a partition",
+      default=os.path.abspath(os.getcwd()))
+  parser.add_argument(
+      '--test_reference',
+      help="Reference of the test",
+      default="missing"
+  )
+  parser.add_argument(
+      '--partition_ipv4',
+      help="IPv4 of a partition"
+  )
+  parser.add_argument(
+      '--test_location',
+      help="Location of the tests"
+  )
+
+  args = parser.parse_args()
+
+  revision = args.revision
+  test_suite_title = args.test_suite_title or args.test_suite
+  os.environ['SOURCE_CODE_TO_TEST'] = args.test_location
+  suite = testsuite.EggTestSuite(
+      1, test_suite=args.test_suite, node_quantity=args.node_quantity,
+      revision=revision)
+  access_url_http = None
+  access_url_https = None
+  if args.partition_ipv4:
+    access_url_http = 'http://%s:10080' % (args.partition_ipv4,)
+    access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
+    os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
+    os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
+  tool = taskdistribution.TaskDistributionTool(
+      args.master_url,
+      logger=logger)
+  test_result = tool.createTestResult(
+      revision, suite.getTestList(), args.test_node_title,
+      suite.allow_restart, test_suite_title, args.project_title)
+  if test_result is None:
+    return
+
+  # Create the site
+  status_dict = waitForSite(args.partition_path)
+
+  status_file = tempfile.NamedTemporaryFile()
+  status_file.write(json.dumps(status_dict))
+  status_file.flush()
+  os.fsync(status_file.fileno())
+  os.environ['TEST_SITE_STATUS_JSON'] = status_file.name
+
+  assert revision == test_result.revision, (revision, test_result.revision)
+  while suite.acquire():
+    test = test_result.start(suite.running.keys())
+    if test is not None:
+      suite.start(test.name, lambda status_dict,
+                  __test=test: __test.stop(**status_dict))
+    elif not suite.running:
+      break
+  return
+
+if __name__ == "__main__":
+    main()

software/erp5testnode/testsuite/deploy-test/software.cfg

+[buildout]
+extends = ../../../kvm/software.cfg
+
+parts =
+  eggs
+  template-deploy-test
+  runTestSuite_py
+  playbook
+
+[runTestSuite_py]
+recipe = zc.recipe.egg
+eggs =
+  erp5.util
+interpreter = ${:_buildout_section_name_}
+
+[python-with-eggs]
+recipe = plone.recipe.command
+
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+command =
+  rm -fr ${:location} &&
+  mkdir -p ${:location} &&
+  ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python &&
+  ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python2.7
+update-command = ${:command}
+
+[playbook]
+recipe = plone.recipe.command
+
+stop-on-error = true
+environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+output = ${:location}/playbook.tar.gz
+command =
+  ${:environment}
+  rm -fr ${:location}
+  mkdir -p ${:location}
+  cd ${slapos-package:location}/playbook
+  tar czf ${:output} .
+update-command = ${:command}
+
+[test-location]
+base = ${slapos-package:location}
+
+[slapos-package]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/slapos.package.git
+branch = cdn-test
+git-executable = ${git:location}/bin/git
+
+[template]
+output = ${buildout:directory}/template-original.kvm.cfg
+
+[deploy-script-controller-script]
+filename = deploy-script-controller
+location = ${:_profile_base_location_}/${:filename}
+md5sum = d2b92f45257a52e5a7ff5c311d46d4ae
+# configuration
+waittime = 360
+tries = 80
+
+[standalone-local-playbook]
+filename = standalone-local-playbook
+location = ${:_profile_base_location_}/${:filename}
+
+[template-deploy-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[versions]
+erp5.util = 0.4.49

software/erp5testnode/testsuite/deploy-test/software.cfg.json

+{
+    "name": "Deploy Test",
+    "description": "Deploy Testing software release",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Standard ERP5TestNode hooked",
+            "request": "instance-input-schema.json",
+            "index": 0
+        }
+    }
+}

software/erp5testnode/testsuite/deploy-test/standalone-local-playbook

+#!/bin/bash
+
+set -e
+
+#
+# Paste content of function-common
+# https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/roles/install-script/files/function-common
+#
+#!/bin/bash
+#
+# functions-common - Common functions used by DevStack components
+#
+# The canonical copy of this file is maintained in the DevStack repo.
+# All modifications should be made there and then sync'ed to other repos
+# as required.
+#
+# This file is sorted alphabetically within the function groups.
+#
+# - Config Functions
+# - Control Functions
+# - Distro Functions
+# - Git Functions
+# - OpenStack Functions
+# - Package Functions
+# - Process Functions
+# - Service Functions
+# - System Functions
+#
+# The following variables are assumed to be defined by certain functions:
+#
+# - ``ENABLED_SERVICES``
+# - ``ERROR_ON_CLONE``
+# - ``FILES``
+# - ``OFFLINE``
+# - ``RECLONE``
+# - ``REQUIREMENTS_DIR``
+# - ``STACK_USER``
+# - ``TRACK_DEPENDS``
+# - ``UNDO_REQUIREMENTS``
+# - ``http_proxy``, ``https_proxy``, ``no_proxy``
+#
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set +o xtrace
+
+# Global Config Variables
+declare -A GITREPO
+declare -A GITBRANCH
+declare -A GITDIR
+
+TRACK_DEPENDS=${TRACK_DEPENDS:-False}
+
+
+# Normalize config values to True or False
+# Accepts as False: 0 no No NO false False FALSE
+# Accepts as True: 1 yes Yes YES true True TRUE
+# VAR=$(trueorfalse default-value test-value)
+function trueorfalse {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local default=$1
+    local literal=$2
+    local testval=${!literal:-}
+
+    [[ -z "$testval" ]] && { echo "$default"; return; }
+    [[ "0 no No NO false False FALSE" =~ "$testval" ]] && { echo "False"; return; }
+    [[ "1 yes Yes YES true True TRUE" =~ "$testval" ]] && { echo "True"; return; }
+    echo "$default"
+    $xtrace
+}
+
+#function isset {
+#    [[ -v "$1" ]]
+#}
+#
+# Control Functions
+# =================
+
+# Prints backtrace info
+# filename:lineno:function
+# backtrace level
+function backtrace {
+    local level=$1
+    local deep=$((${#BASH_SOURCE[@]} - 1))
+    echo "[Call Trace]"
+    while [ $level -le $deep ]; do
+        echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
+        deep=$((deep - 1))
+    done
+}
+
+# Prints line number and "message" then exits
+# die $LINENO "message"
+function die {
+    local exitcode=$?
+    set +o xtrace
+    local line=$1; shift
+    if [ $exitcode == 0 ]; then
+        exitcode=1
+    fi
+    backtrace 2
+    err $line "$*"
+    # Give buffers a second to flush
+    sleep 1
+    exit $exitcode
+}
+
+# Checks an environment variable is not set or has length 0 OR if the
+# exit code is non-zero and prints "message" and exits
+# NOTE: env-var is the variable name without a '$'
+# die_if_not_set $LINENO env-var "message"
+function die_if_not_set {
+    local exitcode=$?
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local line=$1; shift
+    local evar=$1; shift
+    if ! is_set $evar || [ $exitcode != 0 ]; then
+        die $line "$*"
+    fi
+    $xtrace
+}
+
+# Prints line number and "message" in error format
+# err $LINENO "message"
+function err {
+    local exitcode=$?
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
+    echo $msg 1>&2;
+    if [[ -n ${LOGDIR} ]]; then
+        echo $msg >> "${LOGDIR}/error.log"
+    fi
+    $xtrace
+    return $exitcode
+}
+
+# Checks an environment variable is not set or has length 0 OR if the
+# exit code is non-zero and prints "message"
+# NOTE: env-var is the variable name without a '$'
+# err_if_not_set $LINENO env-var "message"
+function err_if_not_set {
+    local exitcode=$?
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local line=$1; shift
+    local evar=$1; shift
+    if ! is_set $evar || [ $exitcode != 0 ]; then
+        err $line "$*"
+    fi
+    $xtrace
+    return $exitcode
+}
+
+# Exit after outputting a message about the distribution not being supported.
+# exit_distro_not_supported [optional-string-telling-what-is-missing]
+function exit_distro_not_supported {
+    if [[ -z "$DISTRO" ]]; then
+        GetDistro
+    fi
+
+    if [ $# -gt 0 ]; then
+        die $LINENO "Support for $DISTRO is incomplete: no support for $@"
+    else
+        die $LINENO "Support for $DISTRO is incomplete."
+    fi
+}
+
+# Test if the named environment variable is set and not zero length
+# is_set env-var
+function is_set {
+    local var=\$"$1"
+    eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this
+}
+
+# Prints line number and "message" in warning format
+# warn $LINENO "message"
+function warn {
+    local exitcode=$?
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
+    echo $msg 1>&2;
+    if [[ -n ${LOGDIR} ]]; then
+        echo $msg >> "${LOGDIR}/error.log"
+    fi
+    $xtrace
+    return $exitcode
+}
+
+
+# Distro Functions
+# ================
+
+# Determine OS Vendor, Release and Update
+# Tested with OS/X, Ubuntu, RedHat, CentOS, Fedora
+# Returns results in global variables:
+# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
+# ``os_RELEASE`` - major release: ``14.04`` (Ubuntu), ``20`` (Fedora)
+# ``os_UPDATE`` - update: ex. the ``5`` in ``RHEL6.5``
+# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
+# ``os_CODENAME`` - vendor's codename for release: ``snow leopard``, ``trusty``
+os_VENDOR=""
+os_RELEASE=""
+os_UPDATE=""
+os_PACKAGE=""
+os_CODENAME=""
+
+# GetOSVersion
+function GetOSVersion {
+
+    # Figure out which vendor we are
+    if [[ -x "`which sw_vers 2>/dev/null`" ]]; then
+        # OS/X
+        os_VENDOR=`sw_vers -productName`
+        os_RELEASE=`sw_vers -productVersion`
+        os_UPDATE=${os_RELEASE##*.}
+        os_RELEASE=${os_RELEASE%.*}
+        os_PACKAGE=""
+        if [[ "$os_RELEASE" =~ "10.7" ]]; then
+            os_CODENAME="lion"
+        elif [[ "$os_RELEASE" =~ "10.6" ]]; then
+            os_CODENAME="snow leopard"
+        elif [[ "$os_RELEASE" =~ "10.5" ]]; then
+            os_CODENAME="leopard"
+        elif [[ "$os_RELEASE" =~ "10.4" ]]; then
+            os_CODENAME="tiger"
+        elif [[ "$os_RELEASE" =~ "10.3" ]]; then
+            os_CODENAME="panther"
+        else
+            os_CODENAME=""
+        fi
+    elif [[ -x $(which lsb_release 2>/dev/null) ]]; then
+        os_VENDOR=$(lsb_release -i -s)
+        os_RELEASE=$(lsb_release -r -s)
+        os_UPDATE=""
+        os_PACKAGE="rpm"
+        if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
+            os_PACKAGE="deb"
+        elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
+            lsb_release -d -s | grep -q openSUSE
+            if [[ $? -eq 0 ]]; then
+                os_VENDOR="openSUSE"
+            fi
+        elif [[ $os_VENDOR == "openSUSE project" ]]; then
+            os_VENDOR="openSUSE"
+        elif [[ $os_VENDOR =~ Red.*Hat ]]; then
+            os_VENDOR="Red Hat"
+        fi
+        os_CODENAME=$(lsb_release -c -s)
+    elif [[ -r /etc/redhat-release ]]; then
+        # Red Hat Enterprise Linux Server release 5.5 (Tikanga)
+        # Red Hat Enterprise Linux Server release 7.0 Beta (Maipo)
+        # CentOS release 5.5 (Final)
+        # CentOS Linux release 6.0 (Final)
+        # Fedora release 16 (Verne)
+        # XenServer release 6.2.0-70446c (xenenterprise)
+        # Oracle Linux release 7
+        os_CODENAME=""
+        for r in "Red Hat" CentOS Fedora XenServer; do
+            os_VENDOR=$r
+            if [[ -n "`grep \"$r\" /etc/redhat-release`" ]]; then
+                ver=`sed -e 's/^.* \([0-9].*\) (\(.*\)).*$/\1\|\2/' /etc/redhat-release`
+                os_CODENAME=${ver#*|}
+                os_RELEASE=${ver%|*}
+                os_UPDATE=${os_RELEASE##*.}
+                os_RELEASE=${os_RELEASE%.*}
+                break
+            fi
+            os_VENDOR=""
+        done
+        if [ "$os_VENDOR" = "Red Hat" ] && [[ -r /etc/oracle-release ]]; then
+            os_VENDOR=OracleLinux
+        fi
+        os_PACKAGE="rpm"
+    elif [[ -r /etc/SuSE-release ]]; then
+        for r in openSUSE "SUSE Linux"; do
+            if [[ "$r" = "SUSE Linux" ]]; then
+                os_VENDOR="SUSE LINUX"
+            else
+                os_VENDOR=$r
+            fi
+
+            if [[ -n "`grep \"$r\" /etc/SuSE-release`" ]]; then
+                os_CODENAME=`grep "CODENAME = " /etc/SuSE-release | sed 's:.* = ::g'`
+                os_RELEASE=`grep "VERSION = " /etc/SuSE-release | sed 's:.* = ::g'`
+                os_UPDATE=`grep "PATCHLEVEL = " /etc/SuSE-release | sed 's:.* = ::g'`
+                break
+            fi
+            os_VENDOR=""
+        done
+        os_PACKAGE="rpm"
+    # If lsb_release is not installed, we should be able to detect Debian OS
+    elif [[ -f /etc/debian_version ]] && [[ $(cat /proc/version) =~ "Debian" ]]; then
+        os_VENDOR="Debian"
+        os_PACKAGE="deb"
+        os_CODENAME=$(awk '/VERSION=/' /etc/os-release | sed 's/VERSION=//' | sed -r 's/\"|\(|\)//g' | awk '{print $2}')
+        os_RELEASE=$(awk '/VERSION_ID=/' /etc/os-release | sed 's/VERSION_ID=//' | sed 's/\"//g')
+    fi
+    export os_VENDOR os_RELEASE os_UPDATE os_PACKAGE os_CODENAME
+}
+
+# Translate the OS version values into common nomenclature
+# Sets global ``DISTRO`` from the ``os_*`` values
+declare DISTRO
+
+function GetDistro {
+    GetOSVersion
+    if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+        # 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
+        DISTRO=$os_CODENAME
+    elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
+        # For Fedora, just use 'f' and the release
+        DISTRO="f$os_RELEASE"
+    elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
+        DISTRO="opensuse-$os_RELEASE"
+    elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
+        # For SLE, also use the service pack
+        if [[ -z "$os_UPDATE" ]]; then
+            DISTRO="sle${os_RELEASE}"
+        else
+            DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
+        fi
+    elif [[ "$os_VENDOR" =~ (Red Hat) || \
+        "$os_VENDOR" =~ (CentOS) || \
+        "$os_VENDOR" =~ (OracleLinux) ]]; then
+        # Drop the . release as we assume it's compatible
+        DISTRO="rhel${os_RELEASE::1}"
+    elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
+        DISTRO="xs$os_RELEASE"
+    else
+        # Catch-all for now is Vendor + Release + Update
+        DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
+    fi
+    export DISTRO
+}
+
+# Utility function for checking machine architecture
+# is_arch arch-type
+function is_arch {
+    [[ "$(uname -m)" == "$1" ]]
+}
+
+# Determine if current distribution is an Oracle distribution
+# is_oraclelinux
+function is_oraclelinux {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [ "$os_VENDOR" = "OracleLinux" ]
+}
+
+
+# Determine if current distribution is a Fedora-based distribution
+# (Fedora, RHEL, CentOS, etc).
+# is_fedora
+function is_fedora {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
+        [ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleLinux" ]
+}
+
+
+# Determine if current distribution is a SUSE-based distribution
+# (openSUSE, SLE).
+# is_suse
+function is_suse {
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    [ "$os_VENDOR" = "openSUSE" ] || [ "$os_VENDOR" = "SUSE LINUX" ]
+}
+
+
+# Determine if current distribution is an Ubuntu-based distribution
+# It will also detect non-Ubuntu but Debian-based distros
+# is_ubuntu
+function is_ubuntu {
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+    [ "$os_PACKAGE" = "deb" ]
+}
+
+
+# Git Functions
+# =============
+
+# Returns openstack release name for a given branch name
+# ``get_release_name_from_branch branch-name``
+function get_release_name_from_branch {
+    local branch=$1
+    if [[ $branch =~ "stable/" || $branch =~ "proposed/" ]]; then
+        echo ${branch#*/}
+    else
+        echo "master"
+    fi
+}
+
+# git clone only if directory doesn't exist already.  Since ``DEST`` might not
+# be owned by the installation user, we create the directory and change the
+# ownership to the proper user.
+# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
+# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
+# does not exist (default is False, meaning the repo will be cloned).
+# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
+# git_clone remote dest-dir branch
+function git_clone {
+    local git_remote=$1
+    local git_dest=$2
+    local git_ref=$3
+    local orig_dir=$(pwd)
+    local git_clone_flags=""
+
+    RECLONE=$(trueorfalse False RECLONE)
+    if [[ "${GIT_DEPTH}" -gt 0 ]]; then
+        git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
+    fi
+
+    if [[ "$OFFLINE" = "True" ]]; then
+        echo "Running in offline mode, clones already exist"
+        # print out the results so we know what change was used in the logs
+        cd $git_dest
+        git show --oneline | head -1
+        cd $orig_dir
+        return
+    fi
+
+    if echo $git_ref | egrep -q "^refs"; then
+        # If our branch name is a gerrit style refs/changes/...
+        if [[ ! -d $git_dest ]]; then
+            [[ "$ERROR_ON_CLONE" = "True" ]] && \
+                die $LINENO "Cloning not allowed in this configuration"
+            git_timed clone $git_clone_flags $git_remote $git_dest
+        fi
+        cd $git_dest
+        git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
+    else
+        # do a full clone only if the directory doesn't exist
+        if [[ ! -d $git_dest ]]; then
+            [[ "$ERROR_ON_CLONE" = "True" ]] && \
+                die $LINENO "Cloning not allowed in this configuration"
+            git_timed clone $git_clone_flags $git_remote $git_dest
+            cd $git_dest
+            # This checkout syntax works for both branches and tags
+            git checkout $git_ref
+        elif [[ "$RECLONE" = "True" ]]; then
+            # if it does exist then simulate what clone does if asked to RECLONE
+            cd $git_dest
+            # set the url to pull from and fetch
+            git remote set-url origin $git_remote
+            git_timed fetch origin
+            # remove the existing ignored files (like pyc) as they cause breakage
+            # (due to the py files having older timestamps than our pyc, so python
+            # thinks the pyc files are correct using them)
+            find $git_dest -name '*.pyc' -delete
+
+            # handle git_ref accordingly to type (tag, branch)
+            if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
+                git_update_tag $git_ref
+            elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
+                git_update_branch $git_ref
+            elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
+                git_update_remote_branch $git_ref
+            else
+                die $LINENO "$git_ref is neither branch nor tag"
+            fi
+
+        fi
+    fi
+
+    # print out the results so we know what change was used in the logs
+    cd $git_dest
+    git show --oneline | head -1
+    cd $orig_dir
+}
+
+# A variation on git clone that lets us specify a project by it's
+# actual name, like oslo.config. This is exceptionally useful in the
+# library installation case
+function git_clone_by_name {
+    local name=$1
+    local repo=${GITREPO[$name]}
+    local dir=${GITDIR[$name]}
+    local branch=${GITBRANCH[$name]}
+    git_clone $repo $dir $branch
+}
+
+
+# git can sometimes get itself infinitely stuck with transient network
+# errors or other issues with the remote end.  This wraps git in a
+# timeout/retry loop and is intended to watch over non-local git
+# processes that might hang.  GIT_TIMEOUT, if set, is passed directly
+# to timeout(1); otherwise the default value of 0 maintains the status
+# quo of waiting forever.
+# usage: git_timed <git-command>
+function git_timed {
+    local count=0
+    local timeout=0
+
+    if [[ -n "${GIT_TIMEOUT}" ]]; then
+        timeout=${GIT_TIMEOUT}
+    fi
+
+    until timeout -s SIGINT ${timeout} git "$@"; do
+        # 124 is timeout(1)'s special return code when it reached the
+        # timeout; otherwise assume fatal failure
+        if [[ $? -ne 124 ]]; then
+            die $LINENO "git call failed: [git $@]"
+        fi
+
+        count=$(($count + 1))
+        warn "timeout ${count} for git call: [git $@]"
+        if [ $count -eq 3 ]; then
+            die $LINENO "Maximum of 3 git retries reached"
+        fi
+        sleep 5
+    done
+}
+
+# git update using reference as a branch.
+# git_update_branch ref
+function git_update_branch {
+    local git_branch=$1
+
+    git checkout -f origin/$git_branch
+    # a local branch might not exist
+    git branch -D $git_branch || true
+    git checkout -b $git_branch
+}
+
+# git update using reference as a branch.
+# git_update_remote_branch ref
+function git_update_remote_branch {
+    local git_branch=$1
+
+    git checkout -b $git_branch -t origin/$git_branch
+}
+
+# git update using reference as a tag. Be careful editing source at that repo
+# as working copy will be in a detached mode
+# git_update_tag ref
+function git_update_tag {
+    local git_tag=$1
+
+    git tag -d $git_tag
+    # fetching given tag only
+    git_timed fetch origin tag $git_tag
+    git checkout -f $git_tag
+}
+
+
+# OpenStack Functions
+# ===================
+
+# Get the default value for HOST_IP
+# get_default_host_ip fixed_range floating_range host_ip_iface host_ip
+function get_default_host_ip {
+    local fixed_range=$1
+    local floating_range=$2
+    local host_ip_iface=$3
+    local host_ip=$4
+
+    # Search for an IP unless an explicit is set by ``HOST_IP`` environment variable
+    if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
+        host_ip=""
+        # Find the interface used for the default route
+        host_ip_iface=${host_ip_iface:-$(ip route | awk '/default/ {print $5}' | head -1)}
+        local host_ips=$(LC_ALL=C ip -f inet addr show ${host_ip_iface} | awk '/inet/ {split($2,parts,"/");  print parts[1]}')
+        local ip
+        for ip in $host_ips; do
+            # Attempt to filter out IP addresses that are part of the fixed and
+            # floating range. Note that this method only works if the ``netaddr``
+            # python library is installed. If it is not installed, an error
+            # will be printed and the first IP from the interface will be used.
+            # If that is not correct set ``HOST_IP`` in ``localrc`` to the correct
+            # address.
+            if ! (address_in_net $ip $fixed_range || address_in_net $ip $floating_range); then
+                host_ip=$ip
+                break;
+            fi
+        done
+    fi
+    echo $host_ip
+}
+
+# Generates hex string from ``size`` byte of pseudo random data
+# generate_hex_string size
+function generate_hex_string {
+    local size=$1
+    hexdump -n "$size" -v -e '/1 "%02x"' /dev/urandom
+}
+
+# Grab a numbered field from python prettytable output
+# Fields are numbered starting with 1
+# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
+# get_field field-number
+function get_field {
+    local data field
+    while read data; do
+        if [ "$1" -lt 0 ]; then
+            field="(\$(NF$1))"
+        else
+            field="\$$(($1 + 1))"
+        fi
+        echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
+    done
+}
+
+# install default policy
+# copy over a default policy.json and policy.d for projects
+function install_default_policy {
+    local project=$1
+    local project_uc=$(echo $1|tr a-z A-Z)
+    local conf_dir="${project_uc}_CONF_DIR"
+    # eval conf dir to get the variable
+    conf_dir="${!conf_dir}"
+    local project_dir="${project_uc}_DIR"
+    # eval project dir to get the variable
+    project_dir="${!project_dir}"
+    local sample_conf_dir="${project_dir}/etc/${project}"
+    local sample_policy_dir="${project_dir}/etc/${project}/policy.d"
+
+    # first copy any policy.json
+    cp -p $sample_conf_dir/policy.json $conf_dir
+    # then optionally copy over policy.d
+    if [[ -d $sample_policy_dir ]]; then
+        cp -r $sample_policy_dir $conf_dir/policy.d
+    fi
+}
+
+# Add a policy to a policy.json file
+# Do nothing if the policy already exists
+# ``policy_add policy_file policy_name policy_permissions``
+function policy_add {
+    local policy_file=$1
+    local policy_name=$2
+    local policy_perm=$3
+
+    if grep -q ${policy_name} ${policy_file}; then
+        echo "Policy ${policy_name} already exists in ${policy_file}"
+        return
+    fi
+
+    # Add a terminating comma to policy lines without one
+    # Remove the closing '}' and all lines following to the end-of-file
+    local tmpfile=$(mktemp)
+    uniq ${policy_file} | sed -e '
+        s/]$/],/
+        /^[}]/,$d
+    ' > ${tmpfile}
+
+    # Append policy and closing brace
+    echo "    \"${policy_name}\": ${policy_perm}" >>${tmpfile}
+    echo "}" >>${tmpfile}
+
+    mv ${tmpfile} ${policy_file}
+}
+
+# Gets or creates a domain
+# Usage: get_or_create_domain <name> <description>
+function get_or_create_domain {
+    local os_url="$KEYSTONE_SERVICE_URI_V3"
+    # Gets domain id
+    local domain_id=$(
+        # Gets domain id
+        openstack --os-token=$OS_TOKEN --os-url=$os_url \
+            --os-identity-api-version=3 domain show $1 \
+            -f value -c id 2>/dev/null ||
+        # Creates new domain
+        openstack --os-token=$OS_TOKEN --os-url=$os_url \
+            --os-identity-api-version=3 domain create $1 \
+            --description "$2" \
+            -f value -c id
+    )
+    echo $domain_id
+}
+
+# Gets or creates group
+# Usage: get_or_create_group <groupname> [<domain> <description>]
+function get_or_create_group {
+    local domain=${2:+--domain ${2}}
+    local desc="${3:-}"
+    local os_url="$KEYSTONE_SERVICE_URI_V3"
+    # Gets group id
+    local group_id=$(
+        # Creates new group with --or-show
+        openstack --os-token=$OS_TOKEN --os-url=$os_url \
+            --os-identity-api-version=3 group create $1 \
+            $domain --description "$desc" --or-show \
+            -f value -c id
+    )
+    echo $group_id
+}
+
+# Gets or creates user
+# Usage: get_or_create_user <username> <password> [<email> [<domain>]]
+function get_or_create_user {
+    if [[ ! -z "$3" ]]; then
+        local email="--email=$3"
+    else
+        local email=""
+    fi
+    local os_cmd="openstack"
+    local domain=""
+    if [[ ! -z "$4" ]]; then
+        domain="--domain=$4"
+        os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
+    fi
+    # Gets user id
+    local user_id=$(
+        # Creates new user with --or-show
+        $os_cmd user create \
+            $1 \
+            --password "$2" \
+            $email \
+            $domain \
+            --or-show \
+            -f value -c id
+    )
+    echo $user_id
+}
+
+# Gets or creates project
+# Usage: get_or_create_project <name> [<domain>]
+function get_or_create_project {
+    # Gets project id
+    local os_cmd="openstack"
+    local domain=""
+    if [[ ! -z "$2" ]]; then
+        domain="--domain=$2"
+        os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
+    fi
+    local project_id=$(
+        # Creates new project with --or-show
+        $os_cmd project create $1 $domain --or-show -f value -c id
+    )
+    echo $project_id
+}
+
+# Gets or creates role
+# Usage: get_or_create_role <name>
+function get_or_create_role {
+    local role_id=$(
+        # Creates role with --or-show
+        openstack role create $1 --or-show -f value -c id
+    )
+    echo $role_id
+}
+
+# Gets or adds user role to project
+# Usage: get_or_add_user_project_role <role> <user> <project>
+function get_or_add_user_project_role {
+    # Gets user role id
+    local user_role_id=$(openstack role list \
+        --user $2 \
+        --project $3 \
+        --column "ID" \
+        --column "Name" \
+        | grep " $1 " | get_field 1)
+    if [[ -z "$user_role_id" ]]; then
+        # Adds role to user
+        user_role_id=$(openstack role add \
+            $1 \
+            --user $2 \
+            --project $3 \
+            | grep " id " | get_field 2)
+    fi
+    echo $user_role_id
+}
+
+# Gets or adds group role to project
+# Usage: get_or_add_group_project_role <role> <group> <project>
+function get_or_add_group_project_role {
+    # Gets group role id
+    local group_role_id=$(openstack role list \
+        --group $2 \
+        --project $3 \
+        --column "ID" \
+        --column "Name" \
+        | grep " $1 " | get_field 1)
+    if [[ -z "$group_role_id" ]]; then
+        # Adds role to group
+        group_role_id=$(openstack role add \
+            $1 \
+            --group $2 \
+            --project $3 \
+            | grep " id " | get_field 2)
+    fi
+    echo $group_role_id
+}
+
+# Gets or creates service
+# Usage: get_or_create_service <name> <type> <description>
+function get_or_create_service {
+    # Gets service id
+    local service_id=$(
+        # Gets service id
+        openstack service show $1 -f value -c id 2>/dev/null ||
+        # Creates new service if not exists
+        openstack service create \
+            $2 \
+            --name $1 \
+            --description="$3" \
+            -f value -c id
+    )
+    echo $service_id
+}
+
+# Gets or creates endpoint
+# Usage: get_or_create_endpoint <service> <region> <publicurl> <adminurl> <internalurl>
+function get_or_create_endpoint {
+    # Gets endpoint id
+    local endpoint_id=$(openstack endpoint list \
+        --column "ID" \
+        --column "Region" \
+        --column "Service Name" \
+        | grep " $2 " \
+        | grep " $1 " | get_field 1)
+    if [[ -z "$endpoint_id" ]]; then
+        # Creates new endpoint
+        endpoint_id=$(openstack endpoint create \
+            $1 \
+            --region $2 \
+            --publicurl $3 \
+            --adminurl $4 \
+            --internalurl $5 \
+            | grep " id " | get_field 2)
+    fi
+    echo $endpoint_id
+}
+
+
+# Package Functions
+# =================
+
+# _get_package_dir
+function _get_package_dir {
+    local base_dir=$1
+    local pkg_dir
+
+    if [[ -z "$base_dir" ]]; then
+        base_dir=$FILES
+    fi
+    if is_ubuntu; then
+        pkg_dir=$base_dir/debs
+    elif is_fedora; then
+        pkg_dir=$base_dir/rpms
+    elif is_suse; then
+        pkg_dir=$base_dir/rpms-suse
+    else
+        exit_distro_not_supported "list of packages"
+    fi
+    echo "$pkg_dir"
+}
+
+# Wrapper for ``apt-get`` to set cache and proxy environment variables
+# Uses globals ``OFFLINE``, ``*_proxy``
+# apt_get operation package [package ...]
+function apt_get {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    [[ "$OFFLINE" = "True" || -z "$@" ]] && return
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+
+    $xtrace
+
+    $sudo DEBIAN_FRONTEND=noninteractive \
+        http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
+        no_proxy=${no_proxy:-} \
+        apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@"
+}
+
+function _parse_package_files {
+    local files_to_parse=$@
+
+    if [[ -z "$DISTRO" ]]; then
+        GetDistro
+    fi
+
+    for fname in ${files_to_parse}; do
+        local OIFS line package distros distro
+        [[ -e $fname ]] || continue
+
+        OIFS=$IFS
+        IFS=$'\n'
+        for line in $(<${fname}); do
+            if [[ $line =~ "NOPRIME" ]]; then
+                continue
+            fi
+
+            # Assume we want this package
+            package=${line%#*}
+            inst_pkg=1
+
+            # Look for # dist:xxx in comment
+            if [[ $line =~ (.*)#.*dist:([^ ]*) ]]; then
+                # We are using BASH regexp matching feature.
+                package=${BASH_REMATCH[1]}
+                distros=${BASH_REMATCH[2]}
+                # In bash ${VAR,,} will lowecase VAR
+                # Look for a match in the distro list
+                if [[ ! ${distros,,} =~ ${DISTRO,,} ]]; then
+                    # If no match then skip this package
+                    inst_pkg=0
+                fi
+            fi
+
+            if [[ $inst_pkg = 1 ]]; then
+                echo $package
+            fi
+        done
+        IFS=$OIFS
+    done
+}
+
+# get_packages() collects a list of package names of any type from the
+# prerequisite files in ``files/{debs|rpms}``.  The list is intended
+# to be passed to a package installer such as apt or yum.
+#
+# Only packages required for the services in 1st argument will be
+# included.  Two bits of metadata are recognized in the prerequisite files:
+#
+# - ``# NOPRIME`` defers installation to be performed later in `stack.sh`
+# - ``# dist:DISTRO`` or ``dist:DISTRO1,DISTRO2`` limits the selection
+#   of the package to the distros listed.  The distro names are case insensitive.
+function get_packages {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local services=$@
+    local package_dir=$(_get_package_dir)
+    local file_to_parse=""
+    local service=""
+
+    if [[ -z "$package_dir" ]]; then
+        echo "No package directory supplied"
+        return 1
+    fi
+    for service in ${services//,/ }; do
+        # Allow individual services to specify dependencies
+        if [[ -e ${package_dir}/${service} ]]; then
+            file_to_parse="${file_to_parse} ${package_dir}/${service}"
+        fi
+        # NOTE(sdague) n-api needs glance for now because that's where
+        # glance client is
+        if [[ $service == n-api ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/nova"
+            fi
+            if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/glance"
+            fi
+        elif [[ $service == c-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/cinder ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/cinder"
+            fi
+        elif [[ $service == ceilometer-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/ceilometer ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/ceilometer"
+            fi
+        elif [[ $service == s-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/swift ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/swift"
+            fi
+        elif [[ $service == n-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/nova"
+            fi
+        elif [[ $service == g-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/glance"
+            fi
+        elif [[ $service == key* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/keystone ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/keystone"
+            fi
+        elif [[ $service == q-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/neutron ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/neutron"
+            fi
+        elif [[ $service == ir-* ]]; then
+            if [[ ! $file_to_parse =~ $package_dir/ironic ]]; then
+                file_to_parse="${file_to_parse} ${package_dir}/ironic"
+            fi
+        fi
+    done
+    echo "$(_parse_package_files $file_to_parse)"
+    $xtrace
+}
+
+# get_plugin_packages() collects a list of package names of any type from a
+# plugin's prerequisite files in ``$PLUGIN/devstack/files/{debs|rpms}``.  The
+# list is intended to be passed to a package installer such as apt or yum.
+#
+# Only packages required for enabled and collected plugins will included.
+#
+# The same metadata used in the main DevStack prerequisite files may be used
+# in these prerequisite files, see get_packages() for more info.
+function get_plugin_packages {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local files_to_parse=""
+    local package_dir=""
+    for plugin in ${DEVSTACK_PLUGINS//,/ }; do
+        local package_dir="$(_get_package_dir ${GITDIR[$plugin]}/devstack/files)"
+        files_to_parse+="$package_dir/$plugin"
+    done
+    echo "$(_parse_package_files $files_to_parse)"
+    $xtrace
+}
+
+# Distro-agnostic package installer
+# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
+# install_package package [package ...]
+function update_package_repo {
+    NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
+    REPOS_UPDATED=${REPOS_UPDATED:-False}
+    RETRY_UPDATE=${RETRY_UPDATE:-False}
+
+    if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
+        return 0
+    fi
+
+    if is_ubuntu; then
+        local xtrace=$(set +o | grep xtrace)
+        set +o xtrace
+        if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
+            # if there are transient errors pulling the updates, that's fine.
+            # It may be secondary repositories that we don't really care about.
+            apt_get update  || /bin/true
+            REPOS_UPDATED=True
+        fi
+        $xtrace
+    fi
+}
+
+function real_install_package {
+    if is_ubuntu; then
+        apt_get install "$@"
+    elif is_fedora; then
+        yum_install "$@"
+    elif is_suse; then
+        zypper_install "$@"
+    else
+        exit_distro_not_supported "installing packages"
+    fi
+}
+
+# Distro-agnostic package installer
+# install_package package [package ...]
+function install_package {
+    update_package_repo
+    real_install_package $@ || RETRY_UPDATE=True update_package_repo && real_install_package $@
+}
+
+# Distro-agnostic function to tell if a package is installed
+# is_package_installed package [package ...]
+function is_package_installed {
+    if [[ -z "$@" ]]; then
+        return 1
+    fi
+
+    if [[ -z "$os_PACKAGE" ]]; then
+        GetOSVersion
+    fi
+    if [[ "$os_PACKAGE" = "deb" ]]; then
+        dpkg -s "$@" > /dev/null 2> /dev/null
+    elif [[ "$os_PACKAGE" = "rpm" ]]; then
+        rpm --quiet -q "$@"
+    else
+        exit_distro_not_supported "finding if a package is installed"
+    fi
+}
+
+# Distro-agnostic package uninstaller
+# uninstall_package package [package ...]
+function uninstall_package {
+    if is_ubuntu; then
+        apt_get purge "$@"
+    elif is_fedora; then
+        sudo ${YUM:-yum} remove -y "$@" ||:
+    elif is_suse; then
+        sudo zypper rm "$@"
+    else
+        exit_distro_not_supported "uninstalling packages"
+    fi
+}
+
+# Wrapper for ``yum`` to set proxy environment variables
+# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
+# yum_install package [package ...]
+function yum_install {
+    [[ "$OFFLINE" = "True" ]] && return
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+
+    # The manual check for missing packages is because yum -y assumes
+    # missing packages are OK.  See
+    # https://bugzilla.redhat.com/show_bug.cgi?id=965567
+    $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
+        no_proxy="${no_proxy:-}" \
+        ${YUM:-yum} install -y "$@" 2>&1 | \
+        awk '
+            BEGIN { fail=0 }
+            /No package/ { fail=1 }
+            { print }
+            END { exit fail }' || \
+                die $LINENO "Missing packages detected"
+
+    # also ensure we catch a yum failure
+    if [[ ${PIPESTATUS[0]} != 0 ]]; then
+        die $LINENO "${YUM:-yum} install failure"
+    fi
+}
+
+# zypper wrapper to set arguments correctly
+# Uses globals ``OFFLINE``, ``*_proxy``
+# zypper_install package [package ...]
+function zypper_install {
+    [[ "$OFFLINE" = "True" ]] && return
+    local sudo="sudo"
+    [[ "$(id -u)" = "0" ]] && sudo="env"
+    $sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
+        no_proxy="${no_proxy:-}" \
+        zypper --non-interactive install --auto-agree-with-licenses "$@"
+}
+
+
+# Process Functions
+# =================
+
+# _run_process() is designed to be backgrounded by run_process() to simulate a
+# fork.  It includes the dirty work of closing extra filehandles and preparing log
+# files to produce the same logs as screen_it().  The log filename is derived
+# from the service name.
+# Uses globals ``CURRENT_LOG_TIME``, ``LOGDIR``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
+# If an optional group is provided sg will be used to set the group of
+# the command.
+# _run_process service "command-line" [group]
+function _run_process {
+    local service=$1
+    local command="$2"
+    local group=$3
+
+    # Undo logging redirections and close the extra descriptors
+    exec 1>&3
+    exec 2>&3
+    exec 3>&-
+    exec 6>&-
+
+    local real_logfile="${LOGDIR}/${service}.log.${CURRENT_LOG_TIME}"
+    if [[ -n ${LOGDIR} ]]; then
+        exec 1>&"$real_logfile" 2>&1
+        ln -sf "$real_logfile" ${LOGDIR}/${service}.log
+        if [[ -n ${SCREEN_LOGDIR} ]]; then
+            # Drop the backward-compat symlink
+            ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${service}.log
+        fi
+
+        # TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
+        export PYTHONUNBUFFERED=1
+    fi
+
+    # Run under ``setsid`` to force the process to become a session and group leader.
+    # The pid saved can be used with pkill -g to get the entire process group.
+    if [[ -n "$group" ]]; then
+        setsid sg $group "$command" & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
+    else
+        setsid $command & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
+    fi
+
+    # Just silently exit this process
+    exit 0
+}
+
+# Helper to remove the ``*.failure`` files under ``$SERVICE_DIR/$SCREEN_NAME``.
+# This is used for ``service_check`` when all the ``screen_it`` are called finished
+# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
+# init_service_check
+function init_service_check {
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+
+    if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
+        mkdir -p "$SERVICE_DIR/$SCREEN_NAME"
+    fi
+
+    rm -f "$SERVICE_DIR/$SCREEN_NAME"/*.failure
+}
+
+# Find out if a process exists by partial name.
+# is_running name
+function is_running {
+    local name=$1
+    ps auxw | grep -v grep | grep ${name} > /dev/null
+    local exitcode=$?
+    # some times I really hate bash reverse binary logic
+    return $exitcode
+}
+
+# Run a single service under screen or directly
+# If the command includes shell metachatacters (;<>*) it must be run using a shell
+# If an optional group is provided sg will be used to run the
+# command as that group.
+# run_process service "command-line" [group]
+function run_process {
+    local service=$1
+    local command="$2"
+    local group=$3
+
+    if is_service_enabled $service; then
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            screen_process "$service" "$command" "$group"
+        else
+            # Spawn directly without screen
+            _run_process "$service" "$command" "$group" &
+        fi
+    fi
+}
+
+# Helper to launch a process in a named screen
+# Uses globals ``CURRENT_LOG_TIME``, ```LOGDIR``, ``SCREEN_LOGDIR``, `SCREEN_NAME``,
+# ``SERVICE_DIR``, ``USE_SCREEN``
+# screen_process name "command-line" [group]
+# Run a command in a shell in a screen window, if an optional group
+# is provided, use sg to set the group of the command.
+function screen_process {
+    local name=$1
+    local command="$2"
+    local group=$3
+
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+    USE_SCREEN=$(trueorfalse True USE_SCREEN)
+
+    screen -S $SCREEN_NAME -X screen -t $name
+
+    local real_logfile="${LOGDIR}/${name}.log.${CURRENT_LOG_TIME}"
+    echo "LOGDIR: $LOGDIR"
+    echo "SCREEN_LOGDIR: $SCREEN_LOGDIR"
+    echo "log: $real_logfile"
+    if [[ -n ${LOGDIR} ]]; then
+        screen -S $SCREEN_NAME -p $name -X logfile "$real_logfile"
+        screen -S $SCREEN_NAME -p $name -X log on
+        ln -sf "$real_logfile" ${LOGDIR}/${name}.log
+        if [[ -n ${SCREEN_LOGDIR} ]]; then
+            # Drop the backward-compat symlink
+            ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${1}.log
+        fi
+    fi
+
+    # sleep to allow bash to be ready to be send the command - we are
+    # creating a new window in screen and then sends characters, so if
+    # bash isn't running by the time we send the command, nothing
+    # happens.  This sleep was added originally to handle gate runs
+    # where we needed this to be at least 3 seconds to pass
+    # consistently on slow clouds. Now this is configurable so that we
+    # can determine a reasonable value for the local case which should
+    # be much smaller.
+    sleep ${SCREEN_SLEEP:-3}
+
+    NL=`echo -ne '\015'`
+    # This fun command does the following:
+    # - the passed server command is backgrounded
+    # - the pid of the background process is saved in the usual place
+    # - the server process is brought back to the foreground
+    # - if the server process exits prematurely the fg command errors
+    # and a message is written to stdout and the process failure file
+    #
+    # The pid saved can be used in stop_process() as a process group
+    # id to kill off all child processes
+    if [[ -n "$group" ]]; then
+        command="sg $group '$command'"
+    fi
+
+    # Append the process to the screen rc file
+    screen_rc "$name" "$command"
+
+    screen -S $SCREEN_NAME -p $name -X stuff "$command & echo \$! >$SERVICE_DIR/$SCREEN_NAME/${name}.pid; fg || echo \"$name failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/${name}.failure\"$NL"
+}
+
+# Screen rc file builder
+# Uses globals ``SCREEN_NAME``, ``SCREENRC``
+# screen_rc service "command-line"
+function screen_rc {
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SCREENRC=$TOP_DIR/$SCREEN_NAME-screenrc
+    if [[ ! -e $SCREENRC ]]; then
+        # Name the screen session
+        echo "sessionname $SCREEN_NAME" > $SCREENRC
+        # Set a reasonable statusbar
+        echo "hardstatus alwayslastline '$SCREEN_HARDSTATUS'" >> $SCREENRC
+        # Some distributions override PROMPT_COMMAND for the screen terminal type - turn that off
+        echo "setenv PROMPT_COMMAND /bin/true" >> $SCREENRC
+        echo "screen -t shell bash" >> $SCREENRC
+    fi
+    # If this service doesn't already exist in the screenrc file
+    if ! grep $1 $SCREENRC 2>&1 > /dev/null; then
+        NL=`echo -ne '\015'`
+        echo "screen -t $1 bash" >> $SCREENRC
+        echo "stuff \"$2$NL\"" >> $SCREENRC
+
+        if [[ -n ${LOGDIR} ]]; then
+            echo "logfile ${LOGDIR}/${1}.log.${CURRENT_LOG_TIME}" >>$SCREENRC
+            echo "log on" >>$SCREENRC
+        fi
+    fi
+}
+
+# Stop a service in screen
+# If a PID is available use it, kill the whole process group via TERM
+# If screen is being used kill the screen window; this will catch processes
+# that did not leave a PID behind
+# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``, ``USE_SCREEN``
+# screen_stop_service service
+function screen_stop_service {
+    local service=$1
+
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+    USE_SCREEN=$(trueorfalse True USE_SCREEN)
+
+    if is_service_enabled $service; then
+        # Clean up the screen window
+        screen -S $SCREEN_NAME -p $service -X kill
+    fi
+}
+
+# Stop a service process
+# If a PID is available use it, kill the whole process group via TERM
+# If screen is being used kill the screen window; this will catch processes
+# that did not leave a PID behind
+# Uses globals ``SERVICE_DIR``, ``USE_SCREEN``
+# stop_process service
+function stop_process {
+    local service=$1
+
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+    USE_SCREEN=$(trueorfalse True USE_SCREEN)
+
+    if is_service_enabled $service; then
+        # Kill via pid if we have one available
+        if [[ -r $SERVICE_DIR/$SCREEN_NAME/$service.pid ]]; then
+            pkill -g $(cat $SERVICE_DIR/$SCREEN_NAME/$service.pid)
+            rm $SERVICE_DIR/$SCREEN_NAME/$service.pid
+        fi
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            # Clean up the screen window
+            screen_stop_service $service
+        fi
+    fi
+}
+
+# Helper to get the status of each running service
+# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
+# service_check
+function service_check {
+    local service
+    local failures
+    SCREEN_NAME=${SCREEN_NAME:-stack}
+    SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
+
+
+    if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
+        echo "No service status directory found"
+        return
+    fi
+
+    # Check if there is any falure flag file under $SERVICE_DIR/$SCREEN_NAME
+    # make this -o errexit safe
+    failures=`ls "$SERVICE_DIR/$SCREEN_NAME"/*.failure 2>/dev/null || /bin/true`
+
+    for service in $failures; do
+        service=`basename $service`
+        service=${service%.failure}
+        echo "Error: Service $service is not running"
+    done
+
+    if [ -n "$failures" ]; then
+        die $LINENO "More details about the above errors can be found with screen, with ./rejoin-stack.sh"
+    fi
+}
+
+# Tail a log file in a screen if USE_SCREEN is true.
+function tail_log {
+    local name=$1
+    local logfile=$2
+
+    USE_SCREEN=$(trueorfalse True USE_SCREEN)
+    if [[ "$USE_SCREEN" = "True" ]]; then
+        screen_process "$name" "sudo tail -f $logfile"
+    fi
+}
+
+
+# Deprecated Functions
+# --------------------
+
+# _old_run_process() is designed to be backgrounded by old_run_process() to simulate a
+# fork.  It includes the dirty work of closing extra filehandles and preparing log
+# files to produce the same logs as screen_it().  The log filename is derived
+# from the service name and global-and-now-misnamed ``SCREEN_LOGDIR``
+# Uses globals ``CURRENT_LOG_TIME``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
+# _old_run_process service "command-line"
+function _old_run_process {
+    local service=$1
+    local command="$2"
+
+    # Undo logging redirections and close the extra descriptors
+    exec 1>&3
+    exec 2>&3
+    exec 3>&-
+    exec 6>&-
+
+    if [[ -n ${SCREEN_LOGDIR} ]]; then
+        exec 1>&${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} 2>&1
+        ln -sf ${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} ${SCREEN_LOGDIR}/screen-${1}.log
+
+        # TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
+        export PYTHONUNBUFFERED=1
+    fi
+
+    exec /bin/bash -c "$command"
+    die "$service exec failure: $command"
+}
+
+# old_run_process() launches a child process that closes all file descriptors and
+# then exec's the passed in command.  This is meant to duplicate the semantics
+# of screen_it() without screen.  PIDs are written to
+# ``$SERVICE_DIR/$SCREEN_NAME/$service.pid`` by the spawned child process.
+# old_run_process service "command-line"
+function old_run_process {
+    local service=$1
+    local command="$2"
+
+    # Spawn the child process
+    _old_run_process "$service" "$command" &
+    echo $!
+}
+
+# Compatibility for existing start_XXXX() functions
+# Uses global ``USE_SCREEN``
+# screen_it service "command-line"
+function screen_it {
+    if is_service_enabled $1; then
+        # Append the service to the screen rc file
+        screen_rc "$1" "$2"
+
+        if [[ "$USE_SCREEN" = "True" ]]; then
+            screen_process "$1" "$2"
+        else
+            # Spawn directly without screen
+            old_run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
+        fi
+    fi
+}
+
+# Compatibility for existing stop_XXXX() functions
+# Stop a service in screen
+# If a PID is available use it, kill the whole process group via TERM
+# If screen is being used kill the screen window; this will catch processes
+# that did not leave a PID behind
+# screen_stop service
+function screen_stop {
+    # Clean up the screen window
+    stop_process $1
+}
+
+
+# Plugin Functions
+# =================
+
+DEVSTACK_PLUGINS=${DEVSTACK_PLUGINS:-""}
+
+# enable_plugin <name> <url> [branch]
+#
+# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
+# ``url`` is a git url
+# ``branch`` is a gitref. If it's not set, defaults to master
+function enable_plugin {
+    local name=$1
+    local url=$2
+    local branch=${3:-master}
+    DEVSTACK_PLUGINS+=",$name"
+    GITREPO[$name]=$url
+    GITDIR[$name]=$DEST/$name
+    GITBRANCH[$name]=$branch
+}
+
+# fetch_plugins
+#
+# clones all plugins
+function fetch_plugins {
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+
+    # short circuit if nothing to do
+    if [[ -z $plugins ]]; then
+        return
+    fi
+
+    echo "Fetching DevStack plugins"
+    for plugin in ${plugins//,/ }; do
+        git_clone_by_name $plugin
+    done
+}
+
+# load_plugin_settings
+#
+# Load settings from plugins in the order that they were registered
+function load_plugin_settings {
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+
+    # short circuit if nothing to do
+    if [[ -z $plugins ]]; then
+        return
+    fi
+
+    echo "Loading plugin settings"
+    for plugin in ${plugins//,/ }; do
+        local dir=${GITDIR[$plugin]}
+        # source any known settings
+        if [[ -f $dir/devstack/settings ]]; then
+            source $dir/devstack/settings
+        fi
+    done
+}
+
+# plugin_override_defaults
+#
+# Run an extremely early setting phase for plugins that allows default
+# overriding of services.
+function plugin_override_defaults {
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+
+    # short circuit if nothing to do
+    if [[ -z $plugins ]]; then
+        return
+    fi
+
+    echo "Overriding Configuration Defaults"
+    for plugin in ${plugins//,/ }; do
+        local dir=${GITDIR[$plugin]}
+        # source any overrides
+        if [[ -f $dir/devstack/override-defaults ]]; then
+            # be really verbose that an override is happening, as it
+            # may not be obvious if things fail later.
+            echo "$plugin has overriden the following defaults"
+            cat $dir/devstack/override-defaults
+            source $dir/devstack/override-defaults
+        fi
+    done
+}
+
+# run_plugins
+#
+# Run the devstack/plugin.sh in all the plugin directories. These are
+# run in registration order.
+function run_plugins {
+    local mode=$1
+    local phase=$2
+
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+    for plugin in ${plugins//,/ }; do
+        local dir=${GITDIR[$plugin]}
+        if [[ -f $dir/devstack/plugin.sh ]]; then
+            source $dir/devstack/plugin.sh $mode $phase
+        fi
+    done
+}
+
+function run_phase {
+    local mode=$1
+    local phase=$2
+    if [[ -d $TOP_DIR/extras.d ]]; then
+        for i in $TOP_DIR/extras.d/*.sh; do
+            [[ -r $i ]] && source $i $mode $phase
+        done
+    fi
+    # the source phase corresponds to settings loading in plugins
+    if [[ "$mode" == "source" ]]; then
+        load_plugin_settings
+    elif [[ "$mode" == "override_defaults" ]]; then
+        plugin_override_defaults
+    else
+        run_plugins $mode $phase
+    fi
+}
+
+
+# Service Functions
+# =================
+
+# remove extra commas from the input string (i.e. ``ENABLED_SERVICES``)
+# _cleanup_service_list service-list
+function _cleanup_service_list {
+    echo "$1" | sed -e '
+        s/,,/,/g;
+        s/^,//;
+        s/,$//
+    '
+}
+
+# disable_all_services() removes all current services
+# from ``ENABLED_SERVICES`` to reset the configuration
+# before a minimal installation
+# Uses global ``ENABLED_SERVICES``
+# disable_all_services
+function disable_all_services {
+    ENABLED_SERVICES=""
+}
+
+# Remove all services starting with '-'.  For example, to install all default
+# services except rabbit (rabbit) set in ``localrc``:
+# ENABLED_SERVICES+=",-rabbit"
+# Uses global ``ENABLED_SERVICES``
+# disable_negated_services
+function disable_negated_services {
+    local tmpsvcs="${ENABLED_SERVICES}"
+    local service
+    for service in ${tmpsvcs//,/ }; do
+        if [[ ${service} == -* ]]; then
+            tmpsvcs=$(echo ${tmpsvcs}|sed -r "s/(,)?(-)?${service#-}(,)?/,/g")
+        fi
+    done
+    ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
+}
+
+# disable_service() removes the services passed as argument to the
+# ``ENABLED_SERVICES`` list, if they are present.
+#
+# For example:
+#   disable_service rabbit
+#
+# This function does not know about the special cases
+# for nova, glance, and neutron built into is_service_enabled().
+# Uses global ``ENABLED_SERVICES``
+# disable_service service [service ...]
+function disable_service {
+    local tmpsvcs=",${ENABLED_SERVICES},"
+    local service
+    for service in $@; do
+        if is_service_enabled $service; then
+            tmpsvcs=${tmpsvcs//,$service,/,}
+        fi
+    done
+    ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
+}
+
+# enable_service() adds the services passed as argument to the
+# ``ENABLED_SERVICES`` list, if they are not already present.
+#
+# For example:
+#   enable_service qpid
+#
+# This function does not know about the special cases
+# for nova, glance, and neutron built into is_service_enabled().
+# Uses global ``ENABLED_SERVICES``
+# enable_service service [service ...]
+function enable_service {
+    local tmpsvcs="${ENABLED_SERVICES}"
+    local service
+    for service in $@; do
+        if ! is_service_enabled $service; then
+            tmpsvcs+=",$service"
+        fi
+    done
+    ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
+    disable_negated_services
+}
+
+# is_service_enabled() checks if the service(s) specified as arguments are
+# enabled by the user in ``ENABLED_SERVICES``.
+#
+# Multiple services specified as arguments are ``OR``'ed together; the test
+# is a short-circuit boolean, i.e it returns on the first match.
+#
+# There are special cases for some 'catch-all' services::
+#   **nova** returns true if any service enabled start with **n-**
+#   **cinder** returns true if any service enabled start with **c-**
+#   **ceilometer** returns true if any service enabled start with **ceilometer**
+#   **glance** returns true if any service enabled start with **g-**
+#   **neutron** returns true if any service enabled start with **q-**
+#   **swift** returns true if any service enabled start with **s-**
+#   **trove** returns true if any service enabled start with **tr-**
+#   For backward compatibility if we have **swift** in ENABLED_SERVICES all the
+#   **s-** services will be enabled. This will be deprecated in the future.
+#
+# Cells within nova is enabled if **n-cell** is in ``ENABLED_SERVICES``.
+# We also need to make sure to treat **n-cell-region** and **n-cell-child**
+# as enabled in this case.
+#
+# Uses global ``ENABLED_SERVICES``
+# is_service_enabled service [service ...]
+function is_service_enabled {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local enabled=1
+    local services=$@
+    local service
+    for service in ${services}; do
+        [[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && enabled=0
+
+        # Look for top-level 'enabled' function for this service
+        if type is_${service}_enabled >/dev/null 2>&1; then
+            # A function exists for this service, use it
+            is_${service}_enabled
+            enabled=$?
+        fi
+
+        # TODO(dtroyer): Remove these legacy special-cases after the is_XXX_enabled()
+        #                are implemented
+
+        [[ ${service} == n-cell-* && ${ENABLED_SERVICES} =~ "n-cell" ]] && enabled=0
+        [[ ${service} == n-cpu-* && ${ENABLED_SERVICES} =~ "n-cpu" ]] && enabled=0
+        [[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && enabled=0
+        [[ ${service} == "cinder" && ${ENABLED_SERVICES} =~ "c-" ]] && enabled=0
+        [[ ${service} == "ceilometer" && ${ENABLED_SERVICES} =~ "ceilometer-" ]] && enabled=0
+        [[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && enabled=0
+        [[ ${service} == "ironic" && ${ENABLED_SERVICES} =~ "ir-" ]] && enabled=0
+        [[ ${service} == "neutron" && ${ENABLED_SERVICES} =~ "q-" ]] && enabled=0
+        [[ ${service} == "trove" && ${ENABLED_SERVICES} =~ "tr-" ]] && enabled=0
+        [[ ${service} == "swift" && ${ENABLED_SERVICES} =~ "s-" ]] && enabled=0
+        [[ ${service} == s-* && ${ENABLED_SERVICES} =~ "swift" ]] && enabled=0
+    done
+    $xtrace
+    return $enabled
+}
+
+# Toggle enable/disable_service for services that must run exclusive of each other
+#  $1 The name of a variable containing a space-separated list of services
+#  $2 The name of a variable in which to store the enabled service's name
+#  $3 The name of the service to enable
+function use_exclusive_service {
+    local options=${!1}
+    local selection=$3
+    local out=$2
+    [ -z $selection ] || [[ ! "$options" =~ "$selection" ]] && return 1
+    local opt
+    for opt in $options;do
+        [[ "$opt" = "$selection" ]] && enable_service $opt || disable_service $opt
+    done
+    eval "$out=$selection"
+    return 0
+}
+
+
+# System Functions
+# ================
+
+# Only run the command if the target file (the last arg) is not on an
+# NFS filesystem.
+function _safe_permission_operation {
+    local xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+    local args=( $@ )
+    local last
+    local sudo_cmd
+    local dir_to_check
+
+    let last="${#args[*]} - 1"
+
+    local dir_to_check=${args[$last]}
+    if [ ! -d "$dir_to_check" ]; then
+        dir_to_check=`dirname "$dir_to_check"`
+    fi
+
+    if is_nfs_directory "$dir_to_check" ; then
+        $xtrace
+        return 0
+    fi
+
+    if [[ $TRACK_DEPENDS = True ]]; then
+        sudo_cmd="env"
+    else
+        sudo_cmd="sudo"
+    fi
+
+    $xtrace
+    $sudo_cmd $@
+}
+
+# Exit 0 if address is in network or 1 if address is not in network
+# ip-range is in CIDR notation: 1.2.3.4/20
+# address_in_net ip-address ip-range
+function address_in_net {
+    local ip=$1
+    local range=$2
+    local masklen=${range#*/}
+    local network=$(maskip ${range%/*} $(cidr2netmask $masklen))
+    local subnet=$(maskip $ip $(cidr2netmask $masklen))
+    [[ $network == $subnet ]]
+}
+
+# Add a user to a group.
+# add_user_to_group user group
+function add_user_to_group {
+    local user=$1
+    local group=$2
+
+    if [[ -z "$os_VENDOR" ]]; then
+        GetOSVersion
+    fi
+
+    # SLE11 and openSUSE 12.2 don't have the usual usermod
+    if ! is_suse || [[ "$os_VENDOR" = "openSUSE" && "$os_RELEASE" != "12.2" ]]; then
+        sudo usermod -a -G "$group" "$user"
+    else
+        sudo usermod -A "$group" "$user"
+    fi
+}
+
+# Convert CIDR notation to a IPv4 netmask
+# cidr2netmask cidr-bits
+function cidr2netmask {
+    local maskpat="255 255 255 255"
+    local maskdgt="254 252 248 240 224 192 128"
+    set -- ${maskpat:0:$(( ($1 / 8) * 4 ))}${maskdgt:$(( (7 - ($1 % 8)) * 4 )):3}
+    echo ${1-0}.${2-0}.${3-0}.${4-0}
+}
+
+# Gracefully cp only if source file/dir exists
+# cp_it source destination
+function cp_it {
+    if [ -e $1 ] || [ -d $1 ]; then
+        cp -pRL $1 $2
+    fi
+}
+
+# HTTP and HTTPS proxy servers are supported via the usual environment variables [1]
+# ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in
+# ``localrc`` or on the command line if necessary::
+#
+# [1] http://www.w3.org/Daemon/User/Proxies/ProxyClients.html
+#
+#     http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
+
+# Unused function
+#function export_proxy_variables {
+#    if isset http_proxy ; then
+#        export http_proxy=$http_proxy
+#    fi
+#    if isset https_proxy ; then
+#        export https_proxy=$https_proxy
+#    fi
+#    if isset no_proxy ; then
+#        export no_proxy=$no_proxy
+#    fi
+#}
+
+# Returns true if the directory is on a filesystem mounted via NFS.
+function is_nfs_directory {
+    local mount_type=`stat -f -L -c %T $1`
+    test "$mount_type" == "nfs"
+}
+
+# Return the network portion of the given IP address using netmask
+# netmask is in the traditional dotted-quad format
+# maskip ip-address netmask
+function maskip {
+    local ip=$1
+    local mask=$2
+    local l="${ip%.*}"; local r="${ip#*.}"; local n="${mask%.*}"; local m="${mask#*.}"
+    local subnet=$((${ip%%.*}&${mask%%.*})).$((${r%%.*}&${m%%.*})).$((${l##*.}&${n##*.})).$((${ip##*.}&${mask##*.}))
+    echo $subnet
+}
+
+# Service wrapper to restart services
+# restart_service service-name
+function restart_service {
+    if is_ubuntu; then
+        sudo /usr/sbin/service $1 restart
+    else
+        sudo /sbin/service $1 restart
+    fi
+}
+
+# Only change permissions of a file or directory if it is not on an
+# NFS filesystem.
+function safe_chmod {
+    _safe_permission_operation chmod $@
+}
+
+# Only change ownership of a file or directory if it is not on an NFS
+# filesystem.
+function safe_chown {
+    _safe_permission_operation chown $@
+}
+
+# Service wrapper to start services
+# start_service service-name
+function start_service {
+    if is_ubuntu; then
+        sudo /usr/sbin/service $1 start
+    else
+        sudo /sbin/service $1 start
+    fi
+}
+
+# Service wrapper to stop services
+# stop_service service-name
+function stop_service {
+    if is_ubuntu; then
+        sudo /usr/sbin/service $1 stop
+    else
+        sudo /sbin/service $1 stop
+    fi
+}
+
+
+# Restore xtrace
+$XTRACE
+
+# Local variables:
+# mode: shell-script
+# End:
+
+if [[ $EUID -gt 0 ]]; then
+    echo "####################################################"
+    echo "#                                                  #"
+    echo "#  ERROR: You must be root to run this script!!!!  #"
+    echo "#                                                  #"
+    echo "####################################################"
+    exit 1
+fi
+
+export PATH=$PATH:/usr/local/bin:/usr/local/sbin
+
+# XXX a bit brutal but otherwise it cannot work, without tty enabled.  
+if [ -f /etc/sudoers ]; then
+  sed -i "/requiretty/d" /etc/sudoers
+fi
+
+# Include Additional Functions
+function download_playbook {
+    if [ ! -f /etc/opt/slapcache.cfg ]; then
+      slapcache-conf
+    fi
+    DFILE="/tmp/tmpplaybook$(basename $0).$$/"
+    TFILE="archive.tar.gz"
+    mkdir -p $DFILE
+    cd $DFILE
+    slapcache-download --destination=$TFILE
+    tar -xzvf $TFILE
+    rm $TFILE
+}
+
+# Determine what system we are running on.  This provides ``os_VENDOR``,
+# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
+# and ``DISTRO``
+GetDistro
+
+if [[ ${DISTRO} =~ (-.) ]] && [[ -f /etc/debian_version ]]; then
+  apt_get install lsb-release
+  GetDistro
+fi
+
+# Warn users who aren't on an explicitly supported distro, but allow them to
+# override check and attempt installation with ``export FORCE=yes``
+if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then
+    echo "WARNING: this script has not been tested on $DISTRO"
+    if [[ "$FORCE" != "yes" ]]; then
+        die $LINENO "If you wish to run this script anyway run with FORCE=yes"
+    fi
+fi
+
+# Make sure wheezy backports are available.
+if [[ $DISTRO == "wheezy" ]]; then
+    echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
+fi
+
+if is_fedora && [[ $DISTRO == "rhel7" ]]; then
+    # RHEL requires EPEL for many Ansible dependencies
+
+    # NOTE: We always remove and install latest -- some environments
+    # use snapshot images, and if EPEL version updates they break
+    # unless we update them to latest version.
+    if sudo yum repolist enabled epel | grep -q 'epel'; then
+        uninstall_package epel-release || true
+    fi
+
+    # This trick installs the latest epel-release from a bootstrap
+    # repo, then removes itself (as epel-release installed the
+    # "real" repo).
+    #
+    # You would think that rather than this, you could use
+    # $releasever directly in .repo file we create below.  However
+    # RHEL gives a $releasever of "6Server" which breaks the path;
+    # see https://bugzilla.redhat.com/show_bug.cgi?id=1150759
+    cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
+[epel-bootstrap]
+name=Bootstrap EPEL
+mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-7&arch=\$basearch
+failovermethod=priority
+enabled=0
+gpgcheck=0
+EOF
+    yum update -y
+    yum_install yum-utils
+    # Enable a bootstrap repo.  It is removed after finishing
+    # the epel-release installation.
+    yum-config-manager --enable epel-bootstrap
+    yum_install epel-release || \
+        die $LINENO "Error installing EPEL repo, cannot continue"
+    # EPEL rpm has installed it's version
+    rm -f /etc/yum.repos.d/epel-bootstrap.repo
+
+    # ... and also optional to be enabled
+    is_package_installed yum-utils || install_package yum-utils
+    yum-config-manager --enable rhel-7-server-optional-rpms
+
+fi
+
+if is_fedora && [[ $DISTRO == "rhel6" ]]; then
+    sed -i "/requiretty/d" /etc/sudoers
+    if sudo yum repolist enabled epel | grep -q 'epel'; then
+        uninstall_package epel-release || true
+    fi
+
+    cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
+[epel-bootstrap]
+name=Bootstrap EPEL
+mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-6&arch=\$basearch
+failovermethod=priority
+enabled=0
+gpgcheck=0
+EOF
+    yum update -y
+    yum_install yum-utils
+    yum-config-manager --enable epel-bootstrap
+    yum_install epel-release || \
+        die $LINENO "Error installing EPEL repo, cannot continue"
+    # EPEL rpm has installed it's version
+    rm -f /etc/yum.repos.d/epel-bootstrap.repo
+
+    is_package_installed yum-utils || install_package yum-utils
+    yum-config-manager --enable rhel-6-server-optional-rpms
+
+    is_package_installed libselinux-python || install_package libselinux-python
+fi
+
+is_package_installed openssl || install_package openssl
+# stick to a specific ansible package version
+if is_ubuntu; then
+  # is_ubuntu also handle debian distro
+  is_package_installed ansible || install_package ansible=1.*
+elif is_fedora; then
+  # is_fedora also handle CentOS distro
+  is_package_installed ansible || install_package ansible1.9.noarch
+else
+  is_package_installed ansible || install_package ansible
+fi
+is_package_installed python-setuptools || install_package python-setuptools
+
+if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
+  is_package_installed python-apt || install_package python-apt
+  is_package_installed python-pycurl || install_package python-pycurl
+fi
+
+if is_ubuntu && [[ $DISTRO == "vivid" ]]; then
+  is_package_installed python-apt || install_package python-apt
+  is_package_installed python-pycurl || install_package python-pycurl
+fi
+
+if [[ $DISTRO == "LinuxMint-17.2" ]]; then
+  is_package_installed python-apt || install_package python-apt
+  is_package_installed python-pycurl || install_package python-pycurl
+fi
+
+ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
+mkdir -p $ANSIBLE_PLUGIN_LOCATION
+/bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
+import os
+import time
+import json
+import ansible
+
+baseModule = object
+ANSIBLE_VERSION = 1
+
+if hasattr(ansible, 'plugins') and  hasattr(ansible.plugins, 'callback'):
+  baseModule = ansible.plugins.callback.CallbackBase
+  ANSIBLE_VERSION = 2
+
+class CallbackModule(baseModule):
+    """
+    logs playbook results, per host, in /var/log/ansible/hosts
+    """
+    log_path = '/var/log/ansible/hosts'
+    fd_list = {}
+
+    def __init__(self):
+
+      if ANSIBLE_VERSION > 1:
+        super(CallbackModule, self).__init__()
+
+      if not os.path.exists(self.log_path):
+        os.makedirs(self.log_path)
+      else:
+        for filename in os.listdir(self.log_path):
+          filepath = os.path.join(self.log_path, filename)
+          if os.path.exists(filepath) and os.path.isfile(filepath):
+            os.unlink(filepath)
+
+    def writeLog(self, host, category, content):
+      if not self.fd_list.has_key(category):
+        self.fd_list[category] = open(
+          os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
+        )
+
+      self.fd_list[category].write(content + '\n')
+
+    def log(self, host, category, data, ignore_errors=False):
+      if host == "localhost":
+        host = "127.0.0.1" # keep compatibility
+
+      if type(data) == dict:
+        if '_ansible_verbose_override' in data:
+          # avoid logging extraneous data
+          return
+
+        data = data.copy()
+        content = json.dumps(data)
+
+        if ignore_errors:
+          category = '%s_IGNORED' % category
+        self.writeLog(host, category, content)
+
+    def _stats(self, stats):
+      for key in self.fd_list:
+        self.fd_list[key].close()
+
+    def runner_on_failed(self, host, res, ignore_errors=False):
+      self.log(host, 'FAILED', res, ignore_errors)
+
+    def runner_on_ok(self, host, res):
+      self.log(host, 'OK', res)
+
+    def runner_on_skipped(self, host, item=None):
+      pass
+
+    def runner_on_unreachable(self, host, res):
+      self.log(host, 'UNREACHABLE', res)
+
+    def runner_on_async_failed(self, host, res, jid):
+      self.log(host, 'ASYNC_FAILED', res)
+
+    def playbook_on_import_for_host(self, host, imported_file):
+      self.log(host, 'IMPORTED', imported_file)
+
+    def playbook_on_not_import_for_host(self, host, missing_file):
+      self.log(host, 'NOTIMPORTED', missing_file)
+
+    def playbook_on_stats(self, stats):
+      self._stats(stats)
+
+EOF
+
+ansible localhost -m easy_install -a name=slapcache --connection=local
+ansible localhost -m easy_install -a name=requests --connection=local
+
+# Include Additional Functions
+if [ ! -f /etc/opt/slapcache.cfg ]; then
+  slapcache-conf
+fi
+sed -i "s/key = slapos-global-key/key = slapos-global-official-vifib-key/g" /etc/opt/slapcache.cfg 
+
+DFILE="/tmp/tmpplaybook_unstable$(basename $0).$$/"
+TFILE="archive.tar.gz"
+mkdir -p $DFILE
+cd $DFILE
+wget -O $TFILE http://10.0.2.100/playbook.tar.gz
+tar -xzvf $TFILE
+rm $TFILE
+
+clear
+
+echo "Starting Ansible playbook:"
+ansible-playbook $TEST_YML_PATH -i hosts --connection=local

software/erp5testnode/testsuite/dummy/software.cfg

@@ -14,4 +14,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 2.9
+slapos.recipe.template = 3.0

software/erp5testnode/testsuite/instance-resilient-test.cfg.jinja2

@@ -50,6 +50,10 @@ config-resiliency-backup-periodicity = */10 * * * *
 config-resilient-clone-number = 1
 config-ignore-known-hosts-file = false
 config-cpu-usage-ratio = 1
+# Use one external folder (same of erp5testnode for build the software
+# This is a way to preserve the latest software release build speeding up
+# the tests.)
+config-software-root = ${buildout:directory}/../../soft
 # XXX hardcoded
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance

software/erp5testnode/testsuite/slaprunner.cfg

@@ -7,4 +7,8 @@ parts += template-erp5testnode
 
 [template-resilient-test]
 filename = instance-resilient-test.cfg.jinja2 
-md5sum = 268d07ed48199d34259aff1ba544d852 
+md5sum = dadf1e8599ba7d96c5af827ec389a9cf 
+
+[exporter-default-configuration]
+# Define shorter interaction to speed up tests
+backup_wait_time = 1

software/fluentd/fluentd-agent.conf.jinja2.in

-{% set conf_text = slapparameter_dict.get('conf_text', '')  -%}
-{{ conf_text }}
\ No newline at end of file

software/fluentd/instance.cfg.in

 [buildout]
 parts =
-  directory
   service-fluentd
-  fluentd-agent-conf
 
 eggs-directory = {{ buildout['eggs-directory'] }}
 develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
@@ -32,14 +30,16 @@ recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/fluentd-service
 command-line    = {{ fluentd_location }}/bin/fluentd
     -v
-    -c ${directory:etc}/fluentd-agent.conf
-environment = 
+    -c ${fluentd-agent-conf:rendered}
+environment =
   GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
 
 [fluentd-agent-conf]
 recipe  = slapos.recipe.template:jinja2
-template = {{ fluentd_conf }}/fluentd-agent.conf.jinja2.in
-rendered = ${buildout:directory}/etc/fluentd-agent.conf
+template = inline:{% raw -%}
+  {{ slapparameter_dict.get('conf_text', '') }}
+  {%- endraw %}
+rendered = ${directory:etc}/fluentd-agent.conf
 mode = 0644
-context = 
+context =
   key slapparameter_dict instance-parameter:configuration

software/fluentd/software.cfg

 [buildout]
-
 extends =
   ../../component/fluentd/buildout.cfg
   ../../stack/slapos.cfg
 
-parts = 
-  fluentd
-  fluentd-agent-conf
+parts =
   instance-profile
   slapos-cookbook
 
@@ -18,17 +15,9 @@ mode = 0644
 extensions = jinja2.ext.do
 context =
   section buildout  buildout
-  raw fluentd_location                ${fluentd:location}
-  raw fluentd_conf                    ${buildout:directory}
-
-[fluentd-agent-conf]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
-#md5sum = 4b81ddcbe2f16d23013caac37151d396 
-output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
-mode = 0644
+  key fluentd_location fluentd:location
 
 [fluentd]
-gems += 
+gems +=
   fluent-plugin-wendelin==0.1
   fluent-plugin-bin

software/gitlab/software.cfg

@@ -384,6 +384,6 @@ strip-top-level-dir = true
 cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
-rubygemsrecipe  = 0.2.2
-slapos.recipe.template = 2.9
+rubygemsrecipe  = 0.2.2+slapos001
+slapos.recipe.template = 3.0
 z3c.recipe.scripts = 1.0.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0

software/ipython_notebook/instance.cfg.in

@@ -74,7 +74,13 @@ command-line =
 wrapper-path = ${directory:service}/ipython_notebook
 parameters-extra = true
 # Explicitly define IPython Directory to be used while starting ipython notebook
-environment = IPYTHONDIR=${directory:ipython_dir}
+# Also explicitly force IPython to use en_US.utf8 encoding
+environment =
+  IPYTHONDIR=${directory:ipython_dir}
+  JUPYTER_CONFIG_DIR=${directory:ipython_dir}
+  LC_ALL=C.UTF-8
+  LANG=C.UTF-8
+  LANGUAGE=C.UTF-8
 
 [ipython-notebook-config]
 <= dynamic-jinja2-template-base
@@ -109,6 +115,7 @@ filename = knowledge0.cfg
 [read-knowledge0]
 recipe = slapos.cookbook:zero-knowledge.read
 filename = knowledge0.cfg
+password =
 
 [monitor-instance-parameter]
 monitor-base-url = ${monitor-frontend-promise:url}

software/ipython_notebook/software.cfg

@@ -47,7 +47,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100
 [erp5-kernel]
 <= download-file-base
 filename = ERP5kernel.py.jinja
-md5sum = 5e7e0acef2f32e75f8f17bc851414aa1
+md5sum = f7be6c3ef9cdfaaea3b8756895a22081
 
 [kernel-json]
 <= download-file-base
@@ -57,14 +57,14 @@ md5sum = ab6e78ea20855e07d388b5b86d1770fe
 [custom-js]
 <= download-file-base
 filename = custom.js.jinja
-md5sum = a851a68667a4e676cfa86b34a86b4239
+md5sum = 584aa839aa20263ba885537b74c83abd
 
 [instance-jupyter]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg.in
 rendered = ${buildout:directory}/template.cfg
 mode = 0644
-md5sum = b6555d25fd89bf92f47eda60a29c2b44
+md5sum = c9d2c7a3658457b7f43cbbbc38ef247c
 context =
   key bin_directory buildout:bin-directory
   key develop_eggs_directory buildout:develop-eggs-directory
@@ -107,7 +107,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 2.10
+slapos.recipe.template = 3.0
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/ipython_notebook/template/ERP5kernel.py.jinja

@@ -106,8 +106,32 @@ class ERP5Kernel(Kernel):
         # Get the magic value recived via code from frontend
         magic_value = code.split()[1]
         # Set magic_value to the required attribute
-        setattr(self, magic_info.variable_name , magic_value)
-        self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+        
+        if magic_info.magic_name == 'notebook_set_reference':
+          required_attributes = ['url', 'password', 'user']
+          missing_attributes = []
+          for attribute in required_attributes:
+            if not getattr(self, attribute):
+              missing_attributes.append(attribute)
+          
+          if missing_attributes != []:
+            self.response = "You still haven't entered all required magics. \
+Please do so before inputting your reference."
+          else:
+            if self.check_existing_reference(reference=magic_value):
+              self.response = 'WARNING: You already have a notebook with \
+reference %s. It might be a good idea to use different references for new \
+notebooks. \n' % magic_value
+            else:
+              self.response = ''
+            setattr(self, magic_info.variable_name , magic_value)
+            self.response = self.response + 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+        elif magic_info.magic_name != 'erp5_password':
+          setattr(self, magic_info.variable_name , magic_value)
+          self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+        else:
+          setattr(self, magic_info.variable_name , magic_value)
+          self.response = ""
 
       # Catch exception while setting attribute and set message in response
       except AttributeError:
@@ -126,7 +150,8 @@ class ERP5Kernel(Kernel):
 
       # Display the message/response from this fucntion before moving forward so
       # as to keep track of the status
-      self.display_response(response=self.response)
+      if self.response != "":
+        self.display_response(response=(self.response + '\n'))
 
   def check_required_attributes(self):
     """
@@ -138,25 +163,32 @@ class ERP5Kernel(Kernel):
       made to ask user to enter value.
     """
     result_list = []
-    required_attributes  = ['url', 'password', 'user', 'reference']
+    required_attributes  = ['url', 'user', 'password', 'reference']
+    missing_attributes = []
 
     # Set response to empty so as to flush the response set by some earlier fucntion call
     self.response = ''
 
-    # Loop to check if the attributes are set
+    # Loop to check if the required attributes are set
     for attribute in required_attributes:
       if getattr(self, attribute):
         result_list.append(True)
       else:
         # Set response/message for attributes which aren't set
-        self.response = '\nPlease enter %s in next cell. '%attribute
+        missing_attributes.append(attribute)
         result_list.append(False)
-
+    
     # Compare result_list to get True for all True results and False for any False result 
     check_attributes = all(result_list)
+    
+    if check_attributes:
+      self.response = 'You have entered all required magics. You may now use your notebook.'
+    else:
+      self.response = '''You have these required magics remaining: %s. \n''' % (
+      ', '.join(map(str, missing_attributes)))
 
     # Display response to frontend before moving forward
-    self.display_response(response=self.response)
+    self.display_response(response=(self.response + '\n'))
 
     return check_attributes
 
@@ -238,7 +270,7 @@ class ERP5Kernel(Kernel):
             checked_attribute = self.check_required_attributes()
             if checked_attribute and magic_info.send_request:
               # Call the function to send request to erp5 with the arguments given
-              self.make_erp5_request(message='\nPlease proceed',
+              self.make_erp5_request(message='Please proceed\n',
               request_reference=magic_info.request_reference,
               display_message=magic_info.display_message)
 
@@ -246,7 +278,8 @@ class ERP5Kernel(Kernel):
               # Since this response would be either success message or failure
               # error message, both of which are string type, so, we can simply
               # display the stream response.
-              self.display_response(response=self.response)
+              if self.response != 'Please proceed\n':
+                self.display_response(response=self.response)
 
           else:
             # Set response if there is no magic or the magic name is not in MAGICS
@@ -337,6 +370,31 @@ class ERP5Kernel(Kernel):
       'user_expressions': {}}
 
     return reply_content
+  
+  # Checks the ERP5 site if there are existing notebooks with the same reference.
+  # Returns True if there are.
+  def check_existing_reference(self, reference):
+    if reference == None:
+      return False
+    
+    modified_url = self.url[:self.url.rfind('/')] + '/Base_checkExistingReference'
+    result = True
+    
+    try:
+      erp5_request = requests.post(
+        modified_url,
+        verify=False,
+        auth=(self.user, self.password),
+        data={
+          'reference': reference,
+          })
+      result = erp5_request.content
+      
+    except requests.exceptions.RequestException as e:
+      self.response = str(e)
+      
+    return result
+
 
 if __name__ == '__main__':
   IPKernelApp.launch_instance(kernel_class=ERP5Kernel)

software/ipython_notebook/template/custom.js.jinja

@@ -85,15 +85,15 @@ $([IPython.events]).on('notebook_loaded.Notebook', function(){
   var kernelname = IPython.notebook.kernel_selector.current_selection;
   var display_text="<div class='output_subarea output_text output_result'>\
   <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
-  1. Add <b>%erp5_url &lt;your_erp5_url&gt;/erp5/Base_executeJupyter</b>.</br>\
-  Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
-  2. <b>%notebook_set_reference &lt;your_notebook_refernce&gt;</b></br>\
+  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
   It would be better to set the reference to match with erp5 reference pattern.</br>\
-  3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
-  4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
   5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
   <p><u>OTHER USEFUL MAGICS</u> -</br>\
   <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
   NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
   so changes to them would be disregarded and would throw an error.</br>\
   <p><u>About classes, functions and global state on modules:</u></p>\

software/jstestnode/software.cfg

@@ -112,6 +112,6 @@ output = ${buildout:directory}/runTestSuite.in
 mode = 0644
 
 [versions]
-erp5.util = 0.4.46
-slapos.recipe.template = 2.9
+erp5.util = 0.4.49
+slapos.recipe.template = 3.0
 selenium = 2.53.1

software/kvm/common.cfg

@@ -89,7 +89,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 5c159864ce41394486867c27d65c2f50 
+md5sum = f40a938400e789361c95d5a9246bf0ef 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +98,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = c66ec96a73470148707937bf96436fd6
+md5sum = a849d4a6060fdb4e9e86917fb77ef153
 download-only = true
 on-update = true
 
@@ -107,7 +107,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 6ef7e38526f024356743e77973f0fb5f
+md5sum = d9745bc9bd1d22a640a628c005f88ffb
 download-only = true
 on-update = true
 
@@ -116,7 +116,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 700676dff4a3835a9d6cde015d91922e
+md5sum = 93e7143b46c6136b7cafe888fac90aba
 download-only = true
 on-update = true
 
@@ -124,7 +124,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-import.cfg.jinja2.in
-md5sum = 5e21ad759c148d26134ecefbb4d3d9f1
+md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7
 mode = 0644
 download-only = true
 on-update = true
@@ -143,7 +143,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = c2ab8a5ff9d8cd351682382ea06b3328
+md5sum = 13387d37bbf430f1d2b827c8f1acc804
 download-only = true
 on-update = true
 
@@ -152,7 +152,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
 filename = kvm-export.sh.jinja2
-md5sum = c18e958bf374d5fa35dbb952686595fb
+md5sum = b617d64de73de1eed518185f310bbc82
 download-only = true
 mode = 0755
 

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -80,7 +80,7 @@ config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
 config-disable-ansible-promise = {{ dumps(kvm_parameter_dict.get('disable-ansible-promise', False)) }}
 config-enable-monitor = {{ enable_monitoring }}
 config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-config-monitor-username = ${monitor-htpasswd:username}
+config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 # Enable simple http server on ipv6 so all VMs will access it
@@ -226,16 +226,10 @@ mode = {{ mode }}
 {{ writefile('cluster-data-content', '${directory:webroot}/${hash-code:passwd}/data', slapparameter_dict.get('cluster-data', ''), '700') }}
 {% endif -%}
 
-[monitor-htpasswd]
-recipe = slapos.cookbook:generate.password
-storage-path = ${directory:etc}/.monitor_user
-bytes = 8
-username = admin
-
 [monitor-instance-parameter]
 monitor-httpd-port = 8060
 cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-username = ${monitor-htpasswd:username}
+username = admin
 password = ${monitor-htpasswd:passwd}
 
 [monitor-conf-parameters]
@@ -254,7 +248,7 @@ recipe = slapos.cookbook:publish
 {% if enable_monitoring -%}
 {%   set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 {% do part_list.append('monitor-base') -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 {% endif -%}
 
 [buildout]

software/kvm/instance-kvm-export.cfg.jinja2

@@ -35,3 +35,16 @@ context =
 [publish-connection-information]
 <= resilient-publish-connection-parameter
 
+{% if str(slapparameter_dict.get('enable-monitor', True)).lower() == 'true' -%}
+[monitor-instance-parameter]
+monitor-httpd-port = 8026
+monitor-title = {{ slapparameter_dict.get('name', 'KVM Standalone') }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{%   if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{%   endif -%}
+{%   if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{%   endif -%}
+
+{% endif -%}

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -4,16 +4,19 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+parts += 
+  kvm-disk-image-corruption-promise
+
 # Here, we don't need KVM to run to import data, so we don't
 # even extend the kvm instance profile.
 extends = 
   {{ pbsready_import_template }}
 
 [resilient-publish-connection-parameter]
-monitor-base-url = ${publish:monitor-base-url}
-monitor-url = ${publish:monitor-url}
-monitor-user = ${publish:monitor-user}
-monitor-password = ${publish:monitor-password}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
 
 [monitor-instance-parameter]
 monitor-httpd-port = 8276
@@ -59,3 +62,21 @@ context =
     raw gzip_binary {{ gzip_binary }}
 
 backup-disk-path = ${directory:backup}/virtual.qcow2
+
+[kvm-disk-image-corruption-promise]
+# Check that disk image is not corrupted
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  QEMU_IMAGE="${directory:srv}/virtual.qcow2"
+  if [ ! -s "$QEMU_IMAGE" ]; then
+    exit 0
+  fi
+  {{ qemu_location }}/bin/qemu-img check $QEMU_IMAGE
+  RETURN_CODE=$?
+  if [ $RETURN_CODE -eq 0 ] || [ $RETURN_CODE -eq 3 ]; then
+    exit 0
+  else
+    exit 1
+  fi
+output = ${directory:promises}/kvm-disk-image-corruption
+mode = 700
\ No newline at end of file

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -68,8 +68,8 @@ backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
 ipv6 = ${request-kvm:connection-ip}
 {% if monitor_dict -%}
-monitor-base-url = ${publish:monitor-base-url}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 {% endif -%}
 
 [kvm-frontend-url-promise]

software/kvm/instance-kvm.cfg.jinja2

@@ -393,10 +393,10 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
 {%   endif %}
 {% endif %}
 {% if monitor -%}
-monitor-base-url = ${publish:monitor-base-url}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 {%   set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 {%   if monitor_interface_url -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 {%   endif -%}
 {% endif -%}
 

software/kvm/instance.cfg.in

@@ -149,6 +149,7 @@ extensions = jinja2.ext.do
 context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
+    raw qemu_location ${kvm:location}
     raw template_kvm_import ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
     raw pbsready_import_template ${pbsready-import:output}
     key slapparameter_dict slap-configuration:configuration

software/kvm/software.cfg

@@ -5,34 +5,34 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.67
-erp5.util = 0.4.46
+slapos.toolbox = 0.69
+erp5.util = 0.4.49
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 2.9
+slapos.recipe.template = 3.0
 smmap = 0.9.0
 # websockify = 0.8.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
@@ -40,9 +40,9 @@ lockfile = 0.12.2
 numpy = 1.11.2rc1
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 passlib = 1.6.5
\ No newline at end of file

software/kvm/template/kvm-export.sh.jinja2

@@ -2,6 +2,7 @@
 #
 # Create a backup of the disk image of the virtual machine
 #
+set -e
 LC_ALL=C
 export LC_ALL
 BACKUP_DIR={{ directory['backup'] }}

software/monitor/instance-monitor.cfg.jinja2

@@ -113,8 +113,8 @@ wrapper-path = ${monitor-directory:promises}/system-MEMORY-usage-check
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
-monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
-server_log_url = ${publish:monitor-base-url}/${slap-configuration:private-hash}/ 
+monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
+server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/ 
 
 [monitor-instance-parameter]
 instance-configuration =

software/monitor/software.cfg

@@ -32,7 +32,7 @@ mode = 0644
 recipe = slapos.recipe.build:download 
 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
 destination = ${buildout:directory}/template-base-monitor.cfg
-md5sum = c513cf70b8671840b2ba049d2826a661
+md5sum = c3fc83c851568bf0e56393676c489e18
 mode = 0644
 
 [template-monitor-distributor]
@@ -102,6 +102,7 @@ scripts =
   monitor.runpromise
   monitor.genstatus
   monitor.configwrite
+  is-process-older-than-dependency-set
 
 [fluentd]
 gems += 

software/nayuos/software.cfg

@@ -11,7 +11,7 @@ parts +=
   install-eggs-for-the-instance
 
 [versions]
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0
 
 [verify-packages-exist-on-host]
 recipe = plone.recipe.command

software/neoppod/instance-neo-admin.cfg.in

@@ -25,12 +25,12 @@ path = ${directory:promises}/neo-admin-promise
 < = logrotate-entry-base
 name = neo-admin
 log = ${neo-admin:logfile}
-post = {{ bin_directory }}/slapos-kill -n neoadmin -s RTMIN+1 ${:log}
+post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- ${neo-admin:binary} -l ${:log}
 
 [neoctl]
 recipe = slapos.cookbook:wrapper
 command-line =
-  {{bin_directory}}/neoctl -a ${neo-admin:ip}:${neo-admin:port} 
+  {{bin_directory}}/neoctl -a ${neo-admin:ip}:${neo-admin:port}
 {%- if slapparameter_dict['ssl'] %}
     --ca ${directory:etc}/ca.crt
     --cert ${directory:etc}/neo.crt

software/neoppod/instance-neo-input-schema.json

@@ -70,6 +70,11 @@
             "default": 1,
             "type": "integer"
           },
+          "disable-drop-partitions": {
+              "description": "Set the --disable-drop-partitions option for storage nodes.",
+              "default": false,
+              "type": "boolean"
+          },
           "mysql": {
             "description": "Dictionary containing parameters for MySQL.",
             "default": {},

software/neoppod/instance-neo-master.cfg.in

@@ -33,4 +33,4 @@ path = ${directory:promises}/neo-master-promise
 < = logrotate-entry-base
 name = neo-master
 log = ${neo-master:logfile}
-post = {{ bin_directory }}/slapos-kill -n neomaster -s RTMIN+1 ${:log}
+post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- ${neo-master:binary} -l ${:log}

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -87,6 +87,7 @@ masters = ${publish:masters}
 database-adapter = MySQL
 wait-database = -1
 engine = {{ slapparameter_dict.get('engine', '') }}
+disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
 
 {% for i in range(slapparameter_dict.get('storage-count', 1)) -%}
 {%   set storage_id = 'neo-storage-' ~ i -%}
@@ -101,7 +102,7 @@ database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
 < = logrotate-entry-base
 name = {{ storage_id }}
 log = {{ '${' +  storage_id + ':logfile}' }}
-post = {{ bin_directory }}/slapos-kill -n neostorage -s RTMIN+1 ${:log}
+post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostorage -l ${:log}
 
 {% endfor -%}
 [init-script]

software/neoppod/software-common.cfg

@@ -91,15 +91,15 @@ md5sum = f3259726bd5d824c569dc7db6b7d26a0
 
 [instance-neo-admin]
 <= download-base-neo
-md5sum = d5eb14e3d83d7787950de913d7e29aad
+md5sum = f671bb5bbb006d6bcecee8c7814bee57
 
 [instance-neo-master]
 <= download-base-neo
-md5sum = c6e0db7344a26a55580102e3cc33c984
+md5sum = 1fee10f02c2fa2a581e21878ca0fd704
 
 [instance-neo-storage-mysql]
 <= download-base-neo
-md5sum = c1cdb4289ff458545de15aca65121b80
+md5sum = 67d623d631c2f99e33bcabc79fc9cccf
 
 [template-neo-my-cnf]
 <= download-base-neo
@@ -113,44 +113,44 @@ apache-libcloud = 1.5.0
 ecdsa = 0.13
 gitdb2 = 2.0.0
 msgpack-python = 0.4.8
-mysqlclient = 1.3.10
+mysqlclient = 1.3.10+SlapOSPatched001
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 2.10
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0
 zodbtools = 0.0.0.dev3
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 GitPython = 2.1.3
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 feedparser = 5.2.1
 
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 paramiko = 2.1.2
 
 # Required by:
-# slapos.toolbox==0.67
+# slapos.toolbox==0.69
 passlib = 1.7.1

software/neoppod/software.cfg

@@ -43,7 +43,7 @@ revision = c0adf81cfcfa2e4785d509e1c5d57559911e829a
 
 
 [versions]
-erp5.util = 0.4.46
+erp5.util = 0.4.49
 # To match ERP5
 ZConfig = 2.9.3
 zc.lockfile = 1.0.2

software/nginx-push-stream/software.cfg

@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 2.7
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -81,7 +81,7 @@ username = admin
 [monitor-instance-parameter]
 monitor-httpd-port = 8099
 cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-username = ${monitor-htpasswd:username}
+username = ${monitor-instance-parameter:username}
 password = ${monitor-htpasswd:passwd}
 
 [monitor-conf-parameters]
@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }}
 {% endfor -%}
 config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
 config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }}
-config-monitor-username = ${monitor-htpasswd:username}
+config-monitor-username = ${monitor-instance-parameter:username}
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 [publish-information]
@@ -117,12 +117,12 @@ recipe = slapos.cookbook:publish
 domain = {{ slapparameter_dict.get('domain') }}
 slave-amount = {{ slave_instance_list | length }}
 ns-record = {{ ns_record }}
-monitor-url = ${publish:monitor-url}
-monitor-user = ${publish:monitor-user}
-monitor-password = ${publish:monitor-password}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
 {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
 {% if monitor_interface_url -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}
 {%   endif -%}
 
 [buildout]
@@ -154,4 +154,4 @@ cert_file = ${slap-connection:cert-file}
 slave_instance_list =
 -dns-quantity = 1
 -dns-type = single-default
-{%- endif %}
+{%- endif %}
\ No newline at end of file

software/powerdns/instance-powerdns.cfg

@@ -101,9 +101,9 @@ port = $${pdns:port}
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
-monitor-url = $${publish:monitor-url}
-monitor-user = $${publish:monitor-user}
-monitor-password = $${publish:monitor-password}
+monitor-url = $${monitor-publish-parameters:monitor-url}
+monitor-user = $${monitor-publish-parameters:monitor-user}
+monitor-password = $${monitor-publish-parameters:monitor-password}
 
 #####################
 # Power DNS Slave configuration
@@ -148,4 +148,4 @@ extends = ${monitor-template:output}
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
-{% endif%}
+{% endif%}
\ No newline at end of file

software/powerdns/software.cfg

@@ -25,7 +25,7 @@ mode = 0644
 [template-powerdns]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-powerdns.cfg
-md5sum = 06ad5bc18936b04b0bd566353047d02f
+md5sum = 7710007c0bd1817e3c1485ea9ece1e23
 output = ${buildout:directory}/template-powerdns.cfg
 mode = 0644
 
@@ -38,7 +38,7 @@ mode = 640
 [template-dns-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
-md5sum = ff2ac1f61fdd57fb349e209b9950f117
+md5sum = 3699d0751632e0c11b466342789e27a3
 mode = 0644
 
 [iso-list]
@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0

software/re6stnet/instance-re6stnet.cfg.in

@@ -201,7 +201,7 @@ port = ${apache-conf:port}
 {% do publish_dict.__setitem__('slave-amount',  '${re6st-registry:slave-amount}') -%}
 [publish]
 recipe = slapos.cookbook:publish
-monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
+monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor -%}

software/re6stnet/software.cfg

@@ -82,7 +82,7 @@ extra-context =
 [template-re6stnet]
 < = download-base
 filename = instance-re6stnet.cfg.in
-md5sum = 9e5e6cf010ec2a329e6bc8def7f95ba7
+md5sum = 7d65efffeae3f9ae4e64699303a083b4
 
 [template-apache-conf]
 < = download-base
@@ -111,16 +111,16 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 2.7
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
@@ -128,11 +128,11 @@ atomize = 0.2.0
 backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
@@ -140,10 +140,10 @@ lockfile = 0.12.2
 miniupnpc = 1.9
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 paramiko = 2.0.1
 
 # Required by:
-# slapos.toolbox = 0.67
+# slapos.toolbox==0.69
 rpdb = 0.1.5
 

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.7
+slapos.recipe.template = 3.0

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 2.7
+slapos.recipe.template = 3.0

software/slapos-master/buildout.hash.cfg

@@ -15,11 +15,11 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 1489091851f27c41243eeaa4967cc91c
+md5sum = 50834c59ded302034cafcce7315a55c3
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = a410f10fe9766c9321ffd7b18ace5a4f
+md5sum = 5562e6738470b0fc96211f46bf8d41bc
 
 [template-apache-backend-conf]
 filename = apache-backend.conf.in

software/slapos-master/instance-balancer.cfg.in

@@ -170,6 +170,7 @@ recipe = slapos.cookbook:publish.serialised
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
 {{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
 {% endfor -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
@@ -232,11 +233,34 @@ private = ${:ca-dir}/private
 certs = ${:ca-dir}/certs
 newcerts = ${:ca-dir}/newcerts
 crl = ${:ca-dir}/crl
+apachedex = ${monitor-directory:private}/apachedex
+
+[monitor-generate-apachedex-report]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${monitor-directory:reports}/${:command}
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
+command = apachedex_every_3_hour
+
+[apachedex-parameters]
+default_parameter = 
+# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
+# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
+apache-log-list = ${apache-conf-parameter-dict:access-log}
+default = ${monitor-directory:etc}/apdex_default
+base-list = ${monitor-directory:etc}/apdex_base_list
+skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
+erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
 monitor-title = Balancer monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
+instance-configuration = 
+  file apachedex-default ${apachedex-parameters:default}
+  file apachedex-base-list ${apachedex-parameters:base-list}
+  file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
+  file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
 
 [buildout]
 extends =
@@ -245,4 +269,5 @@ extends =
 parts +=
   publish
   logrotate-apache
+  monitor-generate-apachedex-report
   {{ part_list | join('\n  ') }}

software/slapos-master/instance-erp5.cfg.in

@@ -8,6 +8,7 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set monitor_base_url_dict = {} -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -22,7 +23,10 @@ software-type = {{ software_type }}
 return = {{ ' '.join(ret) }}
 {% for ret, publish in ret.iteritems() -%}
 {%   if publish -%}
-{%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
+{%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
+{%   endif -%}
+{%   if ret == "monitor-base-url" -%}
+{%     do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-' ~ ret ~ '}') -%}
 {%   endif -%}
 {% endfor -%}
 {{ root_common.sla(name) }}
@@ -34,10 +38,10 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
 {% endmacro -%}
 
-{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }}
-{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
-{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
-{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
+{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
 {%- else %}
@@ -74,7 +78,7 @@ connection-url = smtp://127.0.0.2:0/
 {%   else -%}
 {{     assert(server_type == 'zeo', server_type) -}}
 {# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
-{{     request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
+{{     request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port', 'monitor-base-url')), key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {%   endif -%}
 {% endfor -%}
 
@@ -126,6 +130,7 @@ name = neo-${gen-neo-cluster-base:passwd}
 return =
   zope-address-list
   hosts-dict
+  monitor-base-url
 {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
@@ -141,6 +146,7 @@ config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${publish-early:inituser-password}
 config-kumofs-url = ${request-memcached-persistent:connection-url}
 config-memcached-url = ${request-memcached-volatile:connection-url}
+config-monitor-passwd = ${monitor-htpasswd:passwd}
 config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
 config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
@@ -183,6 +189,7 @@ software-type = zope
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
+{% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
@@ -284,10 +291,12 @@ name = balancer
 software-type = balancer
 {{ root_common.sla('balancer') }}
 return =
+  monitor-base-url
 {%- for family in zope_family_dict %}
   {{ family }}
   {{ family }}-v6
 {% endfor -%}
+{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
@@ -300,6 +309,8 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
 config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-shared-certificate-authority-path = ${directory:ca-dir} 
+config-monitor-passwd = ${monitor-htpasswd:passwd}
+
 
 [request-frontend-base]
 {% if has_frontend -%}
@@ -323,6 +334,7 @@ return = site_url
 {% endif -%}{# if zope_partition_dict -#}
 
 [publish]
+<= monitor-publish
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
 {% if 'neo' in storage_dict -%}
@@ -340,11 +352,25 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {{   name }} = {{ value }}
 {% endfor -%}
 
-{{ root_common.common_section() }}
+[monitor-instance-parameter]
+monitor-httpd-port = 8386
 
+{{ root_common.common_section() }}
 
 [buildout]
+extends = {{ template_monitor }}
 parts += 
   apache-certificate-authority
   fix-ca-folder
   publish
+  monitor-base
+
+[monitor-conf-parameters]
+monitor-title = ERP5 monitor
+password = ${monitor-htpasswd:passwd}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+

software/slapos-master/software.cfg

@@ -33,7 +33,6 @@ update-command = ${:command}
 eggs +=
   suds
   facebook-sdk
-  google-api-python-client
   spyne
   slapos.core
   
@@ -77,7 +76,6 @@ revision = 2acee53ff365ef1f0db80800eb5f8f22a27546f2
 
 [versions]
 python-memcached = 1.47
-facebook-sdk = 0.4.0
-google-api-python-client = 1.2
+facebook-sdk = 2.0.0
 erp5diff = 0.8.1.6
 

software/slapos-testing/instance.cfg

@@ -40,6 +40,10 @@ recipe = plone.recipe.command
 command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
 update-command = $${:command}
 
+[caucase]
+<= download-source
+repository = ${caucase-repository:location}
+
 [slapos.cookbook]
 <= download-source
 repository = ${slapos.cookbook-repository:location}
@@ -84,6 +88,7 @@ test-list =
   $${slapos.recipe.cmmi:location}
   $${slapos.toolbox:location}
   $${erp5-util:location}
+  $${caucase:location}
 prepend-path = ${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
 environment = environment
 

software/slapos-testing/software.cfg

@@ -5,6 +5,7 @@ extends =
   ../../component/lxml-python/buildout.cfg
   ../../component/libxml2/buildout.cfg
   ../../component/libxslt/buildout.cfg
+  ../../component/bcrypt/buildout.cfg
   ../../component/python-2.7/buildout.cfg
   ../../component/python-setuptools/buildout.cfg
   ../../component/zlib/buildout.cfg
@@ -13,6 +14,7 @@ extends =
   ../../stack/slapos.cfg
 
 parts =
+  caucase-repository
   slapos.cookbook-repository
   slapos.core-repository
   slapos.recipe.template-repository
@@ -30,8 +32,10 @@ eggs =
   ${lxml-python:egg}
   ${python-cryptography:egg}
   ${pycurl:egg}
+  ${bcrypt:egg}
   dnspython
   Jinja2
+  caucase
   erp5.util
   slapos.cookbook
   collective.recipe.template
@@ -50,6 +54,10 @@ git-executable = ${git:location}/bin/git
 forbid-download-cache = true
 branch = master
 
+[caucase-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/caucase.git
+
 [slapos.cookbook-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.git
@@ -81,7 +89,7 @@ repository = https://lab.nexedi.com/nexedi/erp5.git
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 51c82ee9139e26c7ab35be65263130c4
+md5sum = 63253c24c5c09716a37f457f2a6b3b23
 output =  ${buildout:directory}/template.cfg
 mode = 640
 
@@ -89,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.7
+slapos.recipe.template = 3.0

software/slaprunner/buildout.hash.cfg

@@ -15,27 +15,27 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 02755403c9f0b52d717160b0b2d0cfb7
+md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 07905ec872e84bc42c6ab089f5270ac2
+md5sum = 8f49df215a1596efcec94e2cca009711
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = 5db39d7f56fd1f96ce6466e9055841a1
+md5sum = 130193114cbbcd014af9704851410605
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
-md5sum = 9db9957f452bda370cb2d5cc2e833e85
+md5sum = 5cfa49bcf20612844e1c50a85740d0b3
 
 [template-runner-export-script]
 filename = template/runner-export.sh.jinja2
-md5sum = 86a429492dba25364f6ec2b318ba1f85
+md5sum = ef87691a9675beb646ae1c99ef69e551
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 852a2ed99af566d27e5e4403334a3376
+md5sum = 00ddcaa6e571505d1930612d12de8fde
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2
@@ -65,6 +65,10 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
 filename = template/bash_profile.in
 md5sum = 1c88cbca0c1e705eeb4b544ef4616097
 
+[template-bashrc]
+filename = template/bashrc.in
+md5sum = d0b9060ec96d4d5b188e36c84240ea7d
+
 [template-supervisord]
 filename = template/supervisord.conf.in
 md5sum = d294d0dafd265048399de6da8c96345f
@@ -80,3 +84,7 @@ md5sum = acaac32cf1bd45714272468a89f4f119
 [template-resilient-software-release-information]
 filename = template/resilient_software_release_information.py.in
 md5sum = 2451072826a9ad9425d62c9e9c7f6284
+
+[template-slapuser-script]
+filename = template/slapos-slapuser-script.in
+md5sum = becafae59ab9973724e0edb238f4cb7a

software/slaprunner/common.cfg

@@ -34,6 +34,7 @@ common-parts =
   instance-runner-import
   instance-runner-export
   template-slapos-cfg
+  template-slapuser-script
 # XXX: we have to manually add this for resilience
   rdiff-backup
   pbs-recipe-egg
@@ -64,6 +65,7 @@ download-only = true
 # Downloads from template directory into current directory
 < = download-only-base
 url = ${:_profile_base_location_}/template/${:filename}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [template]
 < = template-base
@@ -77,14 +79,14 @@ output = ${buildout:directory}/template-runner.cfg.in
 < = template-download-base
 filename = runner-import.sh.jinja2
 
-[instance-runner-import]
-< = template-base
-output = ${buildout:directory}/instance-runner-import.cfg
-
 [template-runner-export-script]
 < = template-download-base
 filename = runner-export.sh.jinja2
 
+[instance-runner-import]
+< = download-base
+recipe = slapos.recipe.build:download
+
 [instance-runner-export]
 < = download-base
 recipe = slapos.recipe.build:download
@@ -102,41 +104,43 @@ recipe = slapos.recipe.build:download
 [template_launcher]
 < = download-base
 recipe = slapos.recipe.build:download
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [template-slapos-cfg]
 < = template-download-base
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = slapos.cfg.in
 
 [template-parameters]
 < = download-only-base
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
 [template-bash-profile]
 < = template-download-base
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = bash_profile.in
 
 [template-supervisord]
 < = template-download-base
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = supervisord.conf.in
 
 [template-listener-slapgrid]
 < = template-download-base
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = listener_slapgrid.py.in
 
 [monitor-check-webrunner-internal-instance]
 < = template-download-base
-destination = ${buildout:parts-directory}/${:filename}
+destination = ${:location}/${:filename}
 filename = monitor-check-webrunner-internal-instances.py
 
 [template-resilient-software-release-information]
 < = template-download-base
 filename = resilient_software_release_information.py.in
 
+[template-slapuser-script]
+< = template-download-base
+filename = slapos-slapuser-script.in
+
+[template-bashrc]
+< = template-download-base
+filename = bashrc.in
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =

software/slaprunner/instance-runner-export.cfg.in

@@ -70,7 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
 wrapper = ${:rendered}
 mode = 700
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   section directory directory
+  raw  backup_wait_time {{ backup_wait_time }}
   raw  output_log_file ${directory:log}/resilient.log
   raw  shell_binary {{ bash_executable_location }}
   raw  rsync_binary {{ rsync_executable_location }}
@@ -81,11 +84,25 @@ minimum = 8437
 maximum = 8446
 ip = ${slap-network-information:global-ipv6}
 
+# XXX Redefine Monitor parameters for runner-export
 [monitor-instance-parameter]
 monitor-httpd-port = ${monitor-httpd-free-port:port}
-# Pass some parameter to dispay in monitoring interface
+{% if slapparameter_dict.get('name', '') -%}
+monitor-title = {{ slapparameter_dict['name'] }}
+{% endif -%}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{% if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-url-list', '') -%}
+monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
+{% endif -%}
 instance-configuration =
   httpdcors cors-domain ${slaprunner-httpd-cors:location} ${httpd-graceful-wrapper:output}
+configuration-file-path = ${buildout:directory}/knowledge0.cfg
 
 # Extends publish section with resilient parameters
 [publish-connection-information]

software/slaprunner/instance-runner-import.cfg.in

 [buildout]
-extends = template-runner.cfg
-          ${pbsready-import:output}
+extends = {{ template_runner_path }}
+          {{ pbsready_import_template_path }}
 
 parts +=
   nginx_conf
@@ -42,26 +42,26 @@ parts +=
 recipe = slapos.cookbook:free_port
 minimum = 49990
 maximum = 49999
-ip = $${slap-network-information:local-ipv4}
+ip = ${slap-network-information:local-ipv4}
 
 [runner-free-port]
 recipe = slapos.cookbook:free_port
 minimum = 50015
 maximum = 50024
-ip = $${slap-network-information:global-ipv6}
+ip = ${slap-network-information:global-ipv6}
 
 [slaprunner]
-proxy_port = $${proxy-free-port:port}
-runner_port = $${runner-free-port:port}
+proxy_port = ${proxy-free-port:port}
+runner_port = ${runner-free-port:port}
 
 [supervisord-free-port]
 recipe = slapos.cookbook:free_port
 minimum = 39996
 maximum = 40005
-ip = $${slaprunner:ipv4}
+ip = ${slaprunner:ipv4}
 
 [supervisord]
-port = $${supervisord-free-port:port}
+port = ${supervisord-free-port:port}
 
 [runner-sshd-port]
 minimum = 22232
@@ -74,17 +74,19 @@ recipe =
 
 [importer]
 recipe = slapos.recipe.template:jinja2
-template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
-rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
+template = {{ importer_script_path }}
+rendered = ${directory:bin}/${slap-parameter:namebase}-importer
 # backward compatibility for resilient stack
-wrapper = $${:rendered}
+wrapper = ${:rendered}
 mode = 700
-restore-exit-code-file = $${directory:srv}/$${:restore-exit-code-file-basename}
+restore-exit-code-file = ${directory:srv}/${:restore-exit-code-file-basename}
 restore-exit-code-file-basename = importer-exit-code-file
-restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-basename}
+restore-error-message-file = ${directory:srv}/${:restore-error-message-file-basename}
 restore-error-message-file-basename = importer-error-message-file
 resilient-log-basename = resilient.log
 context =
+  import sys sys
+  import easy_install zc.buildout.easy_install
   key backend_url slaprunner:access-url
   key ipv4 slaprunner:ipv4
   key ipv6 slaprunner:ipv6
@@ -92,20 +94,20 @@ context =
   key instance_folder slaprunner:instance_root
   section directory directory
   section supervisord supervisord
-  raw  output_log_file $${directory:log}/$${:resilient-log-basename}
-  raw  shell_binary ${bash:location}/bin/bash
-  raw  rsync_binary ${rsync:location}/bin/rsync
-  raw  restore_exit_code_file $${:restore-exit-code-file}
-  raw  restore_error_message_file $${:restore-error-message-file}
+  raw  output_log_file ${directory:log}/${:resilient-log-basename}
+  raw  shell_binary {{ bash_executable_location }}
+  raw  rsync_binary {{ rsync_executable_location }}
+  raw  restore_exit_code_file ${:restore-exit-code-file}
+  raw  restore_error_message_file ${:restore-error-message-file}
 
 [importer-consistency-promise]
 # Test that the importer script and "after-import" subscripts
 # are not older than 2 days (1 day + some slack), and have succeeded
 recipe = collective.recipe.template
 input = inline: #!/bin/sh
-  EXIT_CODE_FILE="$${importer:restore-exit-code-file}"
-  RECENT_EXIT_CODE_FILE=$(find $${directory:srv} -maxdepth 1 -name "$${importer:restore-exit-code-file-basename}" -mtime -2)
-  RESILIENT_LOG_URL=$${publish:monitor-base-url}/log/$${importer:resilient-log-basename}
+  EXIT_CODE_FILE="${importer:restore-exit-code-file}"
+  RECENT_EXIT_CODE_FILE=$(find ${directory:srv} -maxdepth 1 -name "${importer:restore-exit-code-file-basename}" -mtime -2)
+  RESILIENT_LOG_URL=${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
   if [ ! -f "$EXIT_CODE_FILE" ]; then
     exit 0;
   else
@@ -113,23 +115,25 @@ input = inline: #!/bin/sh
        echo "Consistency check is too old.";
        exit 1;
     else
-      cat $${importer:restore-error-message-file}
+      if [ -f "${importer:restore-error-message-file}" ]; then
+        cat ${importer:restore-error-message-file}
+      fi
       echo "More information can be found here : $RESILIENT_LOG_URL";
       exit $(cat $EXIT_CODE_FILE);
     fi
   fi
   exit 1; # Something else went wrong
-output = $${directory:promises}/importer-consistency-promise
+output = ${directory:promises}/importer-consistency-promise
 mode = 755
 
 [software-release-deployment-promise]
 recipe = collective.recipe.template
 input = inline: #!/bin/sh
-  PROJECT_FILE=$(find "$${directory:etc}" -maxdepth 1 -name .project)
+  PROJECT_FILE=$(find "${directory:etc}" -maxdepth 1 -name .project)
   if [ -z "$PROJECT_FILE" ]; then
     exit 0;
   else
-    INSTALLATION_FILE=$(find "$${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
+    INSTALLATION_FILE=$(find "${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
     if [ -n "$INSTALLATION_FILE" ]; then
       exit 0;
     else
@@ -137,41 +141,41 @@ input = inline: #!/bin/sh
     fi
   fi
   exit 1
-output = $${directory:promises}/software-release-deployment-promise
+output = ${directory:promises}/software-release-deployment-promise
 mode = 755
 
 [resilient-software-release-information]
 recipe = slapos.recipe.template
-url = ${template-resilient-software-release-information:destination}/${template-resilient-software-release-information:filename}
-output = $${directory:cgi-bin}/resilient_software_release_information.py
-resilient-log-url = $${publish:monitor-base-url}/log/$${importer:resilient-log-basename}
+url = {{ software_release_information_template }}
+output = ${directory:cgi-bin}/resilient_software_release_information.py
+resilient-log-url = ${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
 mode = 0600
 
 [slap-parameter]
 auto-deploy-instance = false
 auto-deploy = true
-name = Webrunner import
-monitor-cors-domains = 
-monitor-username = $${monitor-htpasswd:username}
-monitor-password = $${monitor-htpasswd:passwd}
 
 [resilient-publish-connection-parameter]
-monitor-base-url = $${publish:monitor-base-url}
-monitor-url = $${publish:monitor-url}
-monitor-user = $${publish:monitor-user}
-monitor-password = $${publish:monitor-password}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-url = ${monitor-publish-parameters:monitor-url}
+monitor-user = ${monitor-publish-parameters:monitor-user}
+monitor-password = ${monitor-publish-parameters:monitor-password}
 
 [monitor-instance-parameter]
 monitor-httpd-port = 8360
-#monitor-title = $${slap-parameter:name}
-#cors-domains = $${slap-parameter:monitor-cors-domains}
-#username = $${slap-parameter:monitor-username}
-#password = $${slap-parameter:monitor-password}
-# Pass some parameter to dispay in monitoring interface
+monitor-title = {{ slapparameter_dict.get('name', 'Webrunner import') }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
+{% if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{% endif -%}
+{% if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{% endif -%}
 instance-configuration =
-  raw takeover-url http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
-  raw takeover-password $${resilient-web-takeover-password:passwd}
+  raw takeover-url http://[${resilient-web-takeover-httpd-configuration-file:listening-ip}]:${resilient-web-takeover-httpd-configuration-file:listening-port}/
+  raw takeover-password ${resilient-web-takeover-password:passwd}
+configuration-file-path = ${buildout:directory}/knowledge0.cfg
 
 [monitor-conf-parameters]
 private-path-list += 
-  $${directory:logrotate-backup}
+  ${directory:logrotate-backup}

software/slaprunner/instance-runner-input-schema.json

@@ -57,11 +57,6 @@
             "type": "integer",
             "default" : 4
         },
-        "instance-name": {
-            "title": "Displayed instance name",
-            "description": "Name of the instance, to show in the window title",
-            "type": "string"
-        },
         "no-ipv4-frontend": {
             "title": "No IPv4 frontend",
             "description": "Prevent the slaprunner to order an IPv4 frontend for itself",
@@ -129,12 +124,6 @@
             "type": "string",
             "default": "monitor.app.officejs.com"
         },
-        "resilient-clone-number": {
-            "title": "Number of Resilient Clones",
-            "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
-            "type": "integer",
-            "enum": [1, 2]
-        },
         "instance-name": {
             "title": "Instance Name",
             "description": "Name of the instance, to show in the window title",

software/slaprunner/instance-runner-resilient-input-schema.json

@@ -12,7 +12,8 @@
                     "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                     "type": "integer",
                     "default": 1,
-                    "enum": [1, 2],
+                    "minimum": 0,
+                    "maximum": 2,
                     "optional": true
                 },
                 "-sla-runner0-computer_guid": {

software/slaprunner/instance-runner.cfg

@@ -38,6 +38,8 @@ parts =
 ## Monitoring part
   monitor-base
   monitor-check-webrunner-internal-instance
+## Usability part
+  template-slapuser-script
 
 extends = ${monitor2-template:rendered}
 
@@ -114,7 +116,7 @@ home = $${directory:srv}/runner/
 test = $${directory:srv}/test/
 project = $${:home}/project
 public = $${:home}/public
-software-root = $${:home}/software
+software-root = {{ slapparameter_dict.get('software-root', '$${:home}/software') }}
 instance-root = $${:home}/instance
 project-test = $${:test}/project
 software-test = $${:test}/software
@@ -544,7 +546,7 @@ init-password = $${runner-htpasswd:password}
 ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
 git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
 git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
-monitor-base-url = $${publish:monitor-base-url}
+monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
 {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' -%}
 url =  https://$${request-frontend:connection-domain}
 webdav-url = $${request-httpd-frontend:connection-secure_access}/share/
@@ -557,11 +559,11 @@ public-url =  $${apache-httpd:access-url}/public/
 
 {% if slapparameter_dict.get('instance-type', '') != 'resilient' -%}
 {% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
-monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${publish:monitor-url}&username=$${publish:monitor-user}&password=$${publish:monitor-password}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${monitor-publish-parameters:monitor-url}&username=$${monitor-publish-parameters:monitor-user}&password=$${monitor-publish-parameters:monitor-password}
 {% else -%}
-monitor-url = $${publish:monitor-url}
-monitor-user = $${publish:monitor-user}
-monitor-password = $${publish:monitor-password}
+monitor-url = $${monitor-publish-parameters:monitor-url}
+monitor-user = $${monitor-publish-parameters:monitor-user}
+monitor-password = $${monitor-publish-parameters:monitor-password}
 {% endif -%}
 #---------------------------
 #--
@@ -712,7 +714,13 @@ rendered = $${buildout:directory}/.bash_profile
 context =
     raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin
     key workdir runnerdirectory:home
-
+    
+[bashrc]
+recipe = slapos.recipe.template:jinja2
+template = ${template-bashrc:location}/${template-bashrc:filename}
+rendered = $${buildout:directory}/.bashrc
+context =
+    raw bin_folder ${buildout:bin-directory}
 
 #---------------------------
 #--
@@ -801,12 +809,8 @@ password = $${runner-htpasswd:password}
 {% if slapparameter_dict.get('monitor-url-list', '') -%}
 monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
 {% endif -%}
-# check if not resilient runner
-{% if not slapparameter_dict.get('authorized-key', '') -%}
-# Pass some parameter to dispay in monitoring interface
 instance-configuration =
   httpdcors cors-domain $${slaprunner-httpd-cors:location} $${httpd-graceful-wrapper:output}
-{% endif -%}
 configuration-file-path = $${buildout:directory}/knowledge0.cfg
 
 [monitor-check-webrunner-internal-instance]
@@ -816,3 +820,12 @@ rendered = $${monitor-directory:reports}/$${:filename}
 filename = monitor-check-webrunner-internal-instance
 mode = 0744
 
+## Slapuser slapos command script
+[template-slapuser-script]
+recipe = slapos.recipe.template:jinja2
+template = ${template-slapuser-script:location}/${template-slapuser-script:filename}
+rendered = $${buildout:bin-directory}/slapos
+mode = 0744
+context =
+  raw config_location $${slapos-cfg:rendered}
+  raw slapos_python_file_location ${buildout:bin-directory}/slapos

software/slaprunner/instance.cfg

@@ -12,7 +12,7 @@ recipe = slapos.cookbook:softwaretype
 default = $${instance-base-runner:rendered}
 resilient = $${instance-resilient:rendered}
 runner = $${instance-base-runner:rendered}
-runner-import = ${instance-runner-import:output}
+runner-import = $${template-runner-import:rendered}
 runner-export = $${template-runner-export:rendered}
 
 frozen = ${instance-frozen:output}
@@ -55,12 +55,29 @@ mode = 640
 context =
   key pbsready_export_template_path template-pbsready-export:rendered
   key template_runner_path instance-base-runner:rendered
+  key slapparameter_dict slap-configuration:configuration
+  raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
   raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
   raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
   raw buildout_executable_location ${buildout:executable}
   raw bash_executable_location ${bash:location}/bin/bash
   raw rsync_executable_location ${rsync:location}/bin/rsync
 
+[template-runner-import]
+recipe = slapos.recipe.template:jinja2
+template = ${instance-runner-import:target}
+rendered = $${buildout:directory}/instance-runner-import.cfg
+mode = 640
+context =
+  key template_runner_path instance-base-runner:rendered
+  key slapparameter_dict slap-configuration:configuration
+  raw importer_script_path ${template-runner-import-script:location}/${template-runner-import-script:filename}
+  raw buildout_executable_location ${buildout:executable}
+  raw bash_executable_location ${bash:location}/bin/bash
+  raw rsync_executable_location ${rsync:location}/bin/rsync
+  raw pbsready_import_template_path ${pbsready-import:output}
+  raw software_release_information_template ${template-resilient-software-release-information:destination}/${template-resilient-software-release-information:filename}
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}

software/slaprunner/software.cfg

@@ -16,42 +16,42 @@ gitdb = 0.6.4
 gunicorn = 19.4.5
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 2.9
-slapos.toolbox = 0.67
+slapos.recipe.template = 3.0
+slapos.toolbox = 0.69
 smmap = 0.9.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 GitPython = 2.0.8
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 PyRSS2Gen = 1.1
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 dnspython = 1.14.0
 
 # Required by:
-# slapos.toolbox==0.61
-erp5.util = 0.4.46
+# slapos.toolbox==0.69
+erp5.util = 0.4.49
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 feedparser = 5.2.1
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 lockfile = 0.12.2
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 paramiko = 2.0.2
 
 # Required by:
-# slapos.toolbox==0.61
+# slapos.toolbox==0.69
 passlib = 1.6.5

software/slaprunner/template/bashrc.in

+export PATH={{ bin_folder }}:$PATH

software/slaprunner/template/runner-export.sh.jinja2

@@ -4,71 +4,105 @@ export LC_ALL
 umask 077
 
 # Exit on any error, to prevent inconsistent backup
-set -e
+# Error on unset variable expansion
+set -eu
 
 # Redirect output to log
-exec > >(tee -ai {{ output_log_file }})
+exec > >(tee -ai '{{ output_log_file }}')
 exec 2>&1
 
 echo -e "\n\n$0 run at : $(date)"
 
-srv_directory={{ directory['srv'] }}
-
-sync_element () {
-  path=$1
-  backup_path=$2
-  shift 2
-  element_list=$*
-
-  # Concatenate the exclude file of each partition of webrunner
-  # to create a global exclude file.
-  exclude_content="instance/supervisord.socket"
-  for partition in $srv_directory/runner/instance/slappart*
-  do
-    exclude_file="$partition/srv/exporter.exclude"
-    if [ -e "$exclude_file" ]; then
-      partition_exclude_content_relative=$(cat "$exclude_file")
-      # For every line of the local exclude file, add the absolute path
-      for line in "$partition_exclude_content_relative"
-      do
-        if [ ! -z "$line" ]; then
-          exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
-       fi
-      done
-    fi
-  done
-  echo -e "$exclude_content" > $srv_directory/exporter.exclude
-
-  for element in $element_list
-  do
-    echo "Changing current directory to $path."
-    cd $path;
-    if [ -f $element ] || [ -d $element ]; then
-       command="{{ rsync_binary }} -rlptgov --stats --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path"
-       echo "Running : $command"
-       $command
-    fi
-  done
+srv_directory='{{ directory["srv"] }}'
+backup_directory='{{ directory["backup"] }}'
+etc_directory='{{ directory["etc"] }}'
+tmp_directory='{{ directory["tmp"] }}'
+
+rsync () {
+  set -x
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
+  set +x
 }
-sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
+
+(
+  # XXX: code duplication with runner-import.sh.jinja2
+  path=$srv_directory/runner
+  backup_path=$backup_directory/runner/
+  cd "$path"
+
+  if [ -d instance ]; then
+    # Concatenate the exclude file of each partition of webrunner
+    # to create a global exclude file.
+    # Also, ignore all buildout-managed files.
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
+  fi
+
+  test -d project  && rsync project "$backup_path"
+  test -d public  && rsync public "$backup_path"
+  test -f proxy.db && rsync proxy.db "$backup_path"
+)
 # We sync .* appart
-date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
-sync_element {{ directory['etc'] }}  {{ directory['backup'] }}/etc/ config.json
-# Hidden files are related to the webrunner's internals
-cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
-if [ -d {{ directory['backup'] }}/runner/software ]; then
-  rm {{ directory['backup'] }}/runner/software/*
+(
+  cd "$etc_directory"
+  date +%s -u > .resilient-timestamp
+  rsync config.json "$backup_directory"/etc/
+  # Hidden files are related to the webrunner's internals
+  cp -r .??* "$backup_directory/etc/"
+)
+if [ -d "$backup_directory"/runner/software ]; then
+  rm "$backup_directory"/runner/software/*
 fi
 
-cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
+(
+  cd "$backup_directory"
+  find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
+)
 
 # Check that export didn't happen during backup of instances
-tmp_backup_sum=$(mktemp -p {{ directory['tmp'] }})
-tmp_filtered_signature=$(mktemp -p {{ directory['tmp'] }})
+tmp_backup_sum=$(mktemp -p "$tmp_directory")
+tmp_filtered_signature=$(mktemp -p "$tmp_directory")
 
 remove_tmp_files () {
-  rm $tmp_backup_sum
-  rm $tmp_filtered_signature
+  rm "$tmp_backup_sum" "$tmp_filtered_signature"
 }
 trap remove_tmp_files EXIT
 
@@ -83,15 +117,14 @@ if [ -z "$backup_directory_path" ]; then
 fi
 
 sleep 5
-sha256sum $backup_directory_path | LC_ALL=C sort -k 66 > $tmp_backup_sum
-cat {{ directory['backup'] }}/backup.signature | egrep "instance/slappart.*/srv/backup/" > $tmp_filtered_signature
+sha256sum $backup_directory_path | sort -k 66 > "$tmp_backup_sum"
+egrep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" > "$tmp_filtered_signature"
 
 # If the diff fails, then the notifier will restart this script
-diff_status=0
-diff $tmp_backup_sum $tmp_filtered_signature || diff_status=1
-if [ $diff_status -ne 0 ]; then
-  echo "ERROR: Some backups are not consistents, exporter should be re-run."
-  echo "Let's sleep 10 minutes, to let the backup end..."
-  sleep 10m
-  exit 1
+if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
+  exit 0
 fi
+echo "ERROR: Some backups are not consistent, exporter should be re-run."
+echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
+sleep {{ backup_wait_time }}m
+exit 1

software/slaprunner/template/runner-import.sh.jinja2

 #!{{ shell_binary }}
-set -e
 LC_ALL=C
 export LC_ALL
 umask 077
 
+# Exit on any error, to prevent inconsistent backup
+# Error on unset variable expansion
+set -eu
+
 # Redirect output to log
-exec > >(tee -ai {{ output_log_file }})
+exec > >(tee -ai '{{ output_log_file }}')
 exec 2>&1
 
-RESTORE_EXIT_CODE_FILE="{{ restore_exit_code_file }}"
-RESTORE_ERROR_MESSAGE_FILE="{{ restore_error_message_file }}"
+echo -e "\n\n$0 run at : $(date)"
+
+srv_directory='{{ directory["srv"] }}'
+backup_directory='{{ directory["backup"] }}'
+etc_directory='{{ directory["etc"] }}'
+
+RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
+RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
 ERROR_MESSAGE=""
 
 fail_with_exit_code () {
@@ -17,7 +26,6 @@ fail_with_exit_code () {
   echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
   exit 1
 }
-
 trap fail_with_exit_code ERR
 
 log_message () {
@@ -25,111 +33,160 @@ log_message () {
     echo -e $1
 }
 # Delete the error message file, to not keep it even after a successful build
-rm $RESTORE_ERROR_MESSAGE_FILE || true
-
-srv_directory={{ directory['srv'] }}
-restore_element () {
-  backup_path=$1
-  restore_path=$2
-  shift 2
-  element_list=$*
-  for element in $element_list
-  do
-    cd $backup_path;
-    if [ -f $element ] || [ -d $element ]; then
-       command="{{ rsync_binary }} --stats -av --delete --exclude *.sock --exclude *.socket --exclude *.pid --exclude .installed.cfg --exclude .installed-switch-softwaretype.cfg $backup_path/$element $restore_path"
-       echo "Running: \"$command\""
-       $command
-    fi
-  done
-}
+rm "$RESTORE_ERROR_MESSAGE_FILE" || true
 
-echo -e "\n\nrunner-import run at : $(date)"
+rsync () {
+  set -x
+  '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
+  set +x
+}
 
 log_message "Restoring WebRunner content..."
-restore_element {{ directory['backup'] }}/runner/ $srv_directory/runner  instance project  proxy.db
+(
+  # XXX: code duplication with runner-export.sh.jinja2
+  path=$srv_directory/runner
+  backup_path=$backup_directory/runner/
+  cd "$backup_path"
+
+  if [ -d instance ]; then
+    # Concatenate the exclude file of each partition of webrunner
+    # to create a global exclude file.
+    # Also, ignore all buildout-managed files.
+    exclude=$({{ sys.executable }} - "$path" <<EOF
+if 1:
+        import glob, errno, os, sys
+        sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
+        from zc.buildout.configparser import parse
+        path = sys.argv[1]
+
+        def print_relative(path_list):
+            for p in path_list:
+                p = p.strip()
+                if p:
+                    print(os.path.relpath(p, path))
+        print("*.sock")
+        print("*.socket")
+        print("*.pid")
+        print(".installed*.cfg")
+        for partition in glob.glob(path + "/instance/slappart*"):
+            os.chdir(partition)
+            try:
+                with open("srv/exporter.exclude") as f:
+                    exclude = f.readlines()
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+            else:
+                print_relative(exclude)
+            for installed in glob.glob(".installed*.cfg"):
+                try:
+                    with open(installed) as f:
+                        installed = parse(f, installed)
+                except IOError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                else:
+                    for section in installed.itervalues():
+                        print_relative(section.get(
+                            '__buildout_installed__', '').splitlines())
+EOF
+)
+    echo "$exclude" |rsync --exclude-from=- instance "$path"
+  fi
 
-log_message "Restoring WebRunner config (etc directory)..."
-restore_element  {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json
-cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
+  test -d project  && rsync project "$path"
+  test -d public  && rsync public "$path"
+  test -f proxy.db && rsync proxy.db "$path"
+)
 
+log_message "Restoring WebRunner config (etc directory)..."
+(
+  cd "$backup_directory"/etc/
+  rsync config.json "$etc_directory"
+  # Hidden files are related to the webrunner's internals
+  cp -r .??* "$etc_directory"
+)
 
 # Invoke arbitrary script to perform specific restoration
 # procedure.
 runner_import_restore=$srv_directory/runner-import-restore
-if [ ! -e "$runner_import_restore" ]; then
-  touch $runner_import_restore
-  chmod +x $runner_import_restore
+if [ -x "$runner_import_restore" ]; then
+  log_message "Running $runner_import_restore..."
+  "$srv_directory/runner-import-restore"
 fi
-log_message "Running $runner_import_restore..."
-$srv_directory/runner-import-restore
 
 # If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume
-# that there is no instnace deployed on runner0
-if [ ! -f "directory['etc']/.project" ] && [ ! -f "$srv_directory/runner/proxy.db" ]; then
+# that there is no instance deployed on runner0
+if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then
   log_message "No Software Requested... Writing status file... End"
   echo 0 > $RESTORE_EXIT_CODE_FILE
   exit 0
 fi
 
 log_message "Updating slapproxy database..."
-HOME="{{ directory['home'] }}"
+HOME='{{ directory["home"] }}'
 # XXX Hardcoded
 export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 export MAKEFLAGS=-j4
-SLAPOS="{{ directory['bin'] }}/slapos"
+SLAPOS='{{ directory["bin"] }}'/slapos
 # XXX hardcoded
 SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
 DATABASE="$HOME/srv/runner/proxy.db"
 # Change slapproxy database to point instances to new software release
 # XXX hardcoded
 PARTITION=$(basename $HOME)
-OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
-SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9]\?/\1'"$PARTITION"'/')
-$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
-$SQLITE3 $DATABASE "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || $SQLITE3 $DATABASE "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
+OLD_SOFTWARE_RELEASE=$("$SQLITE3" "$DATABASE" "select software_release from partition11 where reference='slappart0';")
+SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9][0-9]\?/\1'"$PARTITION"'/')
+"$SQLITE3" "$DATABASE" "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
+"$SQLITE3" "$DATABASE" "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || "$SQLITE3" "$DATABASE" "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
 # Change slapproxy database to have all instances stopped
-$SQLITE3 $DATABASE "update partition11 set requested_state='stopped';"
+"$SQLITE3" "$DATABASE" "update partition11 set requested_state='stopped';"
 # Change slapproxy database to get correct IPs
-IPV4={{ ipv4 }}
-IPV6={{ ipv6 }}
-$SQLITE3 $DATABASE "update partition_network11 set address='$IPV4' where netmask='255.255.255.255';"
-$SQLITE3 $DATABASE "update partition_network11 set address='$IPV6' where netmask='ffff:ffff:ffff::';"
+IPV4='{{ ipv4 }}'
+IPV6='{{ ipv6 }}'
+"$SQLITE3" "$DATABASE" "update partition_network11 set address='$IPV4' where netmask='255.255.255.255';"
+"$SQLITE3" "$DATABASE" "update partition_network11 set address='$IPV6' where netmask='ffff:ffff:ffff::';"
 
-MASTERURL="http://{{ ipv4 }}:{{ proxy_port }}"
+MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
 
 log_message "Removing old supervisord service description files..."
 # XXX: Path hardcoded in slapos.core
-rm {{ instance_folder }}/etc/supervisord.conf.d/* || true
+rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
+
+SLAPOSCFG='{{ supervisord["slapos-cfg"] }}'
+SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}'
+SLAPGRIDSRPID='{{ supervisord["slapgrid-sr-pid"] }}'
+SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}'
+SLAPGRIDCPPID='{{ supervisord["slapgrid-cp-pid"] }}'
 
 log_message "Building newest Software Release..."
-$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
-$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
-$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
-(tail -n 200 {{ supervisord['slapgrid-sr-log'] }} && false)
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
+"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
+(tail -n 200 "$SLAPGRIDSRLOG" && false)
 # Remove defined scripts to force buildout to recreate them to have updated paths
-rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore || true
+rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
 log_message "Fixing Instances as needed after import..."
 # XXX hardcoded
-$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
-$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
-$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
-(tail -n 200 {{ supervisord['slapgrid-cp-log'] }} && false)
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
+"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
+(tail -n 200 "$SLAPGRIDCPLOG" && false)
 
 # Invoke defined scripts for each partition inside of slaprunner
 log_message "Invoke custom import scripts defined by each instances..."
-for partition in $srv_directory/runner/instance/slappart*/
+for partition in "$srv_directory"/runner/instance/slappart*/
 do
   script=$partition/srv/runner-import-restore
-  if [ -e "$script" ]; then
+  if [ -x "$script" ]; then
     log_message "Running custom instance script : $script..."
-    $script
+    "$script"
   fi
 done
 
 # Change back slapproxy database to have all instances started
 log_message "Set instances as to start after takeover..."
-$SQLITE3 $DATABASE "update partition11 set requested_state='started';"
+"$SQLITE3" "$DATABASE" "update partition11 set requested_state='started';"
 
 # Write exit code to an arbitrary file that will be checked by promise/monitor
 log_message "Writing status file... End"

software/slaprunner/template/slapos-slapuser-script.in

+#!/bin/sh
+# run slapos command inside slaprunner with appropriate config
+
+SLAPOS_CONFIGURATION={{ config_location }}   \
+SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION   \
+exec {{ slapos_python_file_location }} "$@"

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 2.7
+slapos.recipe.template = 3.0
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]
@@ -138,5 +138,5 @@ setuptools = 0.9.8
 prettytable = 0.7.3-nxd001
 
 [networkcache]
-download-cache-url = http://www.shacache.org/shacache
-download-dir-url = http://www.shacache.org/shadir
+download-cache-url = http://download.shacache.org
+download-dir-url = http://dir.shacache.org
\ No newline at end of file

software/unstable/lte/instance.jinja2.cfg

+#
+# Deploy LTE instance
+#
+[buildout]
+parts =
+  directory
+  lte
+
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run
+service = ${:etc}/service
+promise = ${:etc}/promise
+log = ${:var}/log
+
+[lte]
+recipe = slapos.recipe.build:download
+url = {{ buildout['directory'] }}/lte.sh
+destination = ${directory:service}/lte.sh
+mode = 0775
+md5sum = 9de4c400d411083d8ecd8d03d62b97f0

software/unstable/lte/lte.sh

+#!/bin/bash
+
+function stopLTE {
+  sudo /bin/systemctl stop lte
+  echo "LTE service stopped"
+  exit 0
+}
+trap stopLTE TERM INT KILL
+
+sudo /bin/systemctl start lte
+
+while (( 1 )); do 
+  sleep 1
+done

software/unstable/lte/readme.rst

+LTE eNodeB software release
+###########################
+
+Simple software release which starts LTE service upon instantiation.
+LTE is managed by systemd and we are taking advantage of this.
+
+It is necessary to install SlapOS from "amarisoft" branch because
+it has modified ``slapos node format`` to give group ``slapsoft``
+the rights to operate ``systemctl <start/stop/enable/disable> lte``.

software/unstable/lte/software.cfg

+[buildout]
+
+extends =
+  ../../../stack/slapos.cfg
+
+parts =
+  slapos-cookbook
+  instance-profile
+  lte.sh
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.jinja2.cfg
+rendered = ${buildout:directory}/instance.cfg
+md5sum = 3bbbb5297db5dd49f5dfc158e4291d4c
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  section buildout buildout
+  
+[copy-to-instance]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_buildout_section_name_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+[lte.sh]
+<= copy-to-instance
+md5sum = 9de4c400d411083d8ecd8d03d62b97f0

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 2.9
+slapos.recipe.template = 3.0

stack/boinc/buildout.cfg

@@ -69,7 +69,7 @@ pycrypto = 2.6
 apache-libcloud = 0.12.4
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10
+mysqlclient = 1.3.10+SlapOSPatched001
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.4.2
 slapos.toolbox = 0.40.4

stack/caucase/buildout.cfg

+[buildout]
+
+extends =
+  buildout.hash.cfg
+  ../../component/apache/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/bcrypt/buildout.cfg
+  ../../stack/logrotate/buildout.cfg
+
+
+parts =
+  instance-caucase
+
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = python_ca
+eggs =
+  gunicorn # for WSGI HTTP Server
+  futures
+  caucase # certificate authority
+  ${bcrypt:egg}
+# are also required
+  plone.recipe.command
+  collective.recipe.template
+  slapos.toolbox
+scripts =
+  slapos-kill
+  gunicorn
+  caucase
+  caucase-cli
+  caucase-cliweb
+
+[template-ca-download-base]
+recipe = hexagonit.recipe.download
+ignore-existing = true
+download-only = true
+url = ${:_profile_base_location_}/${:filename}
+mode = 0644
+
+[template-httpd-auth-conf]
+<= template-ca-download-base
+
+[template-nginx-ca-conf]
+<= template-ca-download-base
+
+[template-caucase]
+<= template-ca-download-base
+url = ${:_profile_base_location_}/${:filename}
+
+[template-authenticated-server]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template-authenticated-server.cfg
+context =
+  key apache_location apache:location
+  key template_logrotate_base template-logrotate-base:rendered
+  raw certificate_request_bin ${buildout:directory}/bin/caucase-cliweb
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw slapos_kill_bin ${buildout:directory}/bin/slapos-kill
+  raw template_httpd_auth_conf ${template-httpd-auth-conf:location}/${template-httpd-auth-conf:filename}
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  raw python_bin ${buildout:directory}/bin/${extra-eggs:interpreter}
+
+[instance-caucase]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template.cfg
+context =
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key nginx_location nginx:location
+  key template_logrotate_base template-logrotate-base:rendered
+  raw caucase_template ${template-caucase:location}/${template-caucase:filename}
+  raw curl_executable_location ${curl:location}/bin/curl
+  raw caucase_bin ${buildout:directory}/bin/caucase
+  raw certificate_request_bin ${buildout:directory}/bin/caucase-cliweb
+  raw template_nginx_ca_conf ${template-nginx-ca-conf:location}/${template-nginx-ca-conf:filename}
+  raw dash_executable_location ${dash:location}/bin/dash
+  raw slapos_kill_bin ${buildout:directory}/bin/slapos-kill
+  raw gunicorn_bin ${buildout:directory}/bin/gunicorn
+  raw openssl_executable_location ${openssl:location}/bin/openssl
+  raw python_bin ${buildout:directory}/bin/${extra-eggs:interpreter}
+
+
+[versions]
+Flask-User = 0.6.11
+SQLAlchemy = 1.1.9
+caucase = 0.1.3
+futures = 3.1.1
+gunicorn = 19.7.1
+slapos.recipe.template = 3.0
+
+APacheDEX = 1.6.2
+bcrypt = 3.1.3
+
+# Required by:
+# caucase==0.1.3
+Flask-AlchemyDumps = 0.0.10
+
+# Required by:
+# Flask-User==0.6.11
+Flask-Login = 0.4.0
+
+# Required by:
+# Flask-User==0.6.11
+Flask-Mail = 0.9.1
+
+# Required by:
+# Flask-AlchemyDumps==0.0.10
+# Flask-User==0.6.11
+Flask-SQLAlchemy = 2.2
+
+# Required by:
+# Flask-AlchemyDumps==0.0.10
+Flask-Script = 2.0.5
+
+# Required by:
+# Flask-User==0.6.11
+Flask-WTF = 0.14.2
+
+# Required by:
+# Flask-AlchemyDumps==0.0.10
+Unipath = 1.1
+
+# Required by:
+# Flask-WTF==0.14.2
+WTForms = 2.1
+
+# Required by:
+# Flask-Mail==0.9.1
+blinker = 1.4
+
+# Required by:
+# caucase==0.1.3
+pem = 16.1.0
+
+# Required by:
+# Flask-User==0.6.11
+pycryptodome = 3.4.6

stack/caucase/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template-httpd-auth-conf]
+filename = template-httpd-auth.conf.in
+md5sum = ea445b0a9b143d12b5700a71ac06293c
+
+[template-nginx-ca-conf]
+filename = ca-nginx.conf.in
+md5sum = d8bebf1629aacffd619541f363687b4a
+
+[template-authenticated-server]
+filename = instance-auth-server.cfg.jinja2.in
+md5sum = a317d2f948cd3d16c860d05cc07ecf42
+
+[template-caucase]
+filename = instance-caucase.cfg.jinja2.in
+md5sum = b801dfe4212ff97dc29191e4610df3a3
+
+[instance-caucase]
+filename = instance.cfg.in
+md5sum = eb9d2ab646717d123b0472da5194d77f

stack/caucase/ca-nginx.conf.in

+worker_processes {{ parameter_dict['workers-processes'] }};
+
+pid {{ parameter_dict['pid-file'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+    # include mime.types;
+    default_type application/octet-stream;
+    access_log {{ parameter_dict['access-log'] }} combined;
+    client_max_body_size 10M;
+    map $http_upgrade $connection_upgrade {
+      default upgrade;
+      ''      close;
+    }
+    sendfile on;
+
+    upstream app_server {
+      # for UNIX domain socket setups
+      server unix:{{ parameter_dict['socket'] }} fail_timeout=0;
+    }
+
+    {% if parameter_dict['cert-file'] and parameter_dict['key-file'] -%}
+    server {
+      listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['https-port'] }} ssl;
+      server_name _;
+      ssl_certificate     {{ parameter_dict['cert-file'] }};
+      ssl_certificate_key {{ parameter_dict['key-file'] }};
+      ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+      ssl_ciphers         ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
+      ssl_prefer_server_ciphers on;
+      keepalive_timeout 90s;
+      client_body_temp_path {{ parameter_dict['client-body-temp-path'] }};
+      proxy_temp_path {{ parameter_dict['proxy-temp-path'] }};
+      fastcgi_temp_path {{ parameter_dict['fastcgi-temp-path'] }};
+      uwsgi_temp_path {{ parameter_dict['uwsgi-temp-path'] }};
+      scgi_temp_path {{ parameter_dict['scgi-temp-path'] }};
+
+      location / {
+          proxy_redirect off;
+          proxy_set_header   X-Forwarded-Proto $scheme;
+          proxy_set_header   X-Real-IP $remote_addr;
+          proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+          proxy_set_header   X-Forwarded-Host  $http_host;
+          proxy_set_header   Host $http_host;
+          proxy_set_header   Authorization $http_authorization;
+          proxy_pass_header  Authorization;
+          proxy_connect_timeout 90;
+          proxy_send_timeout    90;
+          proxy_read_timeout    90;
+          send_timeout          90;
+
+          proxy_pass http://app_server;
+      }
+    }
+    {% endif -%}
+
+    server {
+      listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }};
+      server_name _;
+      keepalive_timeout 90s;
+      client_body_temp_path {{ parameter_dict['client-body-temp-path'] }};
+      proxy_temp_path {{ parameter_dict['proxy-temp-path'] }};
+      fastcgi_temp_path {{ parameter_dict['fastcgi-temp-path'] }};
+      uwsgi_temp_path {{ parameter_dict['uwsgi-temp-path'] }};
+      scgi_temp_path {{ parameter_dict['scgi-temp-path'] }};
+
+      location ~ ^(/admin|/user) {
+        # http is not used for /admin and /user
+      }
+
+      location / {
+          proxy_redirect off;
+          proxy_set_header   X-Forwarded-Proto $scheme;
+          proxy_set_header   X-Real-IP $remote_addr;
+          proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+          proxy_set_header   X-Forwarded-Host  $http_host;
+          proxy_set_header   Host $http_host;
+          proxy_set_header   Authorization $http_authorization;
+          proxy_pass_header  Authorization;
+          proxy_connect_timeout 90;
+          proxy_send_timeout    90;
+          proxy_read_timeout    90;
+          send_timeout          90;
+
+          proxy_pass http://app_server;
+      }
+    }
+}

stack/caucase/instance-auth-server.cfg.jinja2.in

+[buildout]
+
+extends =
+  {{ template_logrotate_base }}
+
+parts = 
+  authenticated-httpd-server
+
+[authenticated-server-parameters]
+ca-url = 
+common-name = instance@${slap-configuration:instance-title}
+server-port = 8286
+custom-httpd-file = 
+web-directory = ${directory:document-root}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+ssl = ${:etc}/ssl
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+document-root = ${:srv}/private
+
+[certificate-request-base]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/request-instance-certificate
+cert-file = ${directory:ssl}/instance.cert.pem
+key-file = ${directory:ssl}/instance.key.pem
+ca-cert = ${directory:ssl}/cacert.pem
+parameters-extra = true
+command-line = {{ certificate_request_bin }}
+  --crt-file ${:cert-file}
+  --key-file ${:key-file}
+  --ca-url ${authenticated-server-parameters:ca-url}
+  --ca-crt-file ${:ca-cert}
+  --no-check-certificate
+# XXX - using --no-check-certificate which is insecure with https.
+
+[server-certificate-request]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/request-instance-certificate
+command-line =
+  ${certificate-request-base:wrapper-path}
+  --cn ${authenticated-server-parameters:common-name}
+  --request
+
+[authenticated-httpd-conf-parameter]
+ip = ${slap-configuration:ipv6-random}
+port = ${authenticated-server-parameters:server-port}
+pid-file = ${directory:run}/httpd-auth.pid
+dav-lock = ${directory:var}/DavLockdb
+access-log = ${directory:log}/httpd-auth-access.log
+error-log = ${directory:log}/httpd-auth-error.log
+cert-file = ${certificate-request-base:cert-file}
+key-file = ${certificate-request-base:key-file}
+ca-cert = ${certificate-request-base:ca-cert}
+url = https://[${:ip}]:${:port}
+private = ${authenticated-server-parameters:web-directory}
+httpd-include-file = ${authenticated-server-parameters:custom-httpd-file}
+crl = 
+
+[authenticated-httpd-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_httpd_auth_conf }}
+rendered = ${directory:etc}/httpd-auth.conf
+mode = 0744
+context =
+  section parameter_dict authenticated-httpd-conf-parameter
+
+[authenticated-httpd-graceful]
+recipe = collective.recipe.template
+input = inline:
+  #!{{ dash_executable_location }}
+  kill -USR1 $(cat ${authenticated-httpd-conf-parameter:pid-file})
+
+output = ${directory:scripts}/authenticated-httpd-graceful
+mode = 700
+
+[authenticated-httpd-server]
+recipe = slapos.cookbook:wrapper
+command-line = {{ apache_location }}/bin/httpd -f ${authenticated-httpd-conf:rendered} -DFOREGROUND
+wrapper-path = ${directory:services}/authenticated-httpd-server
+wait-for-files =
+  ${certificate-request-base:cert-file}
+  ${certificate-request-base:key-file}
+  ${certificate-request-base:ca-cert}
+
+url = ${authenticated-httpd-conf-parameter:url}
+
+depends = 
+  ${authenticated-httpd-promise:filename}
+  ${authenticated-httpd-graceful:output}
+  ${server-certificate-request:wrapper-path}
+  ${logrotate-authenticated-httpd:name}
+  ${certificate-renew-cron-entry:name}
+
+[certificate-renew]
+recipe = collective.recipe.template
+input = inline:
+  #!{{ dash_executable_location }}
+
+  d=$({{ openssl_executable_location }} x509 -enddate -noout -in ${certificate-request-base:cert-file} | cut -d'=' -f 2)
+  cert_time=$(date -d "$d" +"%s")
+  now=$(date +"%s")
+  thresold=2592000  # 30*24*60*60  equivalent to one month in seconds
+  remind=$(($cert_time - $now))
+
+  if [ $remind -lt $thresold ]; then
+    exec ${certificate-request-base:wrapper-path} --renew
+    exec ${authenticated-httpd-graceful:output}
+  fi
+
+output = ${directory:bin}/certificate-renew
+mode = 700
+
+[certificate-renew-cron-entry]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = certificate-auto-renew
+frequency = 5 6 * * 6
+command = ${certificate-renew:output}
+
+[logrotate-authenticated-httpd]
+< = logrotate-entry-base
+name = authenticated-httpd-server
+log = ${authenticated-httpd-conf-parameter:access-log} ${authenticated-httpd-conf-parameter:access-log}
+post = {{ slapos_kill_bin }} --pidfile ${authenticated-httpd-conf-parameter:pid-file} -s USR1
+
+[authenticated-httpd-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/${:filename}
+filename = authenticated-httpd-is-available
+url = ${authenticated-httpd-conf-parameter:url}
+check-secure = 1
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+cert-file = ${certificate-request-base:cert-file}
+key-file = ${certificate-request-base:key-file}
+ca-cert-file = ${certificate-request-base:ca-cert}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}

stack/caucase/instance-caucase.cfg.jinja2.in

+{% set part_list = [] -%}
+{% set ipv6 = (ipv6 | list)[0] -%}
+{% set default_subject = '/C=FR/O=Company/CN=SlapOS Certificate Authority/emailAddress=xx@example.com' -%}
+{% if slapparameter_dict is not defined -%}
+{% set slapparameter_dict = {} -%}
+{% endif -%}
+
+[ca-parameters]
+server-port = {{ slapparameter_dict.get('server-port', 8009) }}
+server-https-port = {{ slapparameter_dict.get('server-https-port', 8010) }}
+ipv6 = {{ ipv6 }}
+# Overrite this to set frontend or DNS URL (URL is used as CRL distribution point)
+# Please set http not HTTPS scheme
+crl-external-url = {{ slapparameter_dict.get('external-url', 'http://[${:ipv6}]:${:server-port}') }}
+# /CN=XXX is required and should be unique
+ca-subject = {{ slapparameter_dict.get('ca-subject', default_subject) }}
+# Number of pending csr to accept
+max-request-amount = {{ slapparameter_dict.get('max-request-amount', 10) }}
+# one year (in seconds)
+crt-life-time = {{ slapparameter_dict.get('crt-life-time', 31536000) }}
+# crl-life-period correspond to about one week
+crl-life-period = {{ slapparameter_dict.get('crl-life-period', 0.02) }}
+# ca-life-period = ca-life-period * crt-life-time
+ca-life-period = {{ slapparameter_dict.get('ca-life-period', 10) }}
+# time before clean certificate on CA: 60*24*60*60
+crt-keep-time = {{ slapparameter_dict.get('crt-keep-time', 5184000) }}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+run = ${:var}/run
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+ssl=${:etc}/ssl
+ca-dir = ${directory:srv}/ca
+ca-temp = ${:ca-dir}/tmp
+client-body-temp-path = ${:ca-temp}/client_body_temp_path
+proxy-temp-path = ${:ca-temp}/proxy_temp_path
+fastcgi-temp-path = ${:ca-temp}/fastcgi_temp_path
+uwsgi-temp-path = ${:ca-temp}/uwsgi_temp_path
+scgi-temp-path = ${:ca-temp}/scgi_temp_path
+
+[nginx-certificate-request-base]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/request-base-certificate
+cert-file = ${ca-nginx-ssl-config:cert}
+key-file = ${ca-nginx-ssl-config:key}
+ca-cert = ${directory:ssl}/cacert.pem
+parameters-extra = true
+command-line = {{ certificate_request_bin }}
+  --crt-file ${:cert-file}
+  --key-file ${:key-file}
+  --ca-url http://[${ca-parameters:ipv6}]:${ca-parameters:server-port}
+  --ca-crt-file ${:ca-cert}
+
+[nginx-certificate-request]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/request-server-certificate
+command-line =
+  ${nginx-certificate-request-base:wrapper-path}
+  --cn nginx@certificate.authority
+  --request
+
+[ca-nginx-ssl-config]
+# if ssl certificate is signed write to file so that zero-knowledge can read
+recipe = plone.recipe.command
+command = 
+  if [ -s "${:key}" ] && [ -s "${:cert}" ]; then
+  cat << EOF > ${:output}
+  [ca-nginx-ssl]
+  key=${:key}
+  cert=${:cert}
+  EOF
+  fi
+key = ${directory:ssl}/ca-cert.key
+cert = ${directory:ssl}/ca-cert.crt
+update-command = ${:command}
+output = ${directory:etc}/ca-nginx-ssl.cfg
+stop-on-error = true
+
+[ca-nginx-ssl]
+recipe = slapos.cookbook:zero-knowledge.read
+file-path = ${ca-nginx-ssl-config:output}
+# initials values are empty, the section https (ssl) in nginx config will be skipped
+cert = 
+key = 
+
+[ca-nginx-conf-parameter]
+ip = ${ca-parameters:ipv6}
+port = ${ca-parameters:server-port}
+https-port = ${ca-parameters:server-https-port}
+pid-file = ${directory:run}/nginx-ca.pid
+access-log = ${directory:log}/nginx-ca-access.log
+error-log = ${directory:log}/nginx-ca-error.log
+cert-file = ${ca-nginx-ssl:cert}
+key-file = ${ca-nginx-ssl:key}
+ca-conf = ${caucase-conf:output}
+workers-processes = 1
+client-body-temp-path = ${directory:client-body-temp-path}
+proxy-temp-path = ${directory:proxy-temp-path}
+fastcgi-temp-path = ${directory:fastcgi-temp-path}
+uwsgi-temp-path = ${directory:uwsgi-temp-path}
+scgi-temp-path = ${directory:scgi-temp-path}
+socket = ${caucase-gunicorn:socket}
+
+[ca-nginx-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_nginx_ca_conf }}
+rendered = ${directory:etc}/nginx-ca.conf
+mode = 0700
+context =
+  section parameter_dict ca-nginx-conf-parameter
+
+[caucase-conf]
+recipe = collective.recipe.template
+# Values here are intended to be changed in your instance. override this section
+input = inline:
+  ca-dir ${directory:ca-dir}
+  # enable debug
+  # debug
+  # log-file ${directory:log}/ca-server.log
+  subject ${ca-parameters:ca-subject}
+  max-request-amount ${ca-parameters:max-request-amount}
+  external-url ${ca-parameters:crl-external-url}
+  # one year (in seconds)
+  crt-life-time ${ca-parameters:crt-life-time}
+  # crl-life-period correspond to about one week
+  crl-life-period ${ca-parameters:crl-life-period}
+  # ca-life-time = ca-life-period * crt-life-time
+  ca-life-period ${ca-parameters:ca-life-period}
+  # time before clean certificate on CA: 60*24*60*60
+  crt-keep-time ${ca-parameters:crt-keep-time}
+
+output = ${directory:etc}/ca.conf
+mode = 700
+
+[ca-nginx-graceful]
+recipe = collective.recipe.template
+input = inline:#!{{ dash_executable_location }}
+  kill -HUP "$(cat '${ca-nginx-conf-parameter:pid-file}')"
+
+output = ${directory:scripts}/ca-server-graceful
+mode = 700
+
+[caucase-gunicorn]
+recipe = slapos.cookbook:wrapper
+socket = ${directory:ca-dir}/ca.flaskserver.sock
+command-line =  {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${caucase-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload
+log-file = ${directory:log}/ca-gunicorn-error.log
+pid-file = ${directory:run}/ca-gunicorn.pid
+wrapper-path = ${directory:services}/ca-gunicorn
+
+[caucase-server]
+recipe = slapos.cookbook:wrapper
+command-line = {{ nginx_location }}/sbin/nginx -p ${directory:ca-dir} -c ${ca-nginx-conf:rendered}
+wrapper-path = ${directory:services}/caucase-server
+url = https://[${ca-parameters:ipv6}]:${ca-parameters:server-https-port}
+http-url = ${ca-parameters:crl-external-url}
+depends = 
+  ${nginx-certificate-request:wrapper-path}
+  ${caucase-server-promise:filename}
+  ${caucase-https-server-promise:filename}
+  ${ca-nginx-graceful:output}
+  ${ca-certificate-renew-cron-entry:name}
+  ${logrotate-ca-nginx:name}
+
+[ca-certificate-renew-cron-entry]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = ca-server-certificate-auto-renew
+# check renew every-week
+time = weekly
+# 2592000 = 30*24*60*60  equivalent to one month in seconds
+command = ${nginx-certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${ca-nginx-graceful:output}"
+
+[logrotate-ca-nginx]
+< = logrotate-entry-base
+name = caucase-nginx-server
+log = ${ca-nginx-conf-parameter:access-log} ${ca-nginx-conf-parameter:access-log}
+post = {{ slapos_kill_bin }} --pidfile ${ca-nginx-conf-parameter:pid-file} -s USR1
+
+[caucase-server-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/${:filename}
+filename = caucase-server-listening-on-tcp
+url = http://[${ca-parameters:ipv6}]:${ca-parameters:server-port}
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+
+[caucase-https-server-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/${:filename}
+filename = caucase-server-https-on-${ca-parameters:server-https-port}
+url = https://[${ca-parameters:ipv6}]:${ca-parameters:server-https-port}
+check-secure = 1
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+
+{% if publish_parameter is defined and publish_parameter == 'yes' -%}
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+http-url = ${caucase-server:http-url}
+https-url = ${caucase-server:url}
+{% do part_list.append('publish-connection-parameter') -%}
+{% endif -%}
+
+[buildout]
+
+extends =
+  {{ template_logrotate_base }}
+
+parts = 
+  caucase-server
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
\ No newline at end of file

stack/caucase/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[dynamic-template-caucase]
+recipe = slapos.recipe.template:jinja2
+filename = ${:_buildout_section_name_}.cfg
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+template = {{ caucase_template }}
+extensions = jinja2.ext.do
+extra-context =
+context =
+    key ipv4 slap-configuration:ipv4
+    key ipv6 slap-configuration:ipv6
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw gunicorn_bin {{ gunicorn_bin }}
+    raw template_logrotate_base {{ template_logrotate_base }}
+    raw certificate_request_bin {{ certificate_request_bin }}
+    raw template_nginx_ca_conf {{ template_nginx_ca_conf }}
+    raw nginx_location {{ nginx_location }}
+    raw slapos_kill_bin {{ slapos_kill_bin }}
+    raw dash_executable_location {{ dash_executable_location }}
+    raw curl_executable_location {{ curl_executable_location }}
+    raw publish_parameter yes
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${dynamic-template-caucase:rendered}
+

stack/caucase/template-httpd-auth.conf.in

+Listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }}
+
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule dir_module modules/mod_dir.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule alias_module modules/mod_alias.so
+LoadModule autoindex_module modules/mod_autoindex.so
+LoadModule proxy_module      modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule env_module modules/mod_env.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
+
+ServerAdmin admin@
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+ServerTokens Prod
+ServerSignature Off
+TraceEnable Off
+
+PidFile "{{ parameter_dict['pid-file'] }}"
+ErrorLog "{{ parameter_dict['error-log'] }}"
+# Default apache log format with request time in microsecond at the end
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+CustomLog "{{ parameter_dict['access-log'] }}" combined
+
+# SSL Configuration
+Define SSLConfigured
+SSLCertificateFile {{ parameter_dict['cert-file'] }}
+SSLCertificateKeyFile {{ parameter_dict['key-file'] }}
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+SSLRandomSeed startup /dev/urandom 256
+SSLRandomSeed connect builtin
+SSLProtocol all -SSLv2 -SSLv3
+SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
+SSLHonorCipherOrder on
+
+SSLEngine   On
+
+
+<Directory />
+  Options FollowSymLinks
+  AllowOverride None
+  Allow from all
+</Directory>
+
+<VirtualHost *:{{ parameter_dict['port'] }}>
+
+  DavLockDB {{ parameter_dict['dav-lock'] }}
+
+  SSLVerifyClient require
+  RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
+  SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
+  {% if parameter_dict['crl'] -%}
+  SSLCARevocationCheck chain
+  SSLCARevocationFile {{ parameter_dict['crl'] }}
+  {%- endif %}
+
+  Alias / {{ parameter_dict['private'] }}/
+  <Directory {{ parameter_dict['private'] }}>
+    DirectoryIndex disabled
+    DAV On
+    Options Indexes FollowSymLinks
+    Order Allow,Deny
+    Allow from all
+  </Directory>
+
+</VirtualHost>
+
+{% if parameter_dict.get('httpd-include-file', '') -%}
+# Custom apache configuration here
+Include {{ parameter_dict['httpd-include-file'] }}
+{% endif -%}

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 2.8
+slapos.recipe.template = 3.0
 
 # Required by:
 # PasteScript==2.0
@@ -98,4 +98,4 @@ PasteDeploy = 1.5.2
 
 # Required by:
 # cloudooo==1.2.5.dev0
-erp5.util = 0.4.46
+erp5.util = 0.4.49

stack/erp5/TODO

@@ -4,8 +4,7 @@ General:
 - ipv6 support (besides frontend-backend apache connection)
   requires important changes at ERP5 level
 - resilience
-- make mariadb user accounts accept connections only from relevant IPs
-  or make x509 mandatory (needs ZMySQLD*A support)
+- mariadb: make x509 mandatory (needs ZMySQLD*A support)
 - make postfix log inside partition
 - document postfix parameters (only once it actually works)
 

stack/erp5/buildout.cfg

@@ -9,6 +9,7 @@ extends =
   ../../component/dbus/buildout.cfg
   ../../component/file/buildout.cfg
   ../../component/fonts/buildout.cfg
+  ../../component/gcc/buildout.cfg
   ../../component/ghostscript/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/graphviz/buildout.cfg
@@ -27,6 +28,7 @@ extends =
   ../../component/matplotlib/buildout.cfg
   ../../component/mesa/buildout.cfg
   ../../component/numpy/buildout.cfg
+  ../../component/ocropy/buildout.cfg
   ../../component/onlyoffice-x2t/buildout.cfg
   ../../component/pandas/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
@@ -58,7 +60,9 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
+  ../monitor/buildout.cfg
   ../../software/ipython_notebook/software.cfg
+  ../../software/caucase/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 
@@ -90,6 +94,8 @@ parts +=
   libreoffice-bin
   w3-validator
   tesseract
+  scipy
+  ocropy
   hookbox
   percona-toolkit
   zabbix-agent
@@ -136,10 +142,17 @@ parts +=
   ipython-notebook
   instance-jupyter
 
+# override python2.7 to add SlapOS libstdc++ in RPATH.
+[python2.7]
+extra-ldflags = -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64
+
 # override instance-jupyter not to render into default template.cfg
 [instance-jupyter]
 rendered = ${buildout:directory}/template-jupyter.cfg
 
+[instance-caucase]
+rendered = ${buildout:directory}/instance-caucase.cfg
+
 [download-base]
 <= download-base-neo
 url = ${:_profile_base_location_}/${:filename}
@@ -154,6 +167,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 <= download-base
 mode = 755
 
+[mariadb-slow-query-report-script]
+<= download-base
+mode = 755
+
 [template-mariadb]
 <= download-base
 link-binary =
@@ -218,6 +235,7 @@ context =
     key bin_directory buildout:bin-directory
     key buildout_bin_directory buildout:bin-directory
     key cairo_location cairo:location
+    key caucase_template instance-caucase:rendered
     key coreutils_location coreutils:location
     key cups_location cups:location
     key curl_location curl:location
@@ -258,10 +276,13 @@ context =
     key logrotate_location logrotate:location
     key mariadb_location mariadb:location
     key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
+    key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
     key matplotlibrc_location matplotlibrc:location
     key mesa_location mesa:location
     key onlyoffice_x2t_location onlyoffice-x2t:location
+    key parts_directory buildout:parts-directory
     key openssl_location openssl:location
+    key percona_toolkit_location percona-toolkit:location
     key pixman_location pixman:location
     key poppler_location poppler:location
     key postfix_location postfix:location
@@ -280,7 +301,7 @@ context =
     key template_kumofs template-kumofs:target
     key template_mariadb template-mariadb:target
     key template_mariadb_initial_setup template-mariadb-initial-setup:target
-    key template_monitor monitor-template:rendered
+    key template_monitor monitor2-template:rendered
     key template_my_cnf template-my-cnf:target
     key template_postfix template-postfix:target
     key template_postfix_aliases template-postfix-aliases:target
@@ -298,13 +319,6 @@ context =
     key zlib_location zlib:location
     key extra_path_list eggs:extra-paths
 
-[monitor-template-dummy]
-<= download-base
-# This is a placeholder, to be overriden by extending monitor SR
-
-[monitor-template]
-rendered = ${monitor-template-dummy:target}
-
 [template-erp5]
 <= download-base
 
@@ -437,14 +451,17 @@ eggs = ${neoppod:eggs}
   ${numpy:egg}
   ${matplotlib:egg}
   ${lxml-python:egg}
+  ${ocropy:egg}
   ${pandas:egg}
   ${pillow-python:egg}
   ${python-ldap-python:egg}
   ${pysvn-python:egg}
   ${pycrypto-python:egg}
+  ${scipy:egg}
   ${scikit-learn:egg}
   lock_file
   astor
+  APacheDEX
   PyStemmer
   PyXML
   Pympler
@@ -542,6 +559,7 @@ eggs = ${neoppod:eggs}
 # installation of python, which we don't want on an instance
 interpreter = ${buildout:python}
 scripts =
+  apachedex
   repozo
   runzope
   runzeo
@@ -590,6 +608,7 @@ eggs +=
 scripts +=
   is-local-tcp-port-opened
   is-process-older-than-dependency-set
+  runApacheDex
   zodbpack
 
 [versions]
@@ -599,6 +618,7 @@ scripts +=
 # patched eggs
 Acquisition = 2.13.9+SlapOSPatched001
 Products.DCWorkflow = 2.2.4+SlapOSPatched001
+ocropy = 1.0+SlapOSPatched001
 pysvn = 1.7.10+SlapOSPatched002
 python-ldap = 2.4.32+SlapOSPatched001
 python-magic = 0.4.12+SlapOSPatched001
@@ -641,6 +661,7 @@ zope.app.publication = 3.14.0
 zope.app.testing = 3.8.1
 
 # Pinned versions
+APacheDEX = 1.6.2
 Pillow = 4.0.0
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1

stack/erp5/buildout.hash.cfg

@@ -15,19 +15,23 @@
 # not need these here).
 [mariadb-resiliency-after-import-script]
 filename = instance-mariadb-resiliency-after-import-script.sh.in
-md5sum = 1e89de954d816b93f76d9b75820d192c
+md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
+
+[mariadb-slow-query-report-script]
+filename = mysql-querydigest.sh.in
+md5sum = dc974bd74cf967ae6250d81322629c44
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 11e7faca5233868c1643854e4c898588
+md5sum = 3b6c19686ac52c2d452597dbacc88888
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
-md5sum = 42d2a2c7cb5bf5122b6cfd8f53a5576f
+md5sum = 253027f37c4e5bd908e6af551848f03a
 
 [template-cloudooo]
 filename = instance-cloudoo.cfg.in
-md5sum = c553bbcdc5f80d893907a05acefc1356
+md5sum = 43253cbfd5d361047ba6a99b116b2370
 
 [template-zope-conf]
 filename = zope.conf.in
@@ -63,7 +67,7 @@ md5sum = 9ac81647368068a1a98a785d08074b43
 
 [template-postfix-main-cf]
 filename = postfix_main.cf.in
-md5sum = 4fe9bb5225b9d7650d2916a0945821bd
+md5sum = 886d1b004049c8dc1f89cbbfe4f95371
 
 [template-postfix-aliases]
 filename = postfix_aliases.in
@@ -71,7 +75,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = 8ab417cf1ca98d2840c80a266f0e2be7
+md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -79,19 +83,19 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 07f45a5d3a9fd0fe227af82c7ede7631
+md5sum = 13638031b6b6c9ad9c0a9c4e6d9a202a
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = 7610bafda245c008ccf0b6ea58ce21c2
+md5sum = 2248d249cfc53c73e7c65665d4d53be0
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = b7e92234825f9d72ccb9b6c4745b6ce7
+md5sum = 6a64d1615c3ef9f6311c863d5aa0c58f
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = d71c49f91b3455e6866f4b2db591009f
+md5sum = f2fb0c537c124622fe8e89afe0188519
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

 {% set part_list = [] -%}
 {% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
+{% set caucase_url = slapparameter_dict.get('caucase-url', '') -%}
 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {#
@@ -36,6 +37,56 @@ context = key content {{content_section_name}}:content
 mode = {{ mode }}
 {%- endmacro %}
 
+[certificate-request-base]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/request-instance-certificate
+parameters-extra = true
+command-line = {{ parameter_dict['bin-directory'] }}/caucase-cliweb
+  --crt-file ${apache-conf-ssl:cert}
+  --key-file ${apache-conf-ssl:key}
+  --crl-file ${apache-conf-ssl:crl}
+  --ca-url {{ caucase_url }}
+  --ca-crt-file ${apache-conf-ssl:ca-cert}
+
+{% macro request_cert(name, common_name) -%}
+{% set get_crl_periodicity = slapparameter_dict.get('crl-update-periodicity', 'daily') -%}
+
+[{{ section(name ~ '-certificate-request') }}]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:services}/request-{{ name }}-certificate
+command-line =
+  ${certificate-request-base:wrapper-path}
+  --cn {{ common_name }}
+  --request
+
+[{{ section(name ~ '-renew-cron-entry') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = {{ name }}-certificate-auto-renew
+time = weekly
+# 2592000 = 30*24*60*60  equivalent to one month in seconds
+command = ${certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${apache-graceful:output}"
+
+[{{ section(name ~ '-download-crl') }}]
+# download the crl for the first time
+recipe = plone.recipe.command
+command = 
+  if [ ! -s "${apache-conf-ssl:crl}" ]; then
+    ${certificate-request-base:wrapper-path} --update-crl
+  fi
+update-command = ${:command}
+stop-on-error = true
+
+[{{ section(name ~ '-update-crl-cron-entry') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = {{ name }}-update-crl
+time = {{ get_crl_periodicity }}
+# XXX - Update crl call apache graceful restart, it's not recommended to check crl too often, Apache
+# has an issue with reload and can be frozen and stop responding. Default periodicity time = daily
+command = ${certificate-request-base:wrapper-path} --update-crl --on-crl-update="${apache-graceful:output}"
+{%- endmacro %}
+
 {% if use_ipv6 -%}
 [zope-tunnel-base]
 recipe = slapos.cookbook:ipv4toipv6
@@ -81,6 +132,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 -#}
 {%   do zope_family_address_list[0][0] -%}
 {%   set haproxy_port = next_port() -%}
+{%   set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
 {%   do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
 {%   if has_webdav -%}
 {%     set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
@@ -89,7 +141,8 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
 {%     set internal_scheme = 'http' -%}
 {%     set external_scheme = 'https' -%}
 {%   endif -%}
-{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ slapparameter_dict['backend-path'])) -%}
+{%   set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
+{%   do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
 {% endfor -%}
 
 [haproxy-cfg-parameter-dict]
@@ -122,6 +175,7 @@ crl = ${directory:apache-conf}/crl.pem
 backend-list = {{ dumps(apache_dict.values()) }}
 ip-list = {{ dumps(apache_ip_list) }}
 pid-file = ${directory:run}/apache.pid
+log-dir = ${directory:log}
 error-log = ${directory:log}/apache-error.log
 access-log = ${directory:log}/apache-access.log
 # Apache 2.4's default value (60 seconds) can be a bit too short
@@ -145,6 +199,18 @@ context = section parameter_dict apache-conf-parameter-dict
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:services}/apache
 command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
+wait-for-files =
+  ${apache-conf-ssl:cert}
+  ${apache-conf-ssl:key}
+
+[apache-graceful]
+recipe = collective.recipe.template
+input = inline:
+  #!/bin/sh
+  kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
+
+output = ${directory:bin}/apache-httpd-graceful
+mode = 700
 
 [{{ section('apache-promise') }}]
 # Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
@@ -155,10 +221,11 @@ port = {{ apache_dict.values()[0][0] }}
 
 [publish]
 recipe = slapos.cookbook:publish.serialised
-{% for family_name, (apache_port, scheme, _) in apache_dict.items() -%}
+{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
 {{   family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
 {{   family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
 {% endfor -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [apache-ssl]
 {% if ssl_parameter_dict.get('key') -%}
@@ -166,6 +233,11 @@ key = ${apache-ssl-key:rendered}
 cert = ${apache-ssl-cert:rendered}
 {{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
 {{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
+{% elif caucase_url -%}
+key = ${apache-conf-ssl:key}
+cert = ${apache-conf-ssl:cert}
+
+{{ request_cert('erp5', 'instance.apache@erp5') }}
 {% else %}
 recipe = plone.recipe.command
 command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
@@ -179,6 +251,10 @@ cert = ${apache-ssl-ca:rendered}
 crl = ${apache-ssl-crl:rendered}
 {{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }}
 {{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }}
+{% elif caucase_url -%}
+cert = ${apache-conf-ssl:ca-cert}
+crl = ${apache-conf-ssl:crl}
+
 {% else %}
 cert =
 crl =
@@ -206,11 +282,34 @@ private = ${:ca-dir}/private
 certs = ${:ca-dir}/certs
 newcerts = ${:ca-dir}/newcerts
 crl = ${:ca-dir}/crl
+apachedex = ${monitor-directory:private}/apachedex
+
+[monitor-generate-apachedex-report]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${monitor-directory:reports}/${:command}
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
+command = apachedex_every_3_hour
+
+[apachedex-parameters]
+default_parameter = 
+# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
+# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
+apache-log-list = ${apache-conf-parameter-dict:access-log}
+default = ${monitor-directory:etc}/apdex_default
+base-list = ${monitor-directory:etc}/apdex_base_list
+skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
+erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
 monitor-title = Balancer monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
+instance-configuration = 
+  file apachedex-default ${apachedex-parameters:default}
+  file apachedex-base-list ${apachedex-parameters:base-list}
+  file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
+  file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
 
 [buildout]
 extends =
@@ -219,4 +318,5 @@ extends =
 parts +=
   publish
   logrotate-apache
+  monitor-generate-apachedex-report
   {{ part_list | join('\n  ') }}

stack/erp5/instance-cloudoo.cfg.in

@@ -20,6 +20,7 @@ url = http://[${ipv6toipv4:ipv6}]:${ipv6toipv4:ipv6-port}/
 {% else -%}
 url = http://${cloudooo-instance:ip}:${cloudooo-instance:port}/
 {% endif -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [cloudooo-instance]
 recipe = slapos.cookbook:generic.cloudooo
@@ -57,9 +58,10 @@ service-folder = ${directory:service}
 
 [resiliency-exclude-file]
 # Generate rdiff exclude file in case of resiliency
-recipe = collective.recipe.template
-input = inline: **
-output = ${directory:srv}/exporter.exclude
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
 
 [promise]
 recipe = slapos.cookbook:check_port_listening
@@ -110,6 +112,7 @@ framebuffer = ${:srv}/framebuffer
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ tcpv4_port + 2 }}
 monitor-title = Cloudooo monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link

stack/erp5/instance-erp5.cfg.in

@@ -8,6 +8,9 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set monitor_base_url_dict = {} -%}
+{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
+{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
 [request-common]
 <= request-common-base
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
@@ -22,7 +25,10 @@ software-type = {{ software_type }}
 return = {{ ' '.join(ret) }}
 {% for ret, publish in ret.iteritems() -%}
 {%   if publish -%}
-{%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
+{%     do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
+{%   endif -%}
+{%   if ret == "monitor-base-url" -%}
+{%     do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-' ~ ret ~ '}') -%}
 {%   endif -%}
 {% endfor -%}
 {{ root_common.sla(name) }}
@@ -34,10 +40,10 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
 {% endfor -%}
 {% endmacro -%}
 
-{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }}
-{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
-{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
-{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
+{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
+{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {% if has_posftix -%}
 {{   request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
 {%- else %}
@@ -46,6 +52,14 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
 connection-url = smtp://127.0.0.2:0/
 {%- endif %}
 
+{% if caucase_url -%}
+{%   do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
+[request-caucase]
+connection-http-url = {{ caucase_url }}
+{%- else %}
+{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891}, {'http-url': True, 'https-url': False}) }}
+{% endif -%}
+
 {# ZODB -#}
 {% set zodb_dict = {} -%}
 {% set storage_dict = {} -%}
@@ -74,7 +88,7 @@ connection-url = smtp://127.0.0.2:0/
 {%   else -%}
 {{     assert(server_type == 'zeo', server_type) -}}
 {# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
-{{     request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
+{{     request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port', 'monitor-base-url')), key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
 {%   endif -%}
 {% endfor -%}
 
@@ -89,16 +103,16 @@ recipe = slapos.cookbook:publish-early
 {%- if neo %}
   neo-cluster gen-neo-cluster:name
 {%-  if neo[0] %}
-neo-cluster = {{ neo[0] }}
+neo-cluster = {{ dumps(neo[0]) }}
 {%-  endif %}
 {%- endif %}
 {%- set inituser_password = slapparameter_dict.get('inituser-password') %}
 {%- if inituser_password %}
-inituser-password = {{ inituser_password }}
+inituser-password = {{ dumps(inituser_password) }}
 {%- endif %}
 {%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
 {%- if deadlock_debugger_password %}
-deadlock-debugger-password = {{ deadlock_debugger_password }}
+deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
 {%- endif %}
 
 [gen-password]
@@ -126,12 +140,14 @@ name = neo-${gen-neo-cluster-base:passwd}
 return =
   zope-address-list
   hosts-dict
+  monitor-base-url
 {% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
 {% if has_jupyter -%}
 {%   set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
 {% endif -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
+config-caucase-url = ${request-caucase:connection-http-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
 config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
@@ -141,6 +157,7 @@ config-inituser-login = {{ dumps(inituser_login) }}
 config-inituser-password = ${publish-early:inituser-password}
 config-kumofs-url = ${request-memcached-persistent:connection-url}
 config-memcached-url = ${request-memcached-volatile:connection-url}
+config-monitor-passwd = ${monitor-htpasswd:passwd}
 config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
 config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
@@ -162,20 +179,26 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
+{% set zope_backend_path_dict = {} -%}
+{% set ssl_authentication_dict = {} -%}
 {% set jupyter_zope_family_default = [] -%}
 {% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
 {%   set partition_name = 'zope-' ~ custom_name -%}
 {%   set section_name = 'request-' ~ partition_name -%}
 {%   set zope_family = zope_parameter_dict.get('family', 'default') -%}
+{%   set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
 {#   # default jupyter zope family is first zope family. -#}
 {#   # use list.append() to update it, because in jinja2 set changes only local scope. -#}
 {%   if not jupyter_zope_family_default -%}
 {%     do jupyter_zope_family_default.append(zope_family) -%}
 {%   endif -%}
 {%   do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
+{%   do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
+{%   do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
 [{{ section_name }}]
 <= request-zope-base
 name = {{ partition_name }}
+{% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
@@ -235,10 +258,13 @@ name = balancer
 software-type = balancer
 {{ root_common.sla('balancer') }}
 return =
+  monitor-base-url
 {%- for family in zope_family_dict %}
   {{ family }}
   {{ family }}-v6
 {% endfor -%}
+{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
+
 config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
 config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
 {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
@@ -246,8 +272,12 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}
 {% endfor -%}
 # XXX: should those really be same for all families ?
 config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
-config-backend-path = {{ dumps(balancer_dict.get('apache-backend-path', '/') % {'site-id': site_id}) }}
 config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
+config-monitor-passwd = ${monitor-htpasswd:passwd}
+config-caucase-url = ${request-caucase:connection-http-url}
+config-crl-update-periodicity = {{ crl_update_period }}
+config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
+config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 
 [request-frontend-base]
 {% if has_frontend -%}
@@ -271,6 +301,7 @@ return = site_url
 {% endif -%}{# if zope_partition_dict -#}
 
 [publish]
+<= monitor-publish
 recipe = slapos.cookbook:publish.serialised
 -extends = publish-early
 {% if 'neo' in storage_dict -%}
@@ -288,4 +319,23 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
 {{   name }} = {{ value }}
 {% endfor -%}
 
+[monitor-instance-parameter]
+monitor-httpd-port = 8386
+
 {{ root_common.common_section() }}
+
+[buildout]
+extends = {{ template_monitor }}
+
+parts +=
+  monitor-base
+
+[monitor-conf-parameters]
+monitor-title = ERP5 monitor
+password = ${monitor-htpasswd:passwd}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+

stack/erp5/instance-kumofs.cfg.in

@@ -20,6 +20,7 @@ url = memcached://[${kumofs-instance:ip}]:${kumofs-instance:gateway-port}/
 {% else -%}
 url = memcached://${kumofs-instance:ip}:${kumofs-instance:gateway-port}/
 {% endif -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [kumofs-instance]
 recipe = slapos.cookbook:generic.kumofs
@@ -77,9 +78,10 @@ kumofs-data = ${:srv}/kumofs
 
 [resiliency-exclude-file]
 # Generate rdiff exclude file in case of resiliency
-recipe = collective.recipe.template
-input = inline: **
-output = ${directory:srv}/exporter.exclude
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
 
 # Deploy zope promises scripts
 [promise-template]
@@ -111,3 +113,4 @@ port = ${kumofs-instance:manager-port}
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ tcpv4_port + 4 }}
 monitor-title = Kumofs monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}

stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in

-#!${:dash}
+#!{{ dash }}
 
 # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
 # OR MYSQL DATA WILL BE ERASED.
@@ -12,14 +12,14 @@
 
 set -e
 
-mysql_executable='${binary-wrap-mysql:wrapper-path}'
-mariadb_data_directory='${directory:mariadb-data}'
-mariadb_backup_directory='${directory:mariadb-backup-full}'
-instance_directory='${buildout:directory}'
-pid_file='${my-cnf-parameters:pid-file}'
-binlog_path='${my-cnf-parameters:binlog-path}'
-server_executable='${mysqld:rendered}'
-update_executable='${update-mysql:output}'
+mysql_executable='{{ mysql_executable }}'
+mariadb_data_directory='{{ mariadb_data_directory }}'
+mariadb_backup_directory='{{ mariadb_backup_directory }}'
+instance_directory='{{ instance_directory }}'
+pid_file='{{ pid_file }}'
+binlog_path='{{ binlog_path }}'
+server_executable='{{ server_executable }}'
+update_executable='{{ update_executable }}'
 
 # Make sure mariadb is not already running
 if [ -e "$pid_file" ]; then

stack/erp5/instance-mariadb.cfg.in

@@ -16,9 +16,11 @@
 {% else -%}
 {%   set ip = (ipv4_set | list)[0] -%}
 {% endif -%}
+{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
 
-[publish]
+[{{ section('publish') }}]
 recipe = slapos.cookbook:publish.serialised
+-extends = publish-early
 {% macro render_database_list(database_list) -%}
 {% set publish_database_list = [] -%}
 {% for database in database_list -%}
@@ -32,6 +34,21 @@ recipe = slapos.cookbook:publish.serialised
 {% endmacro -%}
 database-list = {{ render_database_list(database_list) }}
 test-database-list = {{ render_database_list(test_database_list) }}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+  server-id gen-server-id:value
+{%- set server_id = slapparameter_dict.get('server-id') %}
+{%- if server_id %}
+server-id = {{ dumps(server_id) }}
+{%- endif %}
+
+[gen-server-id]
+recipe = slapos.cookbook:random.integer
+minimum = {{ dumps(1) }}
+maximum = {{ dumps(2**32 - 1) }}
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -124,7 +141,7 @@ relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
 binlog-path = ${directory:mariadb-backup-incremental}/binlog
 # XXX: binlog rotation happens along with other log's rotation
 binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
-server-id = {{ dumps(slapparameter_dict.get('server-id', 1)) }}
+server-id = ${publish-early:server-id}
 {% else %}
 binlog-path =
 {%- endif %}
@@ -148,7 +165,7 @@ rendered = ${directory:etc}/mariadb_initial_setup.sql
 template = {{ parameter_dict['template-mariadb-initial-setup'] }}
 context = section parameter_dict init-script-parameters
 
-[update-mysql]
+[{{ section('update-mysql') }}]
 recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
 output = ${directory:services}/mariadb_update
 binary = ${binary-wrap-mysql_upgrade:wrapper-path}
@@ -162,7 +179,7 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
 # section is installed - so I only use jinja2 as a fancy way to write an
 # executable file with partition-dependent but instance-parameters independent
 # content.
-template = inline:#!{{ parameter_dict['dash-location'] }}/bin/dash
+template = inline:#!{{ dash }}
   '{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
     --defaults-file='${my-cnf:rendered}' \
     --skip-name-resolve \
@@ -173,13 +190,13 @@ template = inline:#!{{ parameter_dict['dash-location'] }}/bin/dash
     "$@"
 rendered = ${directory:services}/mariadb
 
-[logrotate-entry-mariadb]
+[{{ section('logrotate-entry-mariadb') }}]
 < = logrotate-entry-base
 name = mariadb
 log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
 post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
 
-[binary-link]
+[{{ section('binary-link') }}]
 recipe = slapos.cookbook:symbolic.link
 target-directory = ${directory:bin}
 link-binary = {{ dumps(parameter_dict['link-binary']) }}
@@ -213,6 +230,12 @@ command = mysql_upgrade
 command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
 command = mysql_tzinfo_to_sql
 
+[binary-wrap-pt-digest]
+<= binary-wrap-base
+command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
+command = pt-query-digest
+ 
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 bin = ${buildout:directory}/bin
@@ -230,41 +253,60 @@ var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
 
-[resiliency-exclude-file]
+[{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency
-recipe = collective.recipe.template
-input = inline: srv/mariadb/**
-output = ${directory:srv}/exporter.exclude
+< = jinja2-template-base
+template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
+
+[dash]
+dash = {{ dumps(dash) }}
 
 [resiliency-after-import-script]
 # Generate after import script used by importer instance of webrunner
-recipe = collective.recipe.template
-input = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
-output = ${directory:srv}/runner-import-restore
+< = jinja2-template-executable
+template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
+rendered = ${directory:srv}/runner-import-restore
+context =
+  key dash dash:dash
+  key mysql_executable binary-wrap-mysql:wrapper-path
+  key mariadb_data_directory directory:mariadb-data
+  key mariadb_backup_directory directory:mariadb-backup-full
+  key instance_directory buildout:directory
+  key pid_file my-cnf-parameters:pid-file
+  key binlog_path my-cnf-parameters:binlog-path
+  key server_executable mysqld:rendered
+  key update_executable update-mysql:output
+
+[{{ section('monitor-generate-mariadb-slow-query-report') }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
+rendered = ${monitor-directory:reports}/${:filename}
+output-folder = ${directory:srv}/monitor/
+filename = mariadb_slow_query_every_23_hour
 mode = 755
-dash = {{ parameter_dict['dash-location'] }}/bin/dash
+context = 
+  raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
+  raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
+  raw dash {{ parameter_dict['dash-location'] }}/bin/dash
+  key output_folder :output-folder 
 
-[promise]
+[{{ section('promise') }}]
 recipe = slapos.cookbook:wrapper
 command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
 wrapper-path = ${directory:promise}/mariadb
 parameters-extra = true
 
+{% set monitor_passwd = slapparameter_dict['monitor-passwd'] -%}
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ port + 1 }}
 monitor-title = Mariadb monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
 
 [buildout]
 extends =
   {{ logrotate_cfg }}
   {{ parameter_dict['template-monitor'] }}
 parts +=
-  publish
-  logrotate-entry-mariadb
-  binary-link
-  update-mysql
-  resiliency-exclude-file
-  resiliency-after-import-script
-  promise
   {{ part_list | join('\n  ') }}

stack/erp5/instance-zeo.cfg.in

@@ -113,14 +113,16 @@ tidstorage-port = ${tidstorage:port}
 tidstorage-ip =
 tidstorage-port =
 {% endif -%}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 # Used for ERP5 resiliency or (more probably)
 # webrunner resiliency with erp5 inside.
 [{{ section("resiliency-exclude-file") }}]
 # Generate rdiff exclude file
-recipe = collective.recipe.template
-input = inline: srv/zodb/**
-output = ${directory:srv}/exporter.exclude
+recipe = slapos.recipe.template:jinja2
+mode = 644
+template = {{ 'inline:{{ "${directory:zodb}/**\\n" }}' }}
+rendered = ${directory:srv}/exporter.exclude
 
 [{{ section("resiliency-after-import-script") }}]
 # Generate after import script used by importer instance of webrunner
@@ -184,6 +186,7 @@ tidstorage = {{ tidstorage_repozo_path }}
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
 monitor-title = ZODB monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
 
 [buildout]
 extends =

stack/erp5/instance-zope.cfg.in

@@ -379,6 +379,7 @@ dict toward erp5 partition, violating the DRY principle and making the intent
 hard to guess.
 -#}
 hosts-dict = {{ dumps(hosts_dict) }}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
 
 [erp5-promise]
 recipe = slapos.cookbook:erp5.promise
@@ -394,6 +395,7 @@ bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ next_port() }}
 monitor-title = Zope monitor
+password = {{ slapparameter_dict['monitor-passwd'] }}
 
 [buildout]
 extends =

stack/erp5/instance.cfg.in

 [buildout]
-extends = {{ instance_common_cfg }}
+extends =
+  {{ instance_common_cfg }}
+  {{ caucase_template }}
 
 [jinja2-template-base]
 mode = 644
@@ -72,6 +74,7 @@ extra-context =
 [dynamic-template-erp5-parameters]
 jupyter-enable-default = {{ jupyter_enable_default }}
 local-bt5-repository = {{ local_bt5_repository }}
+template-monitor = {{ dumps(template_monitor) }}
 
 [dynamic-template-erp5]
 <= jinja2-template-base
@@ -80,6 +83,7 @@ filename = instance-erp5.cfg
 extra-context =
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
+    key template_monitor dynamic-template-erp5-parameters:template-monitor
     key openssl_location :openssl-location
     import urlparse urlparse
 import-list =
@@ -92,7 +96,10 @@ apache = {{ apache_location }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
 bin-directory = {{ bin_directory }}
+apachedex-location = {{ bin_directory }}/apachedex
+run-apachedex-location = {{ bin_directory }}/runApacheDex
 6tunnel = {{ sixtunnel_location }}
+curl-location = {{ curl_location }}
 dash = {{ dash_location }}
 template-haproxy-cfg = {{ template_haproxy_cfg }}
 template-apache-conf = {{ template_apache_conf }}
@@ -172,6 +179,8 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
 link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
+mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
+percona-tools-location = {{ percona_toolkit_location }}
 template-monitor = {{ template_monitor }}
 
 [dynamic-template-mariadb]
@@ -210,6 +219,7 @@ create-erp5-site = dynamic-template-create-erp5-site:rendered
 RootSoftwareInstance = ${:default}
 # Internal software types
 kumofs = dynamic-template-kumofs:rendered
+caucase = dynamic-template-caucase:rendered
 cloudooo = dynamic-template-cloudooo:rendered
 mariadb = dynamic-template-mariadb:rendered
 balancer = dynamic-template-balancer:rendered

stack/erp5/mysql-querydigest.sh.in

+#!{{dash}}
+# BEWARE: This file is operated by slapgrid
+
+SLOW_QUERY_PATH='{{slow_query_path}}'
+OUTPUT_FOLDER='{{output_folder}}'
+PT_QUERY_EXEC='{{pt_query_exec}}'
+
+if [ ! -d "$OUTPUT_FOLDER" ]; then
+  echo "ERROR: output_folder don't exists"
+  exit 0
+fi
+
+TODAY=`date +%Y%m%d`
+
+SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
+
+if [ ! -f "$SLOW_LOG" ]; then
+  echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
+  exit 1
+fi
+
+eval $PT_QUERY_EXEC $SLOW_LOG

stack/erp5/postfix_main.cf.in

@@ -52,6 +52,10 @@ smtpd_relay_restrictions =
   permit_sasl_authenticated
   defer_unauth_destination
 
+# We do not pass mail address in command lines, so accept those starting with
+# a dash.
+allow_min_user = yes
+
 # Disable local delivery
 local_transport = error
 

stack/lamp.cfg

@@ -42,7 +42,7 @@ keep-compile-dir = false
 apache-libcloud = 0.9.1
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10
+mysqlclient = 1.3.10+SlapOSPatched001
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.3
 slapos.toolbox = 0.40.4

stack/lamp/buildout.cfg

@@ -185,7 +185,7 @@ eggs =
 apache-libcloud = 0.12.3
 async = 0.6.1
 gitdb = 0.5.4
-mysqlclient = 1.3.10
+mysqlclient = 1.3.10+SlapOSPatched001
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
 slapos.recipe.template = 2.4.2

stack/monitor/README.md

@@ -26,7 +26,7 @@ Extend monitor template and a monitor-base to parts:
       ...
       monitor-base
 
-Override monitor configuration section to define your custom parameters.
+Override monitor configuration by adding monitor-instance-parameter section to define your custom parameters.
 
     [monitor-instance-parameter]
     monitor-title = ${slap-configuration:instance-title}
@@ -38,7 +38,7 @@ Override monitor configuration section to define your custom parameters.
     cors-domains = monitor.app.officejs.com
     collector-db = ...
     password = ${monitor-htpasswd:passwd}
-    username = ${monitor-htpasswd:username}
+    username = admin
     instance-configuration = ...
     configuration-file-path = ...
     interface-url = ...
@@ -53,11 +53,11 @@ You don't need to define all parameters, you can only set what is required to be
 - root-instance-title: is the title of the hosting subscription.
 - monitor-httpd-ipv6: is the ipv6 of the computer partition.
 - monitor-httpd-port: the port to bind monitor httpd server on.
-- monitor-base-url: this url that will be used/showed in monitor interface. This url is present in some monitor generated output files. There can be two value, the default: ${monitor-frontend-promise:url} which access monitor httpd server through the frontend and ${monitor-httpd-conf-parameter:url} which is the url with ipv6 (https://[IPv6]:port/).
+- monitor-base-url: this url will be used/showed in monitor interface. This url is present in some monitor generated output files. There can be two value, the default: ${monitor-frontend-promise:url} which access monitor httpd server through the frontend and ${monitor-httpd-conf-parameter:url} which is the url with ipv6 (https://[IPv6]:port/).
 - monitor-url-list: set list of Monitor Base URL of monitor sub-instances, if this is the root instance with at least one child.
 - cors-domains: the domain used by the monitor web interface. The default is: monitor.app.officejs.com.
 - username: monitor username, this should be the same in all sub-instances. Default is: admin.
-- password: monitor password, this should be the same in all sub-instances. Default is generated (${monitor-htpasswd:username}).
+- password: monitor password, this should be the same in all sub-instances. Default is generated (${monitor-instance-parameter:username}).
 - instance-configuration: instance custom information or configuration to show in monitor web interface. There is many possibility:
   raw CONFIG_KEY VALUE => non editable configuration, ie: raw monitor-password resqdsdsd34
   file CONFIG_KEY PATH_TO_RESULT_FILE => editable configuration.
@@ -65,6 +65,18 @@ You don't need to define all parameters, you can only set what is required to be
 - configuration-file-path: path of knowledge0 cfg file where instance configuration will be written.
 - interface-url: The URL of monitor web interface. This URL will be present in generated JSON files.
 
+**Multiple Monitors**
+
+If you have sub-instances, you should collect the base monitor url from all instances with monitor and send it to monitor-url-list or you can override "monitor-base-url-dict" section and add all the urls as key/value pairs in the root instance.
+
+    [monitor-base-url-list]
+    monitor1-url = https://[xxxx:xxx:xxxx:e:11::1fb1]:4200 
+    monitor2-url = https://[xxxx:xxx:xxxx:e:22::2fb2]:4200 
+    ..
+    ..
+
+Also, All monitors of the sub instances need to have same password as the password of the root instance monitor.
+
 NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $ in your instance template file if it's not a jinja template. See:
 - Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance-resilient-test.cfg.jinja2
 - Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance.cfg
@@ -187,13 +199,13 @@ MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to pub
 Access to Monitor
 -----------------
 
-In monitor instance.cfg file, the section [publish] contain information about monitor access.
+In monitor instance.cfg file, the section [monitor-publish-parameters] contain information about monitor access.
 Usefull information are monitor-base-url, monitor-url, monitor-user and monitor-password.
 
-- ${publish:monitor-base-url} is the url of monitor httpd server.
-- ${publish:monitor-base-url}/public/feed is the RSS url of this monitor instance.
-- ${publish:monitor-base-url}/public/feeds is the OPML URL of this monitor instance. To setup monitor instance in your monitoring interface, use OPML URL of the root instance. It should contain URL to others monitor instances.
-- ${publish:monitor-base-url}/private is the monitor private directory. Username and password are reqired to connect.
+- ${monitor-publish-parameters:monitor-base-url} is the url of monitor httpd server.
+- ${monitor-publish-parameters:monitor-base-url}/public/feed is the RSS url of this monitor instance.
+- ${monitor-publish-parameters:monitor-base-url}/public/feeds is the OPML URL of this monitor instance. To setup monitor instance in your monitoring interface, use OPML URL of the root instance. It should contain URL to others monitor instances.
+- ${monitor-publish-parameters:monitor-base-url}/private is the monitor private directory. Username and password are reqired to connect.
 
 The section [monitor-publish] contain parameters to publish with your instance connection information. It will publish "monitor-base-url" and
 "monitor-setup-url" which is used to configure your instance to monitor interface in one click.
@@ -218,7 +230,7 @@ Parameters can be editable (except raw parameter) directly from  monitor interfa
     [monitor-instance-parameter]
     instance-configuration = 
       raw init-user ${publish-connection-information:init-user}
-      htpasswd monitor-password ${monitor-htpassword-file:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
+      htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
       file promise-timeout ${monitor-promise-timeout-file:file}
 
 The user will see parameters:

stack/monitor/buildout.cfg

@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
 [monitor-conf]
 <= monitor-template-base
 filename = monitor.conf.in
-md5sum = c8f024d741c6494d7c9ba01601d0b917
+md5sum = fc65084dd1c2fe2487b58a003b576f61
 
 [monitor-instance-info]
 <= monitor-template-base
@@ -131,5 +131,5 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.67
+slapos.toolbox = 0.69
 

stack/monitor/buildout.hash.cfg

@@ -15,4 +15,4 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = c963c43189a9b50e48ba27f2dd4e7b02
+md5sum = 5027f0b1abcc28ce3817cd70fb667a3b

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -100,7 +100,7 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 
 parameter-list = 
   raw monitor-user ${monitor-instance-parameter:username}
-  htpasswd monitor-password ${monitor-htpassword-file:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
+  htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
   file promise-timeout ${monitor-promise-timeout-file:file}
   file min-free-disk-MB ${promise-check-free-disk-space:config-file}
   ${monitor-instance-parameter:instance-configuration}
@@ -122,12 +122,16 @@ command =
     echo "20" > ${:file}
   fi
 
+[monitor-base-url-dict]
+# place holder to be used to collect erp5 monitor urls
+
 [monitor-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
 rendered = ${directory:etc}/${:filename}
 filename = monitor.conf
 context = section parameter_dict monitor-conf-parameters
+          section monitor_base_urls monitor-base-url-dict
 
 [instance-info-parameters]
 name = ${monitor-instance-parameter:monitor-title}
@@ -156,7 +160,8 @@ update-command = ${:command}
 [start-monitor]
 recipe = slapos.cookbook:wrapper
 command-line = {{ python_executable }} {{ monitor_bin }} --config_file ${monitor-conf:rendered}
-wrapper-path = ${directory:scripts}/bootstrap-monitor
+name = bootstrap-monitor
+wrapper-path = ${directory:scripts}/${:name}
 environment = 
   PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
 
@@ -164,25 +169,18 @@ environment =
 recipe = slapos.cookbook:generate.password
 storage-path = ${directory:etc}/.monitor_pwd
 bytes = 8
-username = admin
-
-[monitor-htpassword-file]
-recipe = plone.recipe.command
-stop-on-error = true
-password-file = ${directory:etc}/.monitor_pwd
-command = 
-  if [ ! -s "${:password-file}" ]; then echo "${monitor-instance-parameter:password}" > ${:password-file}; fi
-update-command = ${:command}
 
 [httpd-monitor-htpasswd]
 recipe = plone.recipe.command
 stop-on-error = true
+password-file = ${directory:etc}/.monitor_pwd
 htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
 # Keep multiple lines as password can end with newline char.
 command = 
   if [ ! -s "${:htpasswd-path}" ]; then 
     {{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}
   fi
+  if [ ! -s "${:password-file}" ]; then echo "${monitor-instance-parameter:password}" > ${:password-file}; fi
 update-command = ${:command}
 user = ${monitor-instance-parameter:username}
 password = ${monitor-instance-parameter:password}
@@ -284,7 +282,7 @@ check-secure = 1
 dash_path = {{ dash_executable_location }}
 curl_path = {{ curl_executable_location }}
 
-[publish]
+[monitor-publish-parameters]
 # XXX depends on monitor-base section
 monitor-base-url = ${monitor-base:base-url}
 monitor-url = ${:monitor-base-url}/public/feeds
@@ -305,7 +303,7 @@ cors-domains = monitor.app.officejs.com
 collector-db = /srv/slapgrid/var/data-log/collector.db
 # Credentials
 password = ${monitor-htpasswd:passwd}
-username = ${monitor-htpasswd:username}
+username = admin
 # XXX: type key value
 # ex raw monitor-password resqdsdsd34
 instance-configuration =
@@ -337,6 +335,7 @@ check-secure = 1
 [monitor-bootstrap-promise]
 recipe = collective.recipe.template
 file = ${monitor-conf-parameters:promise-output-file}
+error-log-file = ${buildout:directory}/.${slap-connection:partition-id}_${start-monitor:name}.log
 input = inline:#!{{ dash_executable_location }}
   pidfile=${monitor-conf-parameters:pid-file}
   if [ -s $pidfile ]; then
@@ -351,7 +350,18 @@ input = inline:#!{{ dash_executable_location }}
       fi
     done
   fi
-  if [ ! -f "${:file}" ]; then echo "Monitor bootstrap exited with error." && exit 2; else echo "Bootstrap OK"; fi
+  if [ ! -f "${:file}" ]; then
+    echo "Monitor bootstrap exited with error."
+    log_file="${:error-log-file}"
+    if [ -s "$log_file" ]; then
+      echo " ---- Latest monitor-boostrap.log ----"
+      echo ""
+      tail -n 3 $log_file
+    fi
+    exit 2
+  else
+    echo "Bootstrap OK";
+  fi
 output = ${directory:promises}/monitor-bootstrap-status
 mode = 700
 
@@ -399,8 +409,8 @@ depends =
   ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
 
 [monitor-publish]
-monitor-base-url = ${publish:monitor-base-url}
-monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
 
 
 [buildout]

stack/monitor/templates/monitor.conf.in

 [monitor]
 {% for key, value in parameter_dict.items() -%}
+{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
+{{ key }} =
+{% for key, value in monitor_base_urls.items() -%}
+{{ '  ' ~ value }}
+{% endfor -%}
+{% else -%}
 {{ key }} = {{ value.strip().replace("\n", "\n  ") }}
-{% endfor -%}
+{% endif -%}
\ No newline at end of file
+{% endfor -%}

stack/resilient/buildout.cfg

@@ -121,6 +121,10 @@ download-only = true
 filename = monitor-check-resilient-feed.in
 mode = 0644
 
+[exporter-default-configuration]
+# Time in minutes to wait for the backup to finish
+backup_wait_time = 10
+
 [rdiff-backup-build]
 # use our own version
 find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz

stack/resilient/buildout.hash.cfg

@@ -15,23 +15,23 @@
 # not need these here).
 [pbsready]
 filename = pbsready.cfg.in
-md5sum = 2adb49f45d27cbcb7430c98c2224d76a
+md5sum = bceb082deb6b3ee9b015c8b04436b52c
 
 [pbsready-import]
 filename = pbsready-import.cfg.in
-md5sum = a8f77b42782ec5e1fb49b5ae9891c757
+md5sum = 9d36d08ac6ae351b598a67db41657cc6
 
 [pbsready-export]
 filename = pbsready-export.cfg.in
-md5sum = bb463da1461d02d575717f1d1572bf97
+md5sum = c6c11db5372150019debb1ce519b907d
 
 [template-pull-backup]
 filename = instance-pull-backup.cfg.in
-md5sum = 7b4f8ac1a62680d624ac632f9601dab5
+md5sum = fe4811540683bfcfac5ac12de084b623
 
 [template-replicated]
 filename = template-replicated.cfg.in
-md5sum = 75686abd9cb8d6e1f4e15931e280fa56
+md5sum = 55ef107f499e5e604b8d277098618fed
 
 [template-parts]
 filename = template-parts.cfg.in

stack/resilient/instance-pull-backup.cfg.in

@@ -137,7 +137,7 @@ slave-instance-list = $${slap-parameter:slave_instance_list}
 ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
 # To get a verbose feed about PBS state
 instance-root-name = $${instance-info-parameters:root-name}
-log-url = $${publish:monitor-base-url}/private/notifier/
+log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
 status-item-directory = $${directory:notifier-status-items}
 
 [pbs-resilient-status-feed]
@@ -260,11 +260,6 @@ parameters-extra = true
 #----------------
 #--
 #-- Slave instance list (empty default).
-[htpasswd]
-recipe = slapos.cookbook:generate.password
-storage-path = $${directory:etc}/.monitor_user
-bytes = 8
-username = admin
 
 [slap-parameter]
 slave_instance_list = []
@@ -272,8 +267,8 @@ ignore-known-hosts-file = false
 monitor-cors-domains = 
 monitor-httpd-port = 8070
 monitor-title = PBS Instance
-monitor-password = $${htpasswd:passwd}
-monitor-username = $${htpasswd:username}
+monitor-password = $${monitor-htpasswd:passwd}
+monitor-username = admin
 
 #----------------
 #--
@@ -341,10 +336,10 @@ recipe = slapos.cookbook:publish
 ssh-key = $${sshkeys-openssh:public-key-value}
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
-monitor-base-url = $${publish:monitor-base-url}
-monitor-url = $${publish:monitor-url}
-monitor-user = $${publish:monitor-user}
-monitor-password = $${publish:monitor-password}
+monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
+monitor-url = $${monitor-publish-parameters:monitor-url}
+monitor-user = $${monitor-publish-parameters:monitor-user}
+monitor-password = $${monitor-publish-parameters:monitor-password}
 
 #----------------
 #--

stack/resilient/pbsready-export.cfg.in

@@ -44,7 +44,7 @@ pidfile = ${resilient-directory:pid}/${:name}.pid
 max-run = 3
 
 [logrotate-entry-notifier]
-output = ${rootdirectory:etc}/logrotate_notifier.conf
+rendered = ${rootdirectory:etc}/logrotate_notifier.conf
 
 [notifier-exporter-promise]
 recipe = slapos.recipe.template:jinja2
@@ -70,7 +70,7 @@ frequency = {{ resiliency_backup_periodicity }}
 {% else %}
 time = ${publish-early:resiliency-backup-periodicity}
 {% endif %}
-command = {{ logrotate_executable_location }} -s ${basedirectory:run}/logrotate.status ${logrotate-entry-notifier:output}; ${notifier-exporter:wrapper} --transaction-id `date +%s`
+command = {{ logrotate_executable_location }} -s ${basedirectory:run}/logrotate.status ${logrotate-entry-notifier:rendered}; ${notifier-exporter:wrapper} --transaction-id `date +%s`
 
 [gen-resiliency-backup-periodicity]
 recipe = slapos.cookbook:random.time
@@ -83,3 +83,4 @@ recipe = slapos.cookbook:publish-early
 [slap-parameter]
 # In cron.d format (i.e things like */15 * * * * are accepted).
 resiliency-backup-periodicity =
+notify =
\ No newline at end of file

stack/resilient/pbsready-import.cfg.in

@@ -14,6 +14,10 @@ parts +=
 
   backup-signature-link
 
+
+[slap-parameter]
+on-notification =
+
 [resilient-publish-connection-parameter]
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
 takeover-url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
@@ -107,7 +111,7 @@ input = ${resilient-web-takeover-cgi-script-download:target}
 output = $${directory:cgi-bin}/web-takeover.cgi
 password = $${resilient-web-takeover-password:passwd}
 mode = 700
-proof-signature-url = $${publish:monitor-base-url}/private/resilient/backup.signature
+proof-signature-url = $${monitor-publish-parameters:monitor-base-url}/private/resilient/backup.signature
 
 # XXX could it be something lighter?
 # XXX Add SSL

stack/resilient/pbsready.cfg.in

@@ -61,6 +61,10 @@ logrotate-backup = $${basedirectory:backup}/logrotate
 cgi-bin = $${rootdirectory:srv}/cgi-bin
 monitor-resilient = $${monitor-directory:private}/resilient
 
+
+[slap-parameter]
+authorized-key =
+
 #----------------
 #--
 #-- Deploy cron.
@@ -132,7 +136,7 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 host = $${slap-network-information:global-ipv6}
 port = $${notifier-port:port}
 instance-root-name = $${instance-info-parameters:root-name}
-log-url = $${publish:monitor-base-url}/resilient/notifier-status-rss
+log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss
 status-item-directory = $${directory:notifier-status-items}
 context =
   key content notifier:command

stack/resilient/template-replicated.cfg.in

@@ -91,6 +91,10 @@ return = ssh-public-key ssh-url notification-url ip takeover-url takeover-passwo
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
 
+{% if slapparameter_dict.get("software-root") %}
+config-software-root = {{ slapparameter_dict.get("software-root") }}
+{% endif %}
+
 config-number = {{id}}
 config-name = {{namebase}}{{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}

stack/shacache-client.cfg

@@ -3,5 +3,5 @@
 networkcache-section = networkcache
 
 [networkcache]
-download-cache-url = http://www.shacache.org/shacache
-download-dir-url = http://www.shacache.org/shadir
+download-cache-url = http://download.shacache.org
+download-dir-url = http://dir.shacache.org

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos007
+zc.buildout = 2.5.2+slapos009
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -137,14 +137,15 @@ setuptools = 33.1.1
 six = 1.10.0
 slapos.cookbook = 1.0.50
 slapos.core = 1.3.18
-slapos.extension.strip = 0.2
-slapos.libnetworkcache = 0.14.5
-slapos.rebootstrap = 3.10
-slapos.recipe.build = 0.33
-slapos.recipe.cmmi = 0.5
+slapos.extension.strip = 0.4
+slapos.libnetworkcache = 0.15
+slapos.rebootstrap = 4.1
+slapos.recipe.build = 0.36
+slapos.recipe.cmmi = 0.7
 stevedore = 1.21.0
 unicodecsv = 0.14.1
 xml-marshaller = 0.9.7
+paramiko = 2.1.3
 
 # Required by:
 # slapos.core==1.3.18
@@ -192,8 +193,8 @@ uritemplate = 3.0.0
 zope.interface = 4.3.3
 
 [networkcache]
-download-cache-url = http://www.shacache.org/shacache
-download-dir-url = http://www.shacache.org/shadir
+download-cache-url = http://download.shacache.org/
+download-dir-url = http://dir.shacache.org/
 
 # signature certificates of the following uploaders.
 #   Romain Courteaud
@@ -204,6 +205,7 @@ download-dir-url = http://www.shacache.org/shadir
 #   Aurélien Calonne
 #   Rafael Monnerat
 #   Ivan Tyagov
+#   Julien Muchembled
 signature-certificate-list =
   -----BEGIN CERTIFICATE-----
   MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
@@ -309,3 +311,16 @@ signature-certificate-list =
   8dEAOMIcdrPBzldf0JS+poCrLUw7qx9VZxa0ee3+F6WGEgsr2xhu2WpmTzpRWaFR
   0I+saiiJCkOkLoyLxeQcgwe62em/DGn6PdcmRDYaVXBHygrRXus=
   -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB6jCCAVOgAwIBAgIJAMhx/tG6zHFJMA0GCSqGSIb3DQEBBQUAMA0xCzAJBgNV
+  BAMMAmptMCAXDTEzMDcwMTE3NDcyOVoYDzIxMTMwNjA3MTc0NzI5WjANMQswCQYD
+  VQQDDAJqbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAyFJp6K3HAle6oUB4
+  /TxIvc06wju+ZzZuPX4fxq2GqOV9QX2jhPfYCyr/cJh4pOZEW1gwMw+9kzghmjNT
+  6RoBEWMS3E0F2K/waYb+doGb1Dffp2FIi3m547b20XnpnaIj3DS7FF30lRNo8jtR
+  1Dg+RDGpA23Pr1/3IAskC5qSydkCAwEAAaNQME4wHQYDVR0OBBYEFHHmUnqEVNEo
+  FpLiBjCmR5ZJsPndMB8GA1UdIwQYMBaAFHHmUnqEVNEoFpLiBjCmR5ZJsPndMAwG
+  A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAPP5oOZWVrYjc5BB+pDVQm/rF
+  gVhgPPg+23oCiz0p9NzWgnYXhGmQWtzXVVVaL4aftkmX04mlT2Fgd4msEzQB6pAa
+  SOvU9E1802JN7Q1mrMbgJeNmk+18o4ze7ABKljqtvo+ZjUXthuWw3BNt+PtQjlX3
+  5wE+Yc4sypYGmAfU7Gc=
+  -----END CERTIFICATE-----

validate-schema

+#!/usr/bin/env python
+import argparse
+import json
+import os.path
+import urllib
+from urlparse import urlparse, urlunparse, ParseResult
+import jsonschema
+
+# Adapted from slapos.core.git/slapos/slap/util.py
+from lxml import etree
+def xml2dict(infile):
+    result_dict = {}
+    for element in etree.parse(infile).iter(tag=etree.Element):
+        if element.tag == 'parameter':
+            key = element.get('id')
+            value = result_dict.get(key, None)
+            if value is not None:
+                value = value + ' ' + element.text
+            else:
+                value = element.text
+            result_dict[key] = value
+    return result_dict
+
+def jsonInXML2dict(infile):
+    wrapped = xml2dict(infile)
+    if '_' in wrapped:
+        return json.loads(wrapped['_'])
+    return wrapped
+
+class ValidatingRefResolver(jsonschema.RefResolver):
+    def resolve_remote(self, *args, **kw):
+        result = super(ValidatingRefResolver, self).resolve_remote(*args, **kw)
+        jsonschema.Draft4Validator.check_schema(result)
+        return result
+
+DEFAULT_SLAPOS_SCHEMA_PATH = os.path.join(
+    os.path.dirname(__file__),
+    'schema.json',
+)
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Validates SlapOS Software Release descriptor schemas, '
+        'and optionally validates specific requests and/or responses with '
+        'these schemas.',
+    )
+    parser.add_argument('--software-type', type=str, help='Software type given request/response is intended for')
+    parser.add_argument('--request', type=argparse.FileType('r'), help='File containing request parameters')
+    parser.add_argument('--response', type=argparse.FileType('r'), help='File containing published values')
+    parser.add_argument('--slapos-schema', type=argparse.FileType('r'), help='SlapOS base schema. Default: %s' % (DEFAULT_SLAPOS_SCHEMA_PATH, ))
+    parser.add_argument('software_release', nargs=1, help='URL of the software release')
+    args = parser.parse_args()
+
+    if args.software_type is None and (
+            args.request is not None or
+            args.response is not None
+        ):
+        parser.error(
+            '--software-type is required if --request or --response is provided',
+        )
+
+    slapos_schema_file = args.slapos_schema
+    if slapos_schema_file is None:
+        slapos_schema_file = open(DEFAULT_SLAPOS_SCHEMA_PATH)
+    slapos_schema = json.load(slapos_schema_file)
+    jsonschema.Draft4Validator.check_schema(slapos_schema)
+
+    software_release_url = urlparse(args.software_release[0])
+    if software_release_url.scheme == software_release_url.netloc == '':
+      # Assume software_release_url.path is a relative path for "file" scheme
+      software_release_url = ParseResult(
+        'file',
+        '',
+        os.path.abspath(software_release_url.path),
+        software_release_url.params,
+        software_release_url.query,
+        software_release_url.fragment,
+      )
+    software_release_base_path, software_release_file = software_release_url.path.rsplit('/', 1)
+    def loadurl(relative_url):
+        url = urlunparse((
+            software_release_url.scheme,
+            software_release_url.netloc,
+            software_release_base_path + '/' + relative_url,
+            software_release_url.params,
+            software_release_url.query,
+            software_release_url.fragment,
+        ))
+        return url, json.loads(urllib.urlopen(url).read())
+
+    _, software_release_descriptor = loadurl(software_release_file + '.json')
+    jsonschema.Draft4Validator.check_schema(software_release_descriptor)
+    jsonschema.validate(software_release_descriptor, slapos_schema)
+    load = {
+        'xml': xml2dict,
+        'json-in-xml': jsonInXML2dict,
+    }[software_release_descriptor['serialisation']]
+    for software_type_name, software_type in software_release_descriptor['software-type'].iteritems():
+        for key in ('request', 'response'):
+            relative_url = software_type[key]
+            url, software_release_schema = loadurl(relative_url)
+            jsonschema.Draft4Validator.check_schema(software_release_schema)
+            validator = jsonschema.Draft4Validator(
+                schema=software_release_schema,
+                resolver=ValidatingRefResolver(
+                    base_uri=url,
+                    referrer=software_release_schema,
+                ),
+            )
+            infile = getattr(args, key)
+            if infile is not None and software_type_name == args.software_type:
+                validator.validate(load(infile))
+            else:
+                validator.validate({})
+
+if __name__ == '__main__':
+    main()