Merge branch 'master' into erp5-cluster

Conflicts:
	software/neoppod/software.cfg
	stack/erp5/buildout.cfg
	stack/erp5/instance-erp5-cluster.cfg.in
	stack/erp5/instance-erp5-single.cfg.in
	stack/erp5/instance-zope.cfg.in

component/apache/buildout.cfg

@@ -17,15 +17,15 @@ extends =
 
 [apr]
 recipe = hexagonit.recipe.download
-version = 1.5.0
-md5sum = cc93bd2c12d0d037f68e21cc6385dc31
+version = 1.5.1
+md5sum = 5486180ec5a23efb5cae6d4292b300ab
 url = https://archive.apache.org/dist/apr/apr-${:version}.tar.bz2
 
 [apr-util]
 recipe = hexagonit.recipe.download
-version = 1.5.3
+version = 1.5.4
 url = https://archive.apache.org/dist/apr/apr-util-${:version}.tar.bz2
-md5sum = 6f3417691c7a27090f36e7cf4d94b36e
+md5sum = 2202b18f269ad606d70e1864857ed93c
 
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/

component/bash/buildout.cfg

@@ -38,6 +38,8 @@ patches =
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-026#922578e2be7ed03729454e92ee8d3f3a
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-027#8ff6948b16f2db5c29b1b9ae1085bbe7
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-028#dd51fa67913b5dca45a702b672b3323f
+  http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-029#0729364c977ef4271e9f8dfafadacf67
+  http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-030#efb709fdb1368945513de23ccbfae053
 configure-options =
   --with-curses
 environment =

component/ca-certificates/buildout.cfg

@@ -11,10 +11,10 @@ parts =
 
 [ca-certificates]
 recipe = slapos.recipe.cmmi
-version = 20140325
+version = 20141019
 url = ftp://ftp.free.fr/mirrors/ftp.debian.org/pool/main/c/ca-certificates/ca-certificates_${:version}.tar.xz
 patch-binary = ${patch:location}/bin/patch
-md5sum = 0436aba482091da310bd762e1deca8b4
+md5sum = f619282081c8bfc65ea64c37fa5285ed
 patches =
   ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
 patch-options = -p0

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.38.0.tar.bz2
-md5sum = af6b3c299bd891f43cb5f76c4091b7b4
+url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2
+md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd
 configure-options =
   --disable-static
   --disable-ldap

component/file/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.19.tar.gz
-md5sum = e3526f59023f3f7d1ffa4d541335edab
+url = ftp://ftp.astron.com/pub/file/file-5.20.tar.gz
+md5sum = 5d5e13eb3e0e13839da869a31790faf2
 configure-options =
   --disable-static
 environment =

component/firefox/buildout.cfg

@@ -26,12 +26,12 @@ depends =
   ${liberation-fonts:location}
   ${ipaex-fonts:location}
 
-version = 32.0
+version = 33.0
 
 # MD5SUMs are available at :
 # https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS
-x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 f086b7d0ac2421232d5cb31857980d2e
-x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ec3b3976caa4ec5b3de137d059c74f8d
+x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 542b27d5f3afcb55406a7af72f68f3d5
+x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ae4e8275750fd5f87963cbe263e8ba34
 
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
@@ -42,7 +42,7 @@ script =
   wrapper = open(wrapper_location, 'w')
   wrapper.write("""#!${dash:location}/bin/dash
   cd %(location)s
-  export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${pango:location}/lib:${p11-kit:location}/lib:${pixman:location}/lib:${xfixes:location}/lib:${zlib:location}/lib
+  export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${p11-kit:location}/lib:${pango:location}/lib:${pixman:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
   exec %(location)s/firefox $*""")
   wrapper.close()

component/gcc/buildout.cfg

@@ -4,6 +4,7 @@
 extends =
   ../m4/buildout.cfg
   ../gmp/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zip/buildout.cfg
 
 parts =
@@ -11,59 +12,51 @@ parts =
 
 [mpfr]
 recipe = slapos.recipe.cmmi
-url = http://www.mpfr.org/mpfr-3.0.1/mpfr-3.0.1.tar.bz2
-md5sum = bfbecb2eacb6d48432ead5cfc3f7390a
+url = http://ftp.gnu.org/gnu/mpfr/mpfr-3.1.2.tar.xz
+md5sum = e3d203d188b8fe60bb6578dd3152e05c
 configure-options =
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --disable-static
 environment =
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib
+  PATH=${xz-utils:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib
 
 [mpc]
 recipe = slapos.recipe.cmmi
-url = http://www.multiprecision.org/mpc/download/mpc-0.9.tar.gz
-md5sum = 0d6acab8d214bd7d1fbbc593e83dd00d
+url = http://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz
+md5sum = 68fadff3358fb3e7976c7a398a0af4c3
 configure-options =
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --disable-static
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
-[ppl]
+[isl]
 recipe = slapos.recipe.cmmi
-# we should use version 0.11.x for gcc-4.7
-url = http://bugseng.com/external/ppl/download/ftp/releases/0.11/ppl-0.11.tar.bz2
-md5sum = c414abd2ecaf1f1f48949c78d383ee1a
-patches =
-  ${:_profile_base_location_}/ppl-0.11-gcc-4.9.patch#09e00eefba5f8e6e77a9c9c611fccc64
-
+url = ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.12.2.tar.bz2
+md5sum = e039bfcfb6c2ab039b8ee69bf883e824
 configure-options =
-  --with-gmp=${gmp-4:location}
-  --with-gmp-prefix=${gmp-4:location}
+  --with-gmp-prefix=${gmp:location}
   --disable-static
 environment =
-  PATH=${m4:location}/bin:%(PATH)s
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib
-  CXXFLAGS=-fpermissive
-
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib
 
-[cloog-ppl]
+[cloog]
 recipe = slapos.recipe.cmmi
-url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.16.1.tar.gz
-md5sum = 947123350d1ff6dcb4b0774947ac015a
+url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.18.1.tar.gz
+md5sum = e34fca0540d840e5d0f6427e98c92252
 configure-options =
-  --with-gmp-prefix=${gmp-4:location}
-  --with-ppl=${ppl:location}
+  --with-gmp-prefix=${gmp:location}
+  --with-isl-prefix=${isl:location}
   --disable-static
 environment =
-  PATH=${m4:location}/bin:%(PATH)s
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.bz2
-md5sum = 4c696da46297de6ae77a82797d2abe28
+url = http://ftp.gnu.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2
+md5sum = 4df8ee253b7f3863ad0b86359cd39c43
 # make install does not work when several core are used
 make-targets = install -j1
 
@@ -72,41 +65,39 @@ make-targets = install -j1
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages="c,c++"
-  --with-ppl=${ppl:location}
-  --with-cloog=${cloog-ppl:location}
-  --enable-cloog-backend=isl
+  --with-isl=${isl:location}
+  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
 [gcc-minimal]
 <= gcc-common
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages=c
-  --without-ppl
+  --without-isl
   --without-cloog
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
 [gcc-fortran]
 <= gcc-common
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages="c,c++,fortran"
-  --with-ppl=${ppl:location}
-  --with-cloog=${cloog-ppl:location}
-  --enable-cloog-backend=isl
+  --with-isl=${isl:location}
+  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib

component/gcc/gcc-java-pre-4.4.patch

-Patch for http://gcc.gnu.org/bugzilla/show_bug.cgi?id=50888
-
---- a/src/libjava/libjava/prims.cc.orig	2012-01-20 11:30:18.586157610 +0100
-+++ b/src/libjava/libjava/prims.cc	2012-01-20 11:30:58.192770947 +0100
-@@ -38,6 +38,14 @@
- #endif
- 
- #ifndef DISABLE_GETENV_PROPERTIES
-+#ifdef __GLIBC__
-+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
-+   Unfortunately those inlines are throw (), and call a function pointer
-+   (which is throw () too, but with -fnon-call-exceptions this results
-+   in a __cxa_call_unexpected call.  This macro disables the optimized
-+   version.  */
-+#define __NO_CTYPE 1
-+#endif
- #include <ctype.h>
- #include <java-props.h>
- #define PROCESS_GCJ_PROPERTIES process_gcj_properties()
---- a/src/libjava/prims.cc.orig	2012-01-20 11:30:23.042818341 +0100
-+++ b/src/libjava/prims.cc	2012-01-20 11:31:01.389433254 +0100
-@@ -38,6 +38,14 @@
- #endif
- 
- #ifndef DISABLE_GETENV_PROPERTIES
-+#ifdef __GLIBC__
-+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
-+   Unfortunately those inlines are throw (), and call a function pointer
-+   (which is throw () too, but with -fnon-call-exceptions this results
-+   in a __cxa_call_unexpected call.  This macro disables the optimized
-+   version.  */
-+#define __NO_CTYPE 1
-+#endif
- #include <ctype.h>
- #include <java-props.h>
- #define PROCESS_GCJ_PROPERTIES process_gcj_properties()

component/gcc/ppl-0.11-gcc-4.9.patch

-# HG changeset patch
-# User Bernhard Walle <bernhard@bwalle.de>
-# Date 1399806036 -7200
-#      Sun May 11 13:00:36 2014 +0200
-# Node ID 5632a171e7c1dc4c0fb5c7a8352afb4188cd29d5
-# Parent  e35fa03cd2043496839aa5fa3a1a3d2d375b5535
-complibs/ppl: Fix build of ppl 0.11.2 with gcc 4.9
-
-  From: Roberto Bagnara <bagnara@cs.unipr.it>
-  Date: Tue, 29 Apr 2014 19:51:43 +0000 (+0200)
-  Subject: Added missing inclusions.  Use std::ptrdiff_t.
-  X-Git-Url: http://www.cs.unipr.it/git/gitweb.cgi?p=ppl%2Fppl.git;a=commitdiff_plain;h=61d4e14dfd9f1121e9b4521dead5728b2424dd7c
-
-  Added missing inclusions.  Use std::ptrdiff_t.
-  (Thanks to Paulo Cesar Pereira de Andrade.)
-
-Signed-off-by: Bernhard Walle <bernhard@bwalle.de>
-
-diff -r e35fa03cd204 -r 5632a171e7c1 patches/ppl/0.11.2/500-ptrdiff_t.patch
---- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/patches/ppl/0.11.2/500-ptrdiff_t.patch	Sun May 11 13:00:36 2014 +0200
-@@ -0,0 +1,95 @@
-+From: Bernhard Walle <bernhard@bwalle.de>
-+Subject: Fix compilation with gcc 4.9 (ptrdiff_t errors)
-+
-+This is a backport of following patch in ppl git
-+
-+  commit 61d4e14dfd9f1121e9b4521dead5728b2424dd7c
-+  Author: Roberto Bagnara <bagnara@cs.unipr.it>
-+  Date:   Tue Apr 29 21:51:43 2014 +0200
-+
-+      Added missing inclusions.  Use std::ptrdiff_t.
-+      (Thanks to Paulo Cesar Pereira de Andrade.)
-+
-+---
-+ src/Congruence_System.defs.hh     |    3 ++-
-+ src/Constraint_System.defs.hh     |    3 ++-
-+ src/Generator_System.defs.hh      |    3 ++-
-+ src/Grid_Generator_System.defs.hh |    3 ++-
-+ 4 files changed, 8 insertions(+), 4 deletions(-)
-+
-+--- a/src/Grid_Generator_System.defs.hh
-++++ b/src/Grid_Generator_System.defs.hh
-+@@ -30,6 +30,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Variables_Set.types.hh"
-+ #include "Grid.types.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -267,7 +268,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Grid_Generator,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Grid_Generator*,
-+ 			   const Grid_Generator&>,
-+       private Generator_System::const_iterator {
-+--- a/src/Congruence_System.defs.hh
-++++ b/src/Congruence_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Grid.types.hh"
-+ #include "Grid_Certificate.types.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -235,7 +236,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Congruence,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Congruence*,
-+ 			   const Congruence&> {
-+   public:
-+--- a/src/Constraint_System.defs.hh
-++++ b/src/Constraint_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Congruence_System.types.hh"
-+ #include <iterator>
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -204,7 +205,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Constraint,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Constraint*,
-+ 			   const Constraint&> {
-+   public:
-+--- a/src/Generator_System.defs.hh
-++++ b/src/Generator_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Polyhedron.types.hh"
-+ #include "Poly_Con_Relation.defs.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -250,7 +251,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Generator,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Generator*,
-+ 			   const Generator&> {
-+   public:

component/gettext/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.2.tar.xz
-md5sum = 1e6a827f5fbd98b3d40bd16b803acc44
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.3.tar.xz
+md5sum = 092c3f460553ceb4a638ff81d36434c4
 
 configure-options =
   --disable-static

component/ghostscript/buildout.cfg

@@ -33,8 +33,8 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.14.tar.bz2
-md5sum = 75f2e8ab8891d052ade9b64eb4eb5294
+url = http://downloads.ghostscript.com/public/ghostscript-9.15.tar.bz2
+md5sum = 0ee21d975c67a4883ba66bcc332418f5
 
 [ghostscript-fonts]
 recipe = hexagonit.recipe.download

component/gmp/buildout.cfg

@@ -6,21 +6,6 @@ extends =
 parts =
   gmp
 
-# we should use version 4.x for ppl-0.10
-[gmp-4]
-recipe = slapos.recipe.cmmi
-url = ftp://ftp.gmplib.org/pub/gmp-4.3.2/gmp-4.3.2.tar.bz2
-md5sum = dd60683d7057917e34630b4a787932e8
-# GMP does not correctly detect achitecture so it have to be given
-# as slapos.recipe.cmmi is using shell expansion in subproceses
-# backticks are working
-configure-options =
-  --build=`uname -m`-linux
-  --enable-cxx
-  --disable-static
-environment =
-  PATH=${m4:location}/bin:%(PATH)s
-
 [gmp]
 recipe = slapos.recipe.cmmi
 version = 6.0.0a

component/gnutls/buildout.cfg

@@ -11,8 +11,8 @@ parts = gnutls
 
 [gpg-error]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.13.tar.bz2
-md5sum = fe0cfa7e15262ef8fdeee366109e9ff6
+url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.17.tar.bz2
+md5sum = b4f8d8b9ff14aed41f279aa844563539
 
 [gcrypt]
 recipe = slapos.recipe.cmmi
@@ -26,8 +26,8 @@ environment =
 
 [gnutls]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.16.tar.xz
-md5sum = d9a3fb49121b49bc332bfe9c3f49b005
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.19.tar.xz
+md5sum = 8884492d0ad4339c831c6d507afbeea9
 configure-options =
   --disable-static
   --disable-libdane

component/groonga/buildout.cfg

@@ -11,9 +11,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-version = 4.0.6
-url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = d7eb0296b0b759d1856c31b6288670fa
+url = http://packages.groonga.org/source/groonga/groonga-4.0.7.tar.gz
+md5sum = 5d11546f921377d040f700bb74f1d7e4
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/gzip/buildout.cfg

@@ -10,3 +10,12 @@ url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
 md5sum = da981f86677d58a106496e68de6f8995
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
+patch-options = -p1
+# The --rsyncable patch is from debian/ubuntu,
+# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
+# It is required to minimize the bandwidth used by rsync.
+# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
+# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
+patches =
+  ${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
+

component/gzip/rsyncable.diff

+--- a/deflate.c
++++ b/deflate.c
+@@ -131,6 +131,14 @@
+ #endif
+ /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+ 
++#ifndef RSYNC_WIN
++#  define RSYNC_WIN 8192
++#endif
++/* Size of rsync window, must be < MAX_DIST */
++
++#define RSYNC_SUM_MATCH(sum) (((sum) & (RSYNC_WIN - 1)) == 0)
++/* Whether window sum matches magic value */
++
+ /* ===========================================================================
+  * Local data used by the "longest match" routines.
+  */
+@@ -212,6 +220,8 @@
+ unsigned good_match;
+ /* Use a faster search when the previous match is longer than this */
+ 
++local ulg rsync_sum;  /* rolling sum of rsync window */
++local ulg rsync_chunk_end; /* next rsync sequence point */
+ 
+ /* Values for max_lazy_match, good_match and max_chain_length, depending on
+  * the desired pack level (0..9). The values given below have been tuned to
+@@ -314,6 +324,10 @@
+ #endif
+     /* prev will be initialized on the fly */
+ 
++    /* rsync params */
++    rsync_chunk_end = 0xFFFFFFFFUL;
++    rsync_sum = 0;
++
+     /* Set the default configuration parameters:
+      */
+     max_lazy_match   = configuration_table[pack_level].max_lazy;
+@@ -550,6 +564,8 @@
+         memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
+         match_start -= WSIZE;
+         strstart    -= WSIZE; /* we now have strstart >= MAX_DIST: */
++	if (rsync_chunk_end != 0xFFFFFFFFUL)
++	    rsync_chunk_end -= WSIZE;
+ 
+         block_start -= (long) WSIZE;
+ 
+@@ -579,13 +595,44 @@
+     }
+ }
+ 
++local void rsync_roll(unsigned start, unsigned num)
++{
++    unsigned i;
++
++    if (start < RSYNC_WIN) {
++	/* before window fills. */
++	for (i = start; i < RSYNC_WIN; i++) {
++	    if (i == start + num) return;
++	    rsync_sum += (ulg)window[i];
++	}
++	num -= (RSYNC_WIN - start);
++	start = RSYNC_WIN;
++    }
++
++    /* buffer after window full */
++    for (i = start; i < start+num; i++) {
++	/* New character in */
++	rsync_sum += (ulg)window[i];
++	/* Old character out */
++	rsync_sum -= (ulg)window[i - RSYNC_WIN];
++	if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
++	    rsync_chunk_end = i;
++    }
++}
++
++/* ===========================================================================
++ * Set rsync_chunk_end if window sum matches magic value.
++ */
++#define RSYNC_ROLL(s, n) \
++   do { if (rsync) rsync_roll((s), (n)); } while(0)
++
+ /* ===========================================================================
+  * Flush the current block, with given end-of-file flag.
+  * IN assertion: strstart is set to the end of the current match.
+  */
+ #define FLUSH_BLOCK(eof) \
+    flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
+-                (char*)NULL, (long)strstart - block_start, (eof))
++                (char*)NULL, (long)strstart - block_start, flush-1, (eof))
+ 
+ /* ===========================================================================
+  * Processes a new input file and return its compressed length. This
+@@ -596,7 +643,7 @@
+ local off_t deflate_fast()
+ {
+     IPos hash_head; /* head of the hash chain */
+-    int flush;      /* set if current block must be flushed */
++    int flush = 0;      /* set if current block must be flushed, 2=>and padded  */
+     unsigned match_length = 0;  /* length of best match */
+ 
+     prev_length = MIN_MATCH-1;
+@@ -626,7 +673,8 @@
+ 
+             lookahead -= match_length;
+ 
+-            /* Insert new strings in the hash table only if the match length
++	    RSYNC_ROLL(strstart, match_length);
++	    /* Insert new strings in the hash table only if the match length
+              * is not too large. This saves time but degrades compression.
+              */
+             if (match_length <= max_insert_length) {
+@@ -654,9 +702,14 @@
+             /* No match, output a literal byte */
+             Tracevv((stderr,"%c",window[strstart]));
+             flush = ct_tally (0, window[strstart]);
++	    RSYNC_ROLL(strstart, 1);
+             lookahead--;
+             strstart++;
+         }
++        if (rsync && strstart > rsync_chunk_end) {
++            flush = 2;
++            rsync_chunk_end = 0xFFFFFFFFUL;
++        }
+         if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ 
+         /* Make sure that we always have enough lookahead, except
+@@ -679,7 +732,7 @@
+ {
+     IPos hash_head;          /* head of hash chain */
+     IPos prev_match;         /* previous match */
+-    int flush;               /* set if current block must be flushed */
++    int flush = 0;           /* set if current block must be flushed */
+     int match_available = 0; /* set if previous match exists */
+     register unsigned match_length = MIN_MATCH-1; /* length of best match */
+ 
+@@ -730,6 +783,7 @@
+              */
+             lookahead -= prev_length-1;
+             prev_length -= 2;
++	    RSYNC_ROLL(strstart, prev_length+1);
+             do {
+                 strstart++;
+                 INSERT_STRING(strstart, hash_head);
+@@ -742,24 +796,38 @@
+             match_available = 0;
+             match_length = MIN_MATCH-1;
+             strstart++;
+-            if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ 
++	    if (rsync && strstart > rsync_chunk_end) {
++		rsync_chunk_end = 0xFFFFFFFFUL;
++		flush = 2;
++	    }
++            if (flush) FLUSH_BLOCK(0), block_start = strstart;
+         } else if (match_available) {
+             /* If there was no match at the previous position, output a
+              * single literal. If there was a match but the current match
+              * is longer, truncate the previous match to a single literal.
+              */
+             Tracevv((stderr,"%c",window[strstart-1]));
+-            if (ct_tally (0, window[strstart-1])) {
+-                FLUSH_BLOCK(0), block_start = strstart;
+-            }
++	    flush = ct_tally (0, window[strstart-1]);
++	    if (rsync && strstart > rsync_chunk_end) {
++		rsync_chunk_end = 0xFFFFFFFFUL;
++		flush = 2;
++	    }
++            if (flush) FLUSH_BLOCK(0), block_start = strstart;
++	    RSYNC_ROLL(strstart, 1);
+             strstart++;
+             lookahead--;
+         } else {
+             /* There is no previous match to compare with, wait for
+              * the next step to decide.
+              */
++	    if (rsync && strstart > rsync_chunk_end) {
++		rsync_chunk_end = 0xFFFFFFFFUL;
++		flush = 2;
++		FLUSH_BLOCK(0), block_start = strstart;
++	    }
+             match_available = 1;
++	    RSYNC_ROLL(strstart, 1);
+             strstart++;
+             lookahead--;
+         }
+--- a/gzip.c
++++ b/gzip.c
+@@ -215,6 +215,7 @@
+ unsigned insize;           /* valid bytes in inbuf */
+ unsigned inptr;            /* index of next byte to be processed in inbuf */
+ unsigned outcnt;           /* bytes in output buffer */
++int rsync = 0;             /* make ryncable chunks */
+ 
+ static int handled_sig[] =
+   {
+@@ -275,7 +276,7 @@
+     {"best",       0, 0, '9'}, /* compress better */
+     {"lzw",        0, 0, 'Z'}, /* make output compatible with old compress */
+     {"bits",       1, 0, 'b'}, /* max number of bits per code (implies -Z) */
+-
++    {"rsyncable",  0, 0, 'R'}, /* make rsync-friendly archive */
+     { 0, 0, 0, 0 }
+ };
+ 
+@@ -359,6 +360,7 @@
+  "  -Z, --lzw         produce output compatible with old compress",
+  "  -b, --bits=BITS   max number of bits per code (implies -Z)",
+ #endif
++ "  --rsyncable       Make rsync-friendly archive",
+  "",
+  "With no FILE, or when FILE is -, read standard input.",
+  "",
+@@ -489,8 +491,11 @@
+ #else
+             recursive = 1;
+ #endif
+-            break;
+-        case 'S':
++	    break;
++        case 'R':
++	    rsync = 1; break;
++
++	case 'S':
+ #ifdef NO_MULTIPLE_DOTS
+             if (*optarg == '.') optarg++;
+ #endif
+--- a/gzip.h
++++ b/gzip.h
+@@ -140,6 +140,7 @@
+ extern unsigned insize; /* valid bytes in inbuf */
+ extern unsigned inptr;  /* index of next byte to be processed in inbuf */
+ extern unsigned outcnt; /* bytes in output buffer */
++extern int rsync;  /* deflate into rsyncable chunks */
+ 
+ extern off_t bytes_in;   /* number of input bytes */
+ extern off_t bytes_out;  /* number of output bytes */
+@@ -287,7 +288,7 @@
+         /* in trees.c */
+ extern void ct_init     (ush *attr, int *method);
+ extern int  ct_tally    (int dist, int lc);
+-extern off_t flush_block (char *buf, ulg stored_len, int eof);
++extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
+ 
+         /* in bits.c */
+ extern void     bi_init    (file_t zipfile);
+--- a/gzip.1
++++ b/gzip.1
+@@ -5,6 +5,7 @@
+ .ll +8
+ .B gzip
+ .RB [ " \-acdfhklLnNrtvV19 " ]
++.RB [ --rsyncable ]
+ .RB [ \-S\ suffix ]
+ [
+ .I "name \&..."
+@@ -287,6 +288,16 @@
+ .I gunzip
+ ).
+ .TP
++.B --rsyncable
++While compressing, synchronize the output occasionally based on the input.
++This increases size by less than 1 percent most cases, but means that the
++.BR rsync (1)
++program can take advantage of similarities in the uncompressed input
++when syncronizing two files compressed with this flag.
++.I gunzip
++cannot tell the difference between a compressed file created with this option,
++and one created without it.
++.TP
+ .B \-S .suf   --suffix .suf
+ When compressing, use suffix .suf instead of .gz.
+ Any non-empty suffix can be given, but suffixes
+--- a/trees.c
++++ b/trees.c
+@@ -856,9 +856,10 @@
+  * trees or store, and output the encoded block to the zip file. This function
+  * returns the total compressed length for the file so far.
+  */
+-off_t flush_block(buf, stored_len, eof)
++off_t flush_block(buf, stored_len, pad, eof)
+     char *buf;        /* input block, or NULL if too old */
+     ulg stored_len;   /* length of input block */
++    int pad;          /* pad output to byte boundary */
+     int eof;          /* true if this is the last block for a file */
+ {
+     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
+@@ -951,6 +952,10 @@
+         Assert (input_len == bytes_in, "bad input size");
+         bi_windup();
+         compressed_len += 7;  /* align on byte boundary */
++    } else if (pad && (compressed_len % 8) != 0) {
++        send_bits((STORED_BLOCK<<1)+eof, 3);  /* send block type */
++        compressed_len = (compressed_len + 3 + 7) & ~7L;
++        copy_block(buf, 0, 1); /* with header */
+     }
+ 
+     return compressed_len >> 3;

component/haproxy/buildout.cfg

@@ -11,8 +11,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.3.tar.gz
-md5sum = e999a547d57445d5a5ab7eb6a06df9a1
+url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.8.tar.gz
+md5sum = 7bffa1afa069d90ce03b7cd9aa0557cd
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/java/buildout.cfg

@@ -3,51 +3,31 @@
 parts =
   java
 
-[jdk-6u27-no-user-interaction-patch]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-download-only = true
-filename = jdk-6u27-no-user-interaction.patch
-md5sum = 4c4303240647a114d07f3c411b2e6b5b
-
 [java]
 <= java-re
 
 [java-re]
-<= java-re-1.6.0
-
-[java-sdk]
-<= java-sdk-1.6.0
+<= java-re-7
 
-[java-re-1.6.0]
+[java-re-7]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
   directory:lib
   directory:man
   directory:plugin
-  directory:javaws
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual_v6.jsp
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
+# http://java.com/en/download/manual_java7.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
 script =
-  import glob
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0755)
-  subprocess.call([auto_extract_bin])
-  self.cleanup_dir_list.append(extract_dir)
+  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
+  self.copyTree(workdir, "%(location)s")
 
-[java-re-1.7.0]
+[java-re-8]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
@@ -56,9 +36,9 @@ slapos_promisee =
   directory:plugin
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual.jsp?locale=en
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
+# http://java.com/en/download/manual_java7.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97358 22d970566c418499d331a2099d77c548
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97360 f4f7f7335eaf2e7b5ff455abece9d5ed
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))

component/libatlas/buildout.cfg

@@ -30,7 +30,7 @@ configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
   --with-netlib-lapack-tarfile=${lapack-download:location}/${lapack-download:filename}
   --shared
-  -b "$(uname -p | grep -q 64 && echo 64 || echo 32)"
+  -b "$(uname -m | grep -q 64 && echo 64 || echo 32)"
   -Fa alg '-fPIC'
 make-options =
   -C build

component/libpng/buildout.cfg

@@ -23,10 +23,10 @@ md5sum = 4efba67fa5aa2b785c6fcec2cc3e90c9
 
 [libpng15]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.18.tar.xz
-md5sum = 23fb6a0e9f4dbaab7b34b6fd317a4f2d
+url = http://download.sourceforge.net/libpng/libpng-1.5.19.tar.xz
+md5sum = 1e2afb372b9979b2133829ee8136f30f
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.12.tar.xz
-md5sum = ee752da8aa75076500e4cf31318e636e
+url = http://download.sourceforge.net/libpng/libpng-1.6.14.tar.xz
+md5sum = aefd9fda2178acab908707737174a0cd

component/libreoffice-bin/buildout.cfg

@@ -9,14 +9,14 @@ parts =
 [libreoffice-bin]
 recipe = slapos.recipe.build
 # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-version = 4.2.6.3
+version = 4.3.3.2
 url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
 # supported architectures md5sums
-md5sum_x86 = 5a19002efc453a7af9443e1b01e7017a
-md5sum_x86-64 = 25d23958e8253182583e1af0ab9d1f5c
+md5sum_x86 = c24d4d7db03a5c983633efde754efae4
+md5sum_x86-64 = 94e0c1027d864f26a65e742874514b92
 
 # where office code can be found?
-officedir = libreoffice4.2
+officedir = libreoffice4.3
 
 # script to install
 script =

component/libtasn1/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [libtasn1]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/gnu/libtasn1/libtasn1-4.1.tar.gz
-md5sum = f9b37df3c2c9c6439d8bf427bfbfc521
+url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.2.tar.gz
+md5sum = 414df906df421dee0a5cf7548788d153
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/mariadb/buildout.cfg

@@ -4,16 +4,17 @@
 [buildout]
 extends =
   ../cmake/buildout.cfg
-  ../zlib/buildout.cfg
-  ../jemalloc/buildout.cfg
   ../groonga/buildout.cfg
+  ../jemalloc/buildout.cfg
   ../libaio/buildout.cfg
   ../libevent/buildout.cfg
+  ../libxml2/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
+  ../pcre/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../readline/buildout.cfg
-  ../libxml2/buildout.cfg
+  ../zlib/buildout.cfg
 
 parts =
   mariadb
@@ -23,8 +24,6 @@ recipe = slapos.recipe.cmmi
 version = 10.0.14
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
 md5sum = 80fea71de54a9cfa7f5508df53d3f06d
-# compile directory is required to build mysql plugins.
-keep-compile-dir = true
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
@@ -38,34 +37,40 @@ configure-options =
   -DWITH_ZLIB=system
   -DWITH_READLINE=0
   -DWITH_PIC=1
+  -DWITH_PCRE=system
   -DENABLE_DTRACE=0
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
-  -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-  -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-  -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib
+  -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
+  -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
+  -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
   -DWITHOUT_TOKUDB=true
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${zlib:location}/include
-  CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${zlib:location}/lib
+  CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${zlib:location}/include
+  CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
+  LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${zlib:location}/lib
 
 [mroonga-mariadb]
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-4.06.tar.gz
-md5sum = 042bc1fbc3f25593be4484529d598144
+url = http://packages.groonga.org/source/mroonga/mroonga-4.07.tar.gz
+md5sum = fcfca7934b1f8df981fca579d3cd41bd
+configure-command = mkdir fake_mariadb_source && ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql && ./configure
 configure-options =
-  --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-mysql-source=fake_mariadb_source
   --with-mysql-config=${mariadb:location}/bin/mysql_config
   --disable-static
   --disable-document
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/mroonga_boolean.patch#36645770ae612515b74b90884ecc59fc
 environment =
   PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  CPPFLAGS=-I${groonga:location}/include/groonga
+  CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig

component/mariadb/mroonga_boolean.patch

+diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
+index dd9967d..c105a62 100644
+--- a/ha_mroonga.cpp
++++ b/ha_mroonga.cpp
+@@ -7678,7 +7678,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
+ 
+   const char *keyword, *keyword_original;
+   uint keyword_length, keyword_length_original;
+-  grn_operator default_operator = GRN_OP_OR;
++  grn_operator default_operator = GRN_OP_ADJUST;
+   grn_bool weight_specified = false;
+   keyword = keyword_original = key->ptr();
+   keyword_length = keyword_length_original = key->length();

component/mesa/buildout.cfg

 [buildout]
 extends =
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../bison/buildout.cfg
+  ../flex/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../libtool/buildout.cfg
+  ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
 
 parts =
   mesa
 
-[libdrm]
-#does not seem to compile with -jX
-recipe = slapos.recipe.cmmi
-url = http://dri.freedesktop.org/libdrm/libdrm-2.4.26.tar.bz2
-md5sum = 062569426773f69b11a47a7712bba770
-configure-options = 
-  --enable-nouveau-experimental-api
-environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig
-
-
 [mesa]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.freedesktop.org//pub/mesa/7.11/MesaLib-7.11.tar.bz2
-md5sum = ff03aca82d0560009a076a87c888cf13
+url = ftp://ftp.freedesktop.org/pub/mesa/10.3.2/MesaLib-10.3.2.tar.bz2
+md5sum = c0a2a975899c4e7485124b388c16cd2f
 configure-options =
   --disable-static
-  --disable-driglx-direct
-  --disable-glu
-  --disable-glw
-  --disable-glut
+  --disable-dri
+  --disable-dri3
   --disable-egl
+  --disable-gbm
+  --disable-xvmc
+  --disable-vdpau
+  --enable-xlib-glx
+  --disable-gallium-gbm
+  --disable-shared-glapi
+  --disable-driglx-direct
   --disable-gallium-llvm
   --with-gallium-drivers=
   --with-dri-drivers=
-  --with-expat=${libexpat:location}
-  --with-driver=xlib
 environment =
-  PATH=${pkgconfig:location}/bin:${makedepend:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${libdrm:location}/lib/pkgconfig:${dri2proto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
-  CPPFLAGS=-I${libexpat:location}/include -I${libdrm:location}/include/libdrm -I${libdrm:location}/include -I${xproto:location}/include -I${libX11:location}/include -I${libXext:location}/include/ -I${xextproto:location}/include
-  LDFLAGS=-L${libexpat:location}/lib
-  PYTHON=${buildout:executable}
+  PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  PYTHON2=${buildout:executable}
+  ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
+  AUTOCONF=${autoconf:location}/bin/autoconf
+  AUTOMAKE=${automake:location}/bin/automake
+
+[glu]
+recipe = slapos.recipe.cmmi
+# tar.bz2 cannot be well extracted why ?
+url = ftp://ftp.freedesktop.org/pub/mesa/glu/glu-9.0.0.tar.gz
+md5sum = bbc57d4fe3bd3fb095bdbef6fcb977c4
+configure-options =
+  --disable-static
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig

component/nginx/buildout.cfg

@@ -11,8 +11,8 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.7.5.tar.gz
-md5sum = e65aad627acc1cbe26527339a5814d57
+url = http://nginx.org/download/nginx-1.7.6.tar.gz
+md5sum = dd444e5333e0d324bec480e2ff67870a
 
 [nginx]
 <= nginx-common
@@ -38,6 +38,7 @@ configure-options =
   --with-http_ssl_module
   --with-mail
   --with-mail_ssl_module
+  --error-log-path=var/log/nginx.error.log
   --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
   --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
   --with-http_dav_module 

component/numpy/buildout.cfg

@@ -15,14 +15,8 @@ ATLAS = ${libatlas:location}/lib/libatlas.a
 [numpy]
 recipe = zc.recipe.egg:custom
 egg = numpy
-# patched version that supports rpath parameter in zc.recipe.egg:custom.
-find-links = http://www.nexedi.org/static/packages/source/numpy-1.8.2nxd001.tar.gz
 environment = numpy-env
 # no need to add libatlas location because they are statically linked.
 rpath =
   ${gcc-fortran:location}/lib
   ${gcc-fortran:location}/lib64
-
-[versions]
-# patched version that supports rpath parameter in zc.recipe.egg:custom.
-numpy = 1.8.2nxd001

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.1i.tar.gz
-md5sum = c8dc151a671b9b92ff3e4c118b174972
+url = https://www.openssl.org/source/openssl-1.0.1j.tar.gz
+md5sum = f7175c9cd3c39bb1907ac8bba9df8ed3
 patch-binary = ${patch:location}/bin/patch
 patches =
   ${:_profile_base_location_}/openssl-nodoc.patch#b4887a7b4e18402447bc6227d2493b92

component/pcre/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [pcre]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.35.tar.bz2
-md5sum = 6aacb23986adccd9b3bc626c00979958
+url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2
+md5sum = b767bc9af0c20bc9c1fe403b0d41ad97
 configure-options =
   --disable-static
   --enable-unicode-properties

component/rdiff-backup/buildout.cfg

@@ -14,6 +14,8 @@ library-dirs =
   ${librsync:location}/lib/
 rpath =
   ${librsync:location}/lib/
+# If you need another release, override this link
+# and pin the egg in the [versions] section of the stack or SR.
 find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
 
 [rdiff-backup]

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2014/sqlite-autoconf-3080600.tar.gz
-md5sum = f7e4a156b583abeba349629e2364224b
+url = http://www.sqlite.org/2014/sqlite-autoconf-3080701.tar.gz
+md5sum = 8ee4541ebb3e5739e7ef5e9046e30063
 configure-options =
   --disable-static
   --enable-readline

component/wget/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.15.tar.xz
-md5sum = 7a279d5ac5594919124d5526e7143e28
+url = http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz
+md5sum = fe102975ab3a6c049777883f1bb9ad07
 configure-options =
   --enable-ipv6
   --enable-opie
@@ -24,7 +24,7 @@ patch-options =
   -p1
 
 patches =
-  ${:_profile_base_location_}/wget-doc.makefile.patch#2b89644c63a0dfe96290471d28a554f2
+  ${:_profile_base_location_}/wget-doc.makefile.patch#aabcc1695c7fb65ac44f295e04a2db78
 
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/wget/wget-doc.makefile.patch

@@ -4,8 +4,8 @@
  ACLOCAL_AMFLAGS = -I m4
  
  # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util
-+SUBDIRS = lib src po tests util
+-SUBDIRS = lib src doc po tests util testenv
++SUBDIRS = lib src po tests util testenv
  
  EXTRA_DIST = ChangeLog.README MAILING-LIST \
               msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
@@ -15,8 +15,8 @@
  ACLOCAL_AMFLAGS = -I m4
  
  # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util
-+SUBDIRS = lib src po tests util
+-SUBDIRS = lib src doc po tests util testenv
++SUBDIRS = lib src po tests util testenv
  EXTRA_DIST = ChangeLog.README MAILING-LIST \
               msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
               msdos/Makefile.WC ABOUT-NLS \

component/xorg/buildout.cfg

@@ -465,6 +465,21 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXt:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libICE:location}/lib/pkgconfig:${libSM:location}/lib/pkgconfig
 
+[compositeproto]
+recipe = slapos.recipe.cmmi
+url = http://www.x.org/releases/X11R7.7/src/everything/compositeproto-0.4.2.tar.bz2
+md5sum = 98482f65ba1e74a08bf5b056a4031ef0
+
+[libXcomposite]
+recipe = slapos.recipe.cmmi
+url = http://www.x.org/releases/X11R7.7/src/everything/libXcomposite-0.4.3.tar.bz2
+md5sum = a60e0b5c276d0aa9e2d3b982c98f61c8
+configure-options =
+  --disable-static
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${compositeproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
+
 [libXcursor]
 recipe = slapos.recipe.cmmi
 url = http://www.x.org/releases/X11R7.7/src/everything/libXcursor-1.1.13.tar.bz2

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.87'
+version = '0.91'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/addresiliency/__init__.py

@@ -40,44 +40,8 @@ class Recipe(GenericSlapRecipe):
     def _install(self):
         path_list = []
 
-        confpath = os.path.join(self.options['etc'], 'bully.conf')
-
-        ip_list = self.parameter_dict['ip-list']
-        print 'Creating bully configuration with ips : %s\n' % ip_list
-
-        conf = self.createFile(confpath,
-                               self.substituteTemplate(
-                               self.getTemplateFilename('bully.conf.in'),
-                               {
-                                   'self_id': int(self.parameter_dict['number']),
-                                   'ip_list': ip_list
-                                   }
-                               ))
-        path_list.append(conf)
-
         slap_connection = self.buildout['slap-connection']
 
-        if self.optionIsTrue('enable-bully-service', default=False):
-            bully_dir = self.options['services']
-        else:
-            bully_dir = self.options['bin']
-
-        bully_wrapper = self.createPythonScript(
-            name=os.path.join(bully_dir, self.options['wrapper-bully']),
-            absolute_function='slapos.recipe.addresiliency.bully.run',
-            arguments={
-                'confpath': confpath,
-                'server_url': slap_connection['server-url'],
-                'key_file': slap_connection.get('key-file'),
-                'cert_file': slap_connection.get('cert-file'),
-                'computer_id': slap_connection['computer-id'],
-                'partition_id': slap_connection['partition-id'],
-                'software': slap_connection['software-release-url'],
-                'namebase': self.parameter_dict['namebase'],
-            })
-
-        path_list.append(bully_wrapper)
-
         takeover_wrapper = self.createPythonScript(
             name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
             absolute_function='slapos.recipe.addresiliency.takeover.run',

slapos/recipe/addresiliency/bully.py

-# -*- coding: utf-8 -*-
-
-import logging
-import Queue
-import socket
-import thread
-import time
-
-import slapos.recipe.addresiliency.renamer
-import slapos
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-BASE_PORT = 50000
-SLEEPING_MINS = 2           # XXX was 30, increase after testing
-
-MSG_PING = 'ping'
-MSG_HALT = 'halt'
-MSG_VICTORY = 'victory'
-
-MSG_OK = 'ok'
-
-STATE_NORMAL = 'normal'
-STATE_WAITINGCONFIRM = 'waitingConfirm'
-STATE_ELECTION = 'election'
-STATE_REORGANIZATION = 'reorganization'
-
-
-
-## Leader is always number 0
-
-class ResilientInstance(object):
-
-    def __init__(self, comm, renamer, confpath):
-        self.comm = comm
-        self.participant_id = 0
-        self.state = STATE_NORMAL
-        self.halter_id = 0
-        self.inElection = False
-        self.alive = True
-
-        self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
-
-        self.renamer = renamer
-        self.okCanal = self.comm.create_canal([MSG_OK])
-        self.confpath = confpath
-        self.loadConnectionInfo()
-
-    def loadConnectionInfo(self):
-        params = open(self.confpath, 'r').readlines()
-        self.total_participants = len(params[0].split())
-        new_id = int(params[1])
-        if self.participant_id != new_id:
-            self.halter_id = new_id
-            self.participant_id = new_id 
-        log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
-
-    ## Needs to be changed to use the master
-    def aliveManagement(self):
-        while self.alive:
-            log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
-            time.sleep(SLEEPING_MINS*60)
-            if self.participant_id == 0:
-                continue
-            self.comm.send(MSG_PING, 0)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            self.election()
-
-    def listen(self):
-        while self.alive:
-            self.comm.recv()
-
-    def main(self):
-        while self.alive:
-            message, sender = self.mainCanal.get()
-            if message == MSG_PING:
-                self.comm.send(MSG_OK, sender)
-
-            elif message == MSG_HALT:
-                self.state = STATE_WAITINGCONFIRM
-                self.halter_id = int(sender)
-                self.comm.send(MSG_OK, sender)
-
-            elif message == MSG_VICTORY:
-                if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
-                    log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
-                    self.comm.send(MSG_OK, sender)
-                self.state = STATE_NORMAL
-
-    def election(self):
-        self.inElection = True
-        self.loadConnectionInfo()
-        # Check if I'm the highest instance alive
-        for higher in range(self.participant_id + 1, self.total_participants):
-            self.comm.send(MSG_PING, higher)
-            message, sender = self.okCanal.get()
-            if message:
-                log.info('{} is alive ({})'.format(higher, self.participant_id))
-                self.inElection = False
-                return False
-            continue
-
-        if not self.alive:
-            return False
-
-        # I should be the new coordinator, halt those below me
-        log.info('Should be ME : {}'.format(self.participant_id))
-        self.state = STATE_ELECTION
-        self.halter_id = self.participant_id
-        ups = []
-        for lower in range(self.participant_id):
-            self.comm.send(MSG_HALT, lower)
-            message, sender = self.okCanal.get()
-            if message:
-                ups.append(lower)
-
-        #Broadcast Victory
-        self.state = STATE_REORGANIZATION
-        for up in ups:
-            self.comm.send(MSG_VICTORY, up)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            log.info('Something is wrong... let\'s start over')
-            return self.election()
-        self.state = STATE_NORMAL
-        self.active = True
-        log.info('{} Is THE LEADER'.format(self.participant_id))
-
-        self.renamer.failover()
-
-        self.inElection = False
-
-        return True
-
-
-
-class FilteredCanal(object):
-
-    def __init__(self, accept, timeout):
-        self.accept = accept
-        self.queue = Queue.Queue()
-        self.timeout = timeout
-
-    def append(self, message, sender):
-        if message in self.accept:
-            self.queue.put([message, sender])
-
-    def get(self):
-        try:
-            return self.queue.get(timeout=self.timeout)
-        except Queue.Empty:
-            return [None, None]
-
-
-
-class Wrapper(object):
-
-    def __init__(self, confpath, timeout=20):
-        self.canals = []
-        self.ips = []
-        self.participant_id = 0
-        self.timeout = timeout
-        self.confpath = confpath
-        self.getConnectionInfo()
-        self.socket = None
-
-    def getConnectionInfo(self):
-        params = open(self.confpath, 'r').readlines()
-        self.ips = params[0].split()
-        self.participant_id = int(params[1])
-        log.debug('I am {} of {}'.format(self.participant_id, self.ips))
-
-    def start(self):
-        self.getConnectionInfo()
-        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
-        self.socket.listen(5)
-
-    def send(self, message, number):
-        self.getConnectionInfo()
-        try:
-            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], BASE_PORT + number))
-            s.send(message + (' {}\n'.format(self.participant_id)))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-            pass
-        finally:
-            s.close()
-
-    def create_canal(self, accept):
-        created = FilteredCanal(accept, self.timeout)
-        self.canals.append(created)
-        return created
-
-    def recv(self):
-        client, _ = self.socket.accept()
-        client_message = client.recv(1024)
-        if client_message:
-            message, sender = client_message.split()
-            for canal in self.canals:
-                canal.append(message, int(sender))
-
-
-
-
-def run(args):
-    confpath = args.pop('confpath')
-
-    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
-                                                          key_file = args.pop('key_file'),
-                                                          cert_file = args.pop('cert_file'),
-                                                          computer_guid = args.pop('computer_id'),
-                                                          partition_id = args.pop('partition_id'),
-                                                          software_release = args.pop('software'),
-                                                          namebase = args.pop('namebase'))
-
-    if args:
-        raise ValueError('Unknown arguments: %s' % ', '.join(args))
-
-    wrapper = Wrapper(confpath=confpath, timeout=20)
-
-    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
-
-    # idle waiting for connection infos
-    while computer.total_participants < 2:
-        computer.loadConnectionInfo()
-        time.sleep(30)
-
-    log.info('Starting')
-
-    computer.comm.start()
-    thread.start_new_thread(computer.listen, ())
-    thread.start_new_thread(computer.aliveManagement, ())
-
-    computer.main()
-

slapos/recipe/addresiliency/template/bully.conf.in

-%(ip_list)s
-%(self_id)s

slapos/recipe/addresiliency/template/bully.py.in

-#!%(executable)s
-
-import select
-import socket
-import threading
-import time
-import sys
-
-sys.path[:] = %(syspath)s
-
-import slapos
-from slapos import slap as slapmodule
-
-port = 50000
-size = 1024
-wait = True
-
-
-def loadConnectionInfos():
-  connectionInfos = {}
-  file = open('%(confpath)s', 'r')
-  params = file.read().split('\n')
-  file.close()
-  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
-  connectionInfos['self_id'] = int(params[1])
-
-  connectionInfos['server_list'] = \
-      [(i, ip_list[i]) for i in range(len(ip_list))]
-
-  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
-
-  return connectionInfos
-
-
-def rename_broken_and_stop():
-  try:
-    slap = slapmodule.slap()
-    slap.initializeConnection('%(server_url)s',
-                              '%(key_file)s',
-                              '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
-                                                        partition_id='%(partition_id)s')
-    broken = computer_partition.request(software_release='%(software)s',
-                                        software_type='frozen',
-                                        partition_reference='%(namebase)s0')
-
-    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
-    broken.stopped()
-    computer_partition.rename('%(namebase)s0')
-    print 'renaming done\n'
-  except slapos.slap.slap.ServerError:
-    print 'Internal server error\n'
-
-def election():
-  global wait
-  connection = loadConnectionInfos()
-  message = "%%s, %%s" %% (connection['self_id'], "Election")
-  victory = True
-
-  for (remote_id, addr) in connection['server_list']:
-    if remote_id > connection['self_id']:
-      try:
-        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        s.connect((addr, port + remote_id))
-        s.send(message)
-        reply = s.recv(size)
-        if reply == "%%s, %%s" %% (remote_id, "Alive"):
-          victory = False
-      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-        pass
-      finally:
-        s.close()
-
-  if victory:
-    wait = True
-    for (remote_id, addr) in connection['server_list']:
-      if remote_id < connection['self_id']:
-        try:
-          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-          s.connect((addr, port + remote_id))
-          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-          pass
-        finally:
-          s.close()
-    rename_broken_and_stop()
-
-
-def failure_detect():
-  global wait
-  connection = loadConnectionInfos()
-  while True:
-    time.sleep(30)
-    if wait:
-      print 'waiting 30 minutes\n'
-      time.sleep(30 * 60)
-      wait = False
-    if not connection['server_list'][0]:
-      continue
-    (remote_id, addr) = connection['server_list'][0]
-    try:
-      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-      s.connect((addr, port + remote_id))
-      s.close()
-    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-      s.close()
-      election()
-
-failure_detect_thread = threading.Thread(target=failure_detect)
-failure_detect_thread.start()
-
-connection = loadConnectionInfos()
-s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-s.bind((connection['self_ip'], port + connection['self_id']))
-s.listen(5)
-#election()
-while True:
-  force_election = False
-  client, _ = s.accept()
-  client_message = client.recv(1024)
-  if client_message:
-    client_id, message = client_message.split(', ')
-    client_id = eval(client_id)
-    if message == "Victory":
-      wait = True
-      print "%%s wins" %% client_id
-    elif message == "Election":
-      print "%%s starts an election" %% client_id
-      if client_id < connection['self_id']:
-        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
-        force_election = True
-  client.close()
-  if force_election:
-    election()

slapos/recipe/addresiliency/template/bully_new.py.in

-#!%(executable)s
-
-import logging
-import os
-import socket
-import sys
-import thread
-import time
-
-sys.path[:] = %(syspath)s
-
-from slapos import slap as slapmodule
-import slapos
-
-BASE_PORT = 50000
-SLEEPING_MINS = 2
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-
-class Renamer(object):
-    def __init__(self, server_url, key_file, cert_file, computer_guid,
-                 partition_id, software_release, namebase):
-        self.server_url = server_url
-        self.key_file = key_file
-        self.cert_file = cert_file
-        self.computer_guid = computer_guid
-        self.partition_id = partition_id
-        self.software_release = software_release
-        self.namebase = namebase
-
-    def _failover(self):
-        slap = slapmodule.slap()
-        slap.initializeConnection(self.server_url,
-                                  self.key_file,
-                                  self.cert_file)
-        computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
-                                                            partition_id=self.partition_id)
-        broken = computer_partition.request(software_release=self.software_release,
-                                            software_type='frozen',
-                                            partition_reference=self.namebase+'0')
-
-        broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
-        broken.stopped()
-        computer_partition.rename(self.namebase+'0')
-
-    def failover(self):
-        try:
-            log.info('renaming done')
-        except slapos.slap.slap.ServerError:
-            log.info('Internal server error')
-
-
-
-## Leader is always number 0
-
-class ResilientInstance(object):
-
-    def __init__(self, comm, renamer, confpath):
-        self.comm = comm
-        self.id = 0
-        self.state = 'normal'
-        self.halter = 0
-        self.inElection = False
-        self.alive = True
-        self.lastPing = time.clock()
-
-        self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
-
-        self.renamer = renamer
-        self.okCanal = self.comm.canal(['ok'])
-        self.confpath = confpath
-        self.loadConnectionInfos()
-
-    def loadConnectionInfos(self):
-        file = open(self.confpath, 'r')
-        params = file.read().split('\n')
-        file.close()
-        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
-        new_id = int(params[1])
-        if self.id != new_id:
-            self.halter = new_id
-            self.id = new_id 
-
-    ## Needs to be changed to use the master
-    def aliveManagement(self):
-        while self.alive:
-            log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
-            time.sleep(SLEEPING_MINS*60)
-            if self.id == 0:
-                continue
-            self.comm.send('ping', 0)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            self.election()
-
-    def listen(self):
-        while self.alive:
-            self.comm.recv()
-
-    def main(self):
-        while self.alive:
-            message, sender = self.mainCanal.get()
-            if message == 'ping':
-                self.comm.send('ok', sender)
-
-            elif message == 'halt':
-                self.state = 'waitingConfirm'
-                self.halter = sender
-                self.comm.send('ok', sender)
-
-            elif message == 'victory':
-                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
-                    log.info('{} thinks {} is the leader'.format(self.id, sender))
-                    self.comm.send('ok', sender)
-                self.state = 'normal'
-
-    def election(self):
-        self.inElection = True
-        self.loadConnectionInfos()
-        #Check if I'm the highest instance alive
-        for higher in range(self.id + 1, self.nbComp):
-            self.comm.send('ping', higher)
-            message, sender = self.okCanal.get()
-            if message:
-                log.info('{} is alive ({})'.format(higher, self.id))
-                self.inElection = False
-                return False
-            continue
-
-        if not self.alive:
-            return False
-
-        #I should be the new coordinator, halt those below me
-        log.info('Should be ME : {}'.format(self.id))
-        self.state = 'election'
-        self.halter = self.id
-        ups = []
-        for lower in range(self.id):
-            self.comm.send('halt', lower)
-            message, sender = self.okCanal.get()
-            if message:
-                ups.append(lower)
-
-        #Broadcast Victory
-        self.state = 'reorganization'
-        for up in ups:
-            self.comm.send('victory', up)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            log.info('Something is wrong... let\'s start over')
-            return self.election()
-        self.state = 'normal'
-        self.active = True
-        log.info('{} Is THE LEADER'.format(self.id))
-
-        self.renamer.failover()
-
-        self.inElection = False
-
-        return True
-
-
-class FilteredCanal(object):
-
-    def __init__(self, accept, timeout):
-        self.accept = accept
-        self.list = []
-        self.lock = thread.allocate_lock()
-        self.timeout = timeout
-
-    def append(self, message, sender):
-        if message in self.accept:
-            self.lock.acquire()
-            self.list.append([message, sender])
-            self.lock.release()
-
-    def get(self):
-        start = time.clock()
-        while (time.clock() - start < self.timeout):
-            self.lock.acquire()
-            if self.list:
-                self.lock.release()
-                return self.list.pop(0)
-            self.lock.release()
-        return [None, None]
-
-
-class Wrapper(object):
-
-    def __init__(self, confpath, timeout=20):
-        self.canals = []
-        self.ips = []
-        self.id = 0
-        self.timeout = timeout
-        self.confpath = confpath
-        self.getConnectionInfos()
-        self.socket = None
-
-    def getConnectionInfos(self):
-        file = open(self.confpath, 'r')
-        params = file.read().split('\n')
-        file.close()
-        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
-        self.id = int(params[1])
-
-    def start(self):
-        self.getConnectionInfos()
-        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
-        self.socket.listen(5)
-
-    def send(self, message, number):
-        self.getConnectionInfos()
-        try:
-            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], BASE_PORT + number))
-            s.send(message + (' {}\n'.format(self.id)))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-            pass
-        finally:
-            s.close()
-
-    def canal(self, accept):
-        created = FilteredCanal(accept, self.timeout)
-        self.canals.append(created)
-        return created
-
-    def recv(self):
-        client, _ = self.socket.accept()
-        client_message = client.recv(1024)
-        if client_message:
-            message, sender = client_message.split()
-            for canal in self.canals:
-                canal.append(message, int(sender))
-
-
-
-def main():
-
-    renamer = Renamer(server_url = '%(server_url)s',
-                      key_file = '%(key_file)s',
-                      cert_file = '%(cert_file)s',
-                      computer_guid = '%(computer_id)s',
-                      partition_id = '%(partition_id)s',
-                      software_release = '%(software)s',
-                      namebase = '%(namebase)s')
-
-    confpath = '%(confpath)s'
-
-    wrapper = Wrapper(confpath=confpath, timeout=20)
-
-    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
-
-    #idle waiting for connection infos
-    while computer.nbComp < 2 :
-        computer.loadConnectionInfos()
-        time.sleep(30)
-
-    log.info('Starting')
-
-    computer.comm.start()
-    thread.start_new_thread(computer.listen, ())
-    thread.start_new_thread(computer.main, ())
-    thread.start_new_thread(computer.aliveManagement, ())
-
-    while True:
-        # XXX tight loop
-        continue
-
-
-if __name__ == '__main__':
-    main()
-

slapos/recipe/dropbear.py

@@ -112,6 +112,8 @@ class Recipe(GenericBaseRecipe):
     dropbear_cmd.extend(['-p', binding_address])
     # Single user mode
     dropbear_cmd.append('-n')
+    # Keep connection alive for 5 minutes
+    dropbear_cmd.extend(['-K', '300'])
 
     if 'dss-keyfile' in self.options:
       dropbear_cmd.extend(['-d', self.options['dss-keyfile']])

slapos/recipe/librecipe/generic.py

@@ -155,7 +155,7 @@ class GenericBaseRecipe(object):
           pidfile=%s
           if [ -e $pidfile ]; then
             pid=$(cat $pidfile)
-            if [ ! -z $(ps -p "$pid" | grep $(basename %s)) ]; then
+            if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
               echo "Already running with pid $pid."
               exit 1
             else

slapos/recipe/notifier.py

@@ -86,6 +86,7 @@ class Notify(GenericBaseRecipe):
                               command=notifier_binary,
                               parameters=parameters,
                               pidfile=pidfile,
+                              parameters_extra=True,
                               comments=[
                                   '',
                                   'Call an executable and send notification(s).',

slapos/recipe/pbs.py

@@ -36,6 +36,7 @@ from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
 from slapos.recipe.notifier import Notify
 from slapos.recipe.notifier import Callback
+from slapos.recipe.librecipe import shlex
 
 
 def promise(args):
@@ -62,6 +63,161 @@ def promise(args):
 
 class Recipe(GenericSlapRecipe, Notify, Callback):
 
+  def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
+    # Create a simple rdiff-backup wrapper that will push
+
+    template = textwrap.dedent("""\
+        #!/bin/sh
+        #
+        # Push data to a PBS *-import instance.
+        #
+
+        LC_ALL=C
+        export LC_ALL
+        RDIFF_BACKUP=%(rdiffbackup_binary)s
+        until $RDIFF_BACKUP \\
+                --remote-schema %(remote_schema)s \\
+                --restore-as-of now \\
+                --force \\
+                %(local_dir)s \\
+                %(remote_dir)s; do
+          echo "repeating rdiff-backup..."
+          sleep 10
+        done
+        """)
+
+    template_dict = {
+      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
+      'remote_schema': shlex.quote(remote_schema),
+      'remote_dir': shlex.quote(remote_dir),
+      'local_dir': shlex.quote(local_dir)
+    }
+
+    return self.createFile(
+      name=rdiff_wrapper_path,
+      content=template % template_dict,
+      mode=0o700
+    )
+
+
+  def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
+    # Wrap rdiff-backup call into a script that checks consistency of backup
+    # We need to manually escape the remote schema
+
+    template = textwrap.dedent("""\
+        #!/bin/sh
+        #
+        # Pull data from a PBS *-export instance.
+        #
+
+        sigint()
+        {
+          exit 1
+        }
+
+        trap sigint SIGINT  # we can CTRL-C for ease of debugging
+
+        LC_ALL=C
+        export LC_ALL
+        is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
+        RDIFF_BACKUP=%(rdiffbackup_binary)s
+
+        TMPDIR=%(tmpdir)s
+        BACKUP_DIR=%(local_dir)s
+        CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
+        CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
+        CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
+        CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind
+
+        SUCCEEDED=false
+        while ! $SUCCEEDED; do
+
+            # not using --fix-corrupted can lead to an infinite loop
+            # in case of manual changes to the backup repository.
+
+            CORRUPTED_ARGS=""
+            if [ "$1" = "--fix-corrupted" ]; then
+                VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
+                echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE
+
+                # Sometimes --verify reports this spurious warning:
+                echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE
+
+                # There can be too many files, better not to provide them through separate command line parameters
+                CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"
+
+                if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
+                    echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
+                else
+                    echo "No corrupted or missing files to retransmit"
+                fi
+            fi
+
+            $RDIFF_BACKUP \\
+                    $CORRUPTED_ARGS \\
+                    --remote-schema %(remote_schema)s \\
+                    %(remote_dir)s \\
+                    $BACKUP_DIR
+
+            [ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"
+
+            if [ ! $? -eq 0 ]; then
+                # Check the backup, go to the last consistent backup, so that next
+                # run will be okay.
+                echo "Checking backup directory..."
+                $RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
+                if [ ! $? -eq 0 ]; then
+                    # Here, two possiblities:
+                    if [ is_first_backup ]; then
+                        continue
+                        # The first backup failed, and check-destination as well.
+                        # we may want to remove the backup.
+                    else
+                        continue
+                        # The backup command has failed, while transferring an increment, and check-destination as well.
+                        # XXX We may need to publish the failure and ask the the equeue, re-run this script again,
+                        # instead do a push to the clone.
+                    fi
+                fi
+            else
+                # Everything's okay, cleaning up...
+                $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
+            fi
+
+            SUCCEEDED=true
+
+            if [ -e %(backup_signature)s ]; then
+              cd $BACKUP_DIR
+              find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > ../proof.signature
+              cmp backup.signature ../proof.signature || SUCCEEDED=false
+              diff -ruw backup.signature ../proof.signature > ../backup.diff
+              # XXX If there is a difference on the backup, we should publish the
+              # failure and ask the equeue, re-run this script again,
+              # instead do a push it to the clone.
+            fi
+
+            $SUCCEEDED || find $BACKUP_DIR -name rdiff-backup.tmp.* -exec rm -rf {} \;
+        done
+        """)
+
+    template_dict = {
+      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
+      'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
+      'backup_signature': shlex.quote(os.path.join(local_dir, 'backup.signature')),
+      'remote_schema': shlex.quote(remote_schema),
+      'remote_dir': shlex.quote(remote_dir),
+      'local_dir': shlex.quote(local_dir),
+      'tmpdir': '/tmp',
+      'remove_backup_older_than': shlex.quote(remove_backup_older_than)
+    }
+
+    return self.createFile(
+      name=rdiff_wrapper_path,
+      content=template % template_dict,
+      mode=0o700
+    )
+
+
   def add_slave(self, entry, known_hosts_file):
     path_list = []
 
@@ -88,86 +244,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
                                       promise_dict)
     path_list.append(promise)
 
-    host = parsed_url.hostname
-    known_hosts_file[host] = entry['server-key']
+    known_hosts_file[parsed_url.hostname] = entry['server-key']
 
     notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)
     rdiff_wrapper_path = notifier_wrapper_path + '_raw'
 
     # Create the rdiff-backup wrapper
-    # It is useful to separate it from the notifier so that we can run it
-    # Manually.
-    rdiffbackup_parameter_list = []
+    # It is useful to separate it from the notifier so that we can run it manually.
 
     # XXX use -y because the host might not yet be in the
     #     trusted hosts file until the next time slapgrid is run.
-    rdiffbackup_remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % {
-            'ssh': self.options['sshclient-binary'],
-            'user': parsed_url.username,
-            'host': parsed_url.hostname,
-    }
-    remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
-                                               'path': parsed_url.path}
-    local_directory = self.createDirectory(self.options['directory'], entry['name'])
+    remote_schema = '{ssh} -y -K 300 -p %s {username}@{hostname}'.format(
+              ssh=self.options['sshclient-binary'],
+              username=parsed_url.username,
+              hostname=parsed_url.hostname
+            )
+    remote_dir = '{port}::{path}'.format(port=parsed_url.port, path=parsed_url.path)
+    local_dir = self.createDirectory(self.options['directory'], entry['name'])
 
     if slave_type == 'push':
-      # Create a simple rdiff-backup wrapper that will push
-      rdiffbackup_parameter_list.extend(['--remote-schema', rdiffbackup_remote_schema])
-      rdiffbackup_parameter_list.extend(['--restore-as-of', 'now'])
-      rdiffbackup_parameter_list.append('--force')
-      rdiffbackup_parameter_list.append(local_directory)
-      rdiffbackup_parameter_list.append(remote_directory)
-      comments = ['', 'Push data to a PBS *-import instance.', '']
-      rdiff_wrapper = self.createWrapper(
-          name=rdiff_wrapper_path,
-          command=self.options['rdiffbackup-binary'],
-          parameters=rdiffbackup_parameter_list,
-          comments=comments,
-          pidfile=os.path.join(self.options['run-directory'], '%s_raw.pid' % slave_id),
-      )
+      rdiff_wrapper = self.wrapper_push(remote_schema,
+                                        local_dir,
+                                        remote_dir,
+                                        rdiff_wrapper_path)
     elif slave_type == 'pull':
-      # Wrap rdiff-backup call into a script that checks consistency of backup
-      # We need to manually escape the remote schema
-      rdiffbackup_parameter_list.extend(['--remote-schema', '"%s"' % rdiffbackup_remote_schema])
-      rdiffbackup_parameter_list.append(remote_directory)
-      rdiffbackup_parameter_list.append(local_directory)
-      comments = ['', 'Pull data from a PBS *-export instance.', '']
-      rdiff_wrapper_template = textwrap.dedent("""\
-          #!/bin/sh
-          # %(comment)s
-          RDIFF_BACKUP="%(rdiffbackup_binary)s"
-          $RDIFF_BACKUP %(rdiffbackup_parameter)s
-          if [ ! $? -eq 0 ]; then
-              # Check the backup, go to the last consistent backup, so that next
-              # run will be okay.
-              echo "Checking backup directory..."
-              $RDIFF_BACKUP --check-destination-dir %(local_directory)s
-              if [ ! $? -eq 0 ]; then
-                  # Here, two possiblities:
-                  # * The first backup failed. It is safe to remove it since there is nothing valuable there.
-                  # * The backup has been complete, but is now in a really weird state. Not safe to remove it.
-                  echo "Impossible to check backup: we move it to a safe place."
-                  # XXX: bang
-                  mv %(local_directory)s %(local_directory)s.$(date +%%s)
-              fi
-          else
-              # Everything's okay, cleaning up...
-              $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_directory)s
-          fi
-          """)
-      rdiff_wrapper_content = rdiff_wrapper_template % {
-              'comment': comments,
-              'rdiffbackup_binary': self.options['rdiffbackup-binary'],
-              'local_directory': local_directory,
-              'rdiffbackup_parameter': ' \\\n    '.join(rdiffbackup_parameter_list),
-              # XXX: only 10 increments is not enough by default.
-              'remove_backup_older_than': entry.get('remove-backup-older-than', '3B')
-      }
-      rdiff_wrapper = self.createFile(
-          name=rdiff_wrapper_path,
-          content=rdiff_wrapper_content,
-          mode=0700
-      )
+      # XXX: only 3 increments is not enough by default.
+      rdiff_wrapper = self.wrapper_pull(remote_schema,
+                                        local_dir,
+                                        remote_dir,
+                                        rdiff_wrapper_path,
+                                        entry.get('remove-backup-older-than', '3B'))
 
     path_list.append(rdiff_wrapper)
 

slapos/recipe/request.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 import logging
+from zc.buildout import UserError
 from slapos.recipe.librecipe import wrap, JSON_SERIALISED_MAGIC_KEY
 import json
 from slapos import slap as slapmodule
@@ -33,12 +34,6 @@ import traceback
 
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
-def getListOption(option_dict, key, default=()):
-  result = option_dict.get(key, default)
-  if isinstance(result, basestring):
-    result = result.split()
-  return result
-
 class Recipe(object):
   """
   Request a partition to a slap master.
@@ -104,19 +99,20 @@ class Recipe(object):
     self.logger = logging.getLogger(name)
     software_url = options['software-url']
     name = options['name']
-    return_parameters = getListOption(options, 'return')
+    return_parameters = options.get('return', '').split()
     if not return_parameters:
       self.logger.debug("No parameter to return to main instance."
         "Be careful about that...")
     software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
-    filter_kw = dict(
-      (x, options['sla-' + x]) for x in getListOption(options, 'sla')
-      if options['sla-' + x]
-    )
-    partition_parameter_kw = self._filterForStorage(dict(
-      (x, options['config-' + x])
-      for x in getListOption(options, 'config')
-    ))
+    if 'config' in options or 'sla' in options:
+      raise UserError("'config' & 'sla' options are obsolete."
+                      " Clean up your software release.")
+    filter_kw = {k: v
+      for k, v in options.iteritems()
+      if k.startswith('sla-') and v}
+    partition_parameter_kw = self._filterForStorage({k: v
+      for k, v in options.iteritems()
+      if k.startswith('config-')})
     slave = options.get('slave', 'false').lower() in \
       librecipe.GenericBaseRecipe.TRUE_VALUES
 
@@ -267,7 +263,6 @@ class RequestEdge(Recipe):
       # Request will have its own copy of options dict
       local_options = original_options.copy()
       local_options['name'] = '%s-%s' % (country, name)
-      local_options['sla'] = "region"
       local_options['sla-region'] = country
 
       self.request_dict[country] = Recipe(buildout, name, local_options)

slapos/test/recipe/test_pbs.py

+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from slapos.recipe import pbs
+
+
+class PBSTest(unittest.TestCase):
+
+    def new_recipe(self):
+        buildout = {
+                'buildout': {
+                    'bin-directory': '',
+                    'find-links': '',
+                    'allow-hosts': '',
+                    'develop-eggs-directory': '',
+                    'eggs-directory': '',
+                    'python': 'testpython',
+                    },
+                 'testpython': {
+                     'executable': sys.executable,
+                     },
+                 'slap-connection': {
+                     'computer-id': '',
+                     'partition-id': '',
+                     'server-url': '',
+                     'software-release-url': '',
+                     }
+                }
+
+        options = {
+                'rdiffbackup-binary': ''
+                }
+
+        return pbs.Recipe(buildout=buildout, name='pbs', options=options)
+
+    def test_push(self):
+        recipe = self.new_recipe()
+
+        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+            recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
+                                local_dir='TEST_LOCAL_DIR',
+                                remote_dir='TEST_REMOTE_DIR',
+                                rdiff_wrapper_path=rdiff_wrapper.name)
+            content = rdiff_wrapper.read()
+            self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
+            self.assertIn('TEST_LOCAL_DIR', content)
+            self.assertIn('TEST_REMOTE_DIR', content)
+
+    def test_pull(self):
+        recipe = self.new_recipe()
+
+        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+            recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
+                                local_dir='TEST_LOCAL_DIR',
+                                remote_dir='TEST_REMOTE_DIR',
+                                rdiff_wrapper_path=rdiff_wrapper.name,
+                                remove_backup_older_than='TEST_OLDER')
+            content = rdiff_wrapper.read()
+            self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
+            self.assertIn('TEST_LOCAL_DIR', content)
+            self.assertIn('TEST_REMOTE_DIR', content)
+            self.assertIn('--remove-older-than TEST_OLDER', content)
+
+    def test_invalid_type(self):
+        recipe = self.new_recipe()
+
+        entry = {
+                'url': 'http://url.to.something/',
+                'type': 'invalid'
+                }
+
+        self.assertRaisesRegexp(ValueError,
+                                'type parameter must be either pull or push',
+                                recipe.add_slave, entry=entry, known_hosts_file={})
+
+    def test_install(self):
+        recipe = self.new_recipe()
+
+        promises_directory = tempfile.mkdtemp()
+        wrappers_directory = tempfile.mkdtemp()
+        directory = tempfile.mkdtemp()
+        feeds_directory = tempfile.mkdtemp()
+        run_directory = tempfile.mkdtemp()
+        cron_directory = tempfile.mkdtemp()
+
+        recipe.options.update({
+            'promises-directory': promises_directory,
+            'wrappers-directory': wrappers_directory,
+            'sshclient-binary': 'TEST_SSH_CLIENT',
+            'directory': directory,
+            'notifier-binary': 'TEST_NOTIFIER',
+            'feeds': feeds_directory,
+            'notifier-url': 'http://url.to.notifier/',
+            'run-directory': run_directory,
+            'cron-entries': cron_directory,
+            'known-hosts': 'TEST_KNOWN_HOSTS',
+            'slave-instance-list': '''[
+                {
+                 "url": "http://url.to.pull/",
+                 "type": "pull",
+                 "notification-id": "pulltest",
+                 "server-key": "TEST_SERVER_KEY",
+                 "name": "TEST_NAME",
+                 "notify": "http://url.to.notify/",
+                 "frequency": "TEST_FREQUENCY"
+                    }, {
+                 "url": "http://url.to.push/",
+                 "type": "push",
+                 "notification-id": "pushtest",
+                 "server-key": "TEST_SERVER_KEY",
+                 "name": "TEST_NAME",
+                 "notify": "http://url.to.notify/",
+                 "frequency": "TEST_FREQUENCY"
+                        }
+                ]
+                '''
+            })
+
+        recipe._install()
+
+        self.assertItemsEqual(os.listdir(promises_directory),
+                              ['pulltest', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(wrappers_directory),
+                              ['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(directory),
+                              ['TEST_NAME'])
+
+        self.assertItemsEqual(os.listdir(feeds_directory),
+                              ['pulltest', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(run_directory),
+                              [])
+
+        self.assertItemsEqual(os.listdir(cron_directory),
+                              ['pulltest', 'pushtest'])
+
+        shutil.rmtree(promises_directory)
+        shutil.rmtree(wrappers_directory)
+        shutil.rmtree(directory)
+        shutil.rmtree(feeds_directory)
+        shutil.rmtree(run_directory)
+        shutil.rmtree(cron_directory)
+
+

software/ajaxplorer/instance-apachephp.cfg

@@ -177,7 +177,6 @@ name = MariaDB
 software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
@@ -260,7 +259,6 @@ name = Frontend-Website
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}
@@ -272,7 +270,6 @@ name = Frontend-FileManager
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port2}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain2}
@@ -284,7 +281,6 @@ name = Frontend-Website2
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port3}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain3}

software/ajaxplorer/software.cfg

@@ -35,7 +35,7 @@ md5sum = 4c7936accb3658871b635158198b7905
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apachephp.cfg
 output = ${buildout:directory}/template-apachephp.cfg
-md5sum = ef5c61127a21c016014021477d1791e3
+md5sum = 8be713bd2656e184651c8ec10579d668
 mode = 0644
 
 [instance-mariadb]
@@ -86,7 +86,6 @@ numpy = 1.6.2
 # websockify 0.4.1 doesn't install well
 websockify = 0.3.0
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.11.6
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
 z3c.recipe.mkdir = 0.5

software/apache-frontend/TODO.txt

 Apache:
 =======
-
+* (Minor Bug) Check if slave in slave_list before publishing information on it

software/apache-frontend/common.cfg

@@ -74,7 +74,7 @@ mode = 0644
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = 140b3be4ce46aa18b33aea5d379d90e9
+md5sum = aafa98333cd51025938b33c2516f0b8d
 mode = 0644
 
 [template-slave-list]

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -86,7 +86,6 @@ software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{frontend_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-config = _
 connection-monitor_url =
 
 {% for section, frontend_request in request_dict.iteritems() %}
@@ -102,7 +101,6 @@ state = {{ frontend_request.get('state') }}
 {%   do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
 config-_ = {{ json_module.dumps(slave_configuration_dict) }}
 {%   if frontend_request.get('sla') %}
-sla = {{ ' '.join(frontend_request.get('sla').keys()) }}
 {%     for parameter, value in frontend_request.get('sla').iteritems() -%}
 sla-{{ parameter }} = {{ value }}
 {%     endfor -%}

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -13,7 +13,7 @@
       "title": "Custom Domain",
       "description": "Custom Domain to use for the website",
       "type": "string",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
     },
     "type": {
       "title": "Backend Type",

software/apache-frontend/software.cfg

@@ -12,9 +12,8 @@ gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

software/backupserver/software.cfg

@@ -120,6 +120,5 @@ mode = 0644
 rdiff-backup = 1.0.5
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.11.6
 slapos.recipe.template = 2.4.2
 PyRSS2Gen = 1.1

software/cloudooo/instance-cloudoo.cfg.in

@@ -19,7 +19,7 @@ url = cloudooo://${haproxy:ip}:${haproxy:port}/
 recipe = slapos.cookbook:generic.cloudooo
 ip = {{ ipv4 }}
 environment =
-  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib
+  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
   PATH = ${binary-link:target-directory}
 # Binary information

software/cloudooo/instance.cfg.in

@@ -25,31 +25,36 @@ context =
     ${:extra-context}
 
 [dynamic-template-cloudooo-parameters]
+buildout-bin-directory = {{ buildout_bin_directory }}
+coreutils = {{ coreutils_location }}
 cups = {{ cups_location }}
 dbus = {{ dbus_location }}
 dbus-glib = {{ dbus_glib_location }}
 file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
+fonts = {{ fonts_location }}
 freetype = {{ freetype_location }}
 glib = {{ glib_location }}
+glu = {{ glu_location }}
 haproxy = {{ haproxy_location }}
-libffi = {{ libffi_location }}
+imagemagick = {{ imagemagick_location }}
 libICE = {{ libICE_location }}
-libpng12 = {{ libpng12_location }}
 libSM = {{ libSM_location }}
 libX11 = {{ libX11_location }}
 libXau = {{ libXau_location }}
 libXdmcp = {{ libXdmcp_location }}
 libXext = {{ libXext_location }}
-libxcb = {{ libxcb_location }}
 libXrender = {{ libXrender_location }}
-zlib = {{ zlib_location }}
+libexpat = {{ libexpat_location }}
+libffi = {{ libffi_location }}
+libpng12 = {{ libpng12_location }}
 libreoffice-bin = {{ libreoffice_bin_location }}
-fonts = {{ fonts_location }}
-coreutils = {{ coreutils_location }}
-imagemagick = {{ imagemagick_location }}
+libxcb = {{ libxcb_location }}
+mesa = {{ mesa_location }}
 poppler = {{ poppler_location }}
-buildout-bin-directory = {{ buildout_bin_directory }}
+xdamage = {{ xdamage_location }}
+xfixes = {{ xfixes_location }}
+zlib = {{ zlib_location }}
 
 [dynamic-template-cloudooo]
 < = jinja2-template-base

software/cloudooo/software.cfg

@@ -37,12 +37,13 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 1c5dab821393845b118644686154710f
+md5sum = 2ad39c607cfa80bf87dd6258c29754c3
 extra-context =
     key buildout_bin_directory buildout:bin-directory
+    key coreutils_location coreutils:location
     key cups_location cups:location
-    key dbus_location dbus:location
     key dbus_glib_location dbus-glib:location
+    key dbus_location dbus:location
     key dcron_location dcron:location
     key file_location file:location
     key fontconfig_location fontconfig:location
@@ -50,6 +51,7 @@ extra-context =
     key freetype_location freetype:location
     key git_location git:location
     key glib_location glib:location
+    key glu_location glu:location
     key haproxy_location haproxy:location
     key imagemagick_location imagemagick:location
     key libICE_location libICE:location
@@ -59,18 +61,25 @@ extra-context =
     key libXdmcp_location libXdmcp:location
     key libXext_location libXext:location
     key libXrender_location libXrender:location
+    key libexpat_location libexpat:location
     key libffi_location libffi:location
     key libpng12_location libpng12:location
     key libreoffice_bin_location libreoffice-bin:location
     key libxcb_location libxcb:location
+    key mesa_location mesa:location
     key openssl_location openssl:location
     key poppler_location poppler:location
     key template_cloudooo template-cloudooo:target
+    key xdamage_location xdamage:location
+    key xfixes_location xfixes:location
     key zlib_location zlib:location
-    key coreutils_location coreutils:location
 
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = 740a313b0d9d98800b099af8bf7cd0a7
+md5sum = 4bede3be20dbc2ecfdb5d49b3184742e
 mode = 640
+
+[versions]
+# use newest version of slapos.cookbook
+slapos.cookbook =

software/davstorage/common.cfg

@@ -44,6 +44,6 @@ mode = 0644
 [instance-davstorage]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-davstorage.cfg
-md5sum = 699ecf4678386667f58a3391bab7af0f
+md5sum = b83b0351a0f46aac35d52e681270ff03
 output = ${buildout:directory}/template-davstorage.cfg
 mode = 0644

software/davstorage/instance-davstorage.cfg

@@ -166,7 +166,6 @@ name = Frontend Ajax
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url https-only
 config-https-only = true
 config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
 return = domain
@@ -178,7 +177,6 @@ name = Frontend Webdav
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url https-only
 config-https-only = true
 config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
 return = domain

software/erp5testnode/instance-default.cfg

@@ -27,7 +27,7 @@ recipe = slapos.cookbook:erp5testnode
 slapos-directory = $${directory:slapos}
 working-directory = $${directory:testnode}
 test-suite-directory = $${directory:test-suite}
-proxy-host = $${slap-network-information:global-ipv6}
+proxy-host = $${slap-network-information:local-ipv4}
 proxy-port = 5000
 log-directory = $${directory:log}
 srv-directory = $${rootdirectory:srv}

software/erp5testnode/software.cfg

@@ -45,7 +45,12 @@ eggs =
   zc.buildout
   slapos.libnetworkcache
   slapos.core
+  jsonschema
+  hexagonit.recipe.download
+  netaddr
   inotifyx
+  lock_file
+  pytz
   erp5.util
   PyXML
 
@@ -70,11 +75,11 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-default.cfg
 output = ${buildout:directory}/template-default.cfg
 mode = 0644
-md5sum = b4556610d17ae769d759c49b0955b63e 
+md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
 
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.41
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
+jsonschema = 2.4.0

software/etherpad-lite/common.cfg

@@ -58,7 +58,7 @@ mode = 0644
 [instance-etherpad-lite]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
-md5sum = fd7249be8988155110234c7bb877abb9
+md5sum = 28710a9a23320dd472387d81a698d12d
 output = ${buildout:directory}/template-etherpad-lite.cfg
 mode = 0644
 

software/etherpad-lite/instance-etherpad-lite.cfg

@@ -67,7 +67,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url
 config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
 return = site_url
 

software/html5as/software.cfg

@@ -72,9 +72,8 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.11.6
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 plone.recipe.command = 1.1

software/kvm/common.cfg

@@ -11,9 +11,12 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../stack/resilient/buildout.cfg
-  ../../stack/slapos.cfg
+
+# stacks are listed from most generic to most specific,
+# to avoid versioning issues
 
 parts =
   template
@@ -44,6 +47,7 @@ eggs =
   slapos.cookbook
   slapos.toolbox
   erp5.util
+  cns.recipe.symlink
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -80,7 +84,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bc5a986c7208d02d3284a897ea90b39d
+md5sum = 4c8f07da2217e54163c265fe6fe3d41d
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -88,7 +92,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e16c15f72fdeb92ce1854bc25daf5ad7
+md5sum = ac69266206830226185e576fb6e4935a
 download-only = true
 on-update = true
 
@@ -96,14 +100,14 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = a07c96b53fe9145278cd64a3b27a459a
+md5sum = 412d8540e054aa436bee723d85213a2b
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = b4894680283d3912df4e9740f3e7848b
+md5sum = 080b5790b95a979e44c4a8e26dc02835
 mode = 0644
 download-only = true
 on-update = true
@@ -111,15 +115,15 @@ on-update = true
 [template-kvm-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
-md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
+md5sum = 6835c9309ff4bf4a0efd1850e6c66b24
 output = ${buildout:directory}/template-kvm-import.cfg
 mode = 0644
 
 [template-kvm-import-script]
 recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/template/kvm-import.sh.in
-filename = kvm-import.sh.in
-md5sum = e03ed049cddd8d157228b09e1ebc071a
+url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2
+filename = kvm-import.sh.jinja2
+md5sum = 926a11421921c29f91fae8240bbcf585
 download-only = true
 mode = 0755
 
@@ -127,15 +131,15 @@ mode = 0755
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = 900f416956903fa4858e67e93b5169a1
+md5sum = c9f13c1f481ed08c75089aef1d3c6981
 download-only = true
 on-update = true
 
 [template-kvm-export-script]
 recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/template/kvm-export.sh.in
-filename = kvm-export.sh.in
-md5sum = 95fde96f35cbf90d677c44d18b60fafb
+url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
+filename = kvm-export.sh.jinja2
+md5sum = 22bd2e0c8fdb39a764a14c403a3bd752
 download-only = true
 mode = 0755
 

software/kvm/development.cfg

@@ -40,4 +40,3 @@ command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
-

software/kvm/instance-kvm-export.cfg.jinja2

@@ -17,13 +17,17 @@ parts +=
 
 # Create the exporter executable, which is a simple shell script
 [exporter]
-recipe = slapos.recipe.template
-url = {{ template_kvm_export }}
-output = ${directory:bin}/${slap-parameter:namebase}-exporter
-mode = 0755
-backup-disk-path = ${directory:backup}/virtual.qcow2
+recipe = slapos.recipe.template:jinja2
+template = {{ template_kvm_export }}
+rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
 # Resilient stack wants a "wrapper" parameter
-wrapper = ${:output}
+wrapper = ${:rendered}
+mode = 0700
+context =
+    section directory directory
+    section buildout buildout
+    key socket_path kvm-instance:socket-path
+    raw gzip_binary {{ gzip_binary }}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]

software/kvm/instance-kvm-import.cfg.in

@@ -25,13 +25,15 @@ crontabs = $${:etc}/crontabs
 cronstamps = $${:etc}/cronstamps
 
 [importer]
-recipe = slapos.recipe.template
-url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
-output = $${directory:bin}/$${slap-parameter:namebase}-importer
-mode = 0755
-backup-disk-path = $${directory:backup}/virtual.qcow2
-disk-path = $${directory:srv}/virtual.qcow2
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
+mode = 0700
 # Resilient stack wants a "wrapper" parameter
-wrapper = $${:output}
+wrapper = $${:rendered}
+context =
+    section directory directory
+    raw zcat_binary ${gzip:location}/bin/zcat
+    raw gzip_binary ${gzip:location}/bin/gzip
 
 backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient-input-schema.json

@@ -34,6 +34,12 @@
                 "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                 "type": "string",
                 "default": "3B"
+            },
+            "resilient-clone-number": {
+                "title": "Amount of backup(s) to create",
+                "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
+                "type": "integer",
+                "default": 2
             }
         }
     }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -42,16 +42,13 @@ recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
 software-type = kvm-resilient
 name = Resilient KVM (Root Instance)
-{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
-config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity {{ cluster_parameter_dict.keys() | join(' ') }}
-{% for key, value in cluster_parameter_dict.items() -%}
+{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
 config-resiliency-backup-periodicity = */5 * * * *
 # XXX What to do?
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -3,6 +3,8 @@
 {% import 'parts' as parts %}
 {% import 'replicated' as replicated with context %}
 
+{% set backup_amount = slapparameter_dict.pop('resilient-clone-number', "1")|int + 1 -%}
+
 [buildout]
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
@@ -10,12 +12,12 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("kvm", "3") }}
+  {{ parts.replicate("kvm", backup_amount) }}
   publish-connection-informations
   kvm-frontend-url-promise
   kvm-backend-url-promise
 
-{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
+{{ replicated.replicate("kvm", backup_amount, "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory

software/kvm/instance-kvm.cfg.jinja2

@@ -196,11 +196,9 @@ partition-id = ${slap-connection:partition-id}
 name = VNC Frontend
 software-type = ${slap-parameter:frontend-software-type}
 slave = true
-config = host port
 config-host = ${novnc-instance:ip}
 config-port = ${novnc-instance:port}
 return = url resource port domainname
-sla = instance_guid
 sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 
 [frontend-promise]

software/kvm/instance.cfg.in

@@ -82,6 +82,7 @@ context =
     raw kvm_template $${dynamic-template-kvm:rendered}
     raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
     raw pbsready_export_template ${pbsready-export:output}
+    raw gzip_binary ${gzip:location}/bin/gzip
 mode = 0644
 
 [dynamic-template-kvm-resilient-test]

software/kvm/software.cfg

@@ -5,35 +5,43 @@ extends = common.cfg
 PyRSS2Gen = 1.1
 apache-libcloud = 0.15.1
 async = 0.6.1
+cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 ecdsa = 0.11
 erp5.util = 0.4.41
 gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-rdiff-backup = 1.0.5
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 websockify = 0.6.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# websockify==0.6.0
+numpy = 1.8.2
+
+# Required by:
+# slapos.toolbox==0.40.4
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 feedparser = 5.1.3
 
+# Required by:
+# slapos.cookbook==0.87
+jsonschema = 2.4.0
+
 # Required by:
 # websockify==0.6.0
-numpy = 1.8.2
+numpy = 1.9.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 paramiko = 1.15.1

software/kvm/template/kvm-export.sh.in

-#!/bin/bash
-# Create a backup of the disk image of the virtual machine
-BACKUP_PATH=${:backup-disk-path}
-QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
-
-if [ ! -f $DISK_PATH ]; then
-  echo "Nothing to backup, disk image doesn't exist yet."
-  exit 0;
-fi
-
-$QMP_CLIENT --socket ${kvm-instance:socket-path} --drive-backup $BACKUP_PATH
-

software/kvm/template/kvm-export.sh.jinja2

+#!/bin/bash
+#
+# Create a backup of the disk image of the virtual machine
+#
+LC_ALL=C
+export LC_ALL
+BACKUP_DIR={{ directory['backup'] }}
+BACKUP_FILE=virtual.qcow2
+
+QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
+
+$QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE
+
+# Due to the way qmp works, the VM file cannot be compressed on the fly.
+# Although the compression step is optional, the importer uses the .gz file
+# if present. So, remove it if you are disabling the compression.
+
+# The downside of compression, here, is the temporary usage of more disk space
+# in the exporter node. The goal is to minimize disk usage on the PBS server.
+
+# If you want to compress the file in-place:
+# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
+# but the importer script would have to be adapted.
+
+echo "Compressing backup..."
+{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
+
+cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
+

software/kvm/template/kvm-import.sh.in

-#!/bin/bash
-DISK_PATH=${:disk-path}
-BACKUP_PATH=${:backup-disk-path}
-
-# TODO: Use rdiff
-rm $DISK_PATH
-cp $BACKUP_PATH $DISK_PATH

software/kvm/template/kvm-import.sh.jinja2

+#!/bin/bash
+
+VM_DIR={{ directory['srv'] }}
+BACKUP_DIR={{ directory['backup'] }}
+VM_FILE=virtual.qcow2
+LC_ALL=C
+export LC_ALL
+umask 077
+
+write_backup_proof () {
+  cd {{ directory['backup'] }}
+  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
+  diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
+}
+
+# For now we just make the diff before 
+write_backup_proof
+
+if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
+    {{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
+    {{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
+else
+    rm $VM_DIR/$VM_FILE
+    cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+fi
+

software/mariadb/software.cfg

@@ -67,5 +67,4 @@ output = ${buildout:directory}/template-recover.cfg
 mode = 0644
 
 [versions]
-slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2

software/monitor/instance.cfg

@@ -71,7 +71,6 @@ name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = $${monitor-parameters:url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain

software/monitor/software.cfg

@@ -15,7 +15,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/instance.cfg
-md5sum = 6ad84a0061df0c00a9f41d8b302adc45
+md5sum = b86685e4007296f4808c2f1f1b74a290
 mode = 0644
 
 # stupify index for now

software/neoppod/cluster.cfg.in

@@ -5,7 +5,6 @@
 [request-common]
 recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
@@ -13,22 +12,15 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 config-cluster = {{ slapparameter_dict['cluster'] }}
-config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
-config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
-
-[request-config]
-base = cluster
-master = partitions replicas
-node = masters
 
 [node-base]
 < = request-common
-config = ${request-config:base} ${request-config:node}
 config-masters = ${all-masters:addresses}
 
 [mysql-storage-base]
 < = request-common
-config = ${request-config:base} ${request-config:master}
+config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
+config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
 software-type = {{ mysql_storage_software_type }}
 
 [mysql-storage-request-base]
@@ -57,7 +49,6 @@ name = {{ section_id }}
 < = mysql-storage-base
 	node-base
 name = {{ section_id }}
-config = ${request-config:base} ${request-config:master} ${request-config:node}
 
 {% endfor -%}
 {% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}

software/neoppod/software.cfg

@@ -37,7 +37,7 @@ scripts =
 [cluster]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
-md5sum = 84cba584198a26289daacb3e6d199e2b
+md5sum = 3939e229108f96ac0d68224fc1630880
 
 [instance-neo-admin]
 < = cluster

software/nginx-push-stream/software.cfg

@@ -43,5 +43,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/powerdns/instance-powerdns-input-schema.json

@@ -45,7 +45,7 @@
       "description": "Zone to be handled by the DNS cluster",
       "type": "string",
       "default": "domain.com",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
     },
 
     "server-admin": {
@@ -59,7 +59,7 @@
       "title": "DNS domains template string",
       "description": "Template used to generate DNS domain name",
       "type": "string",
-      "default": "ns%s." + zone
+      "default": "ns%s. + zone"
     }
   }
 }

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -64,12 +64,9 @@ state = {{ slapparameter_dict.pop(state_key) }}
 {%   endif%}
 config-zone = {{ zone }}
 config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
-{%   if sla_parameters %}
-sla = {{ ' '.join(sla_parameters) }}
 {%   for parameter in sla_parameters -%}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
 {%   endfor -%}
-{%   endif -%}
 {% endfor -%}
 
 
@@ -86,7 +83,6 @@ software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{dns_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-config = {{ ' '.join(slapparameter_dict.keys()) + ' zone soa server-admin ns-record ' + slave_list_name }}
 config-server-admin = {{ server_admin }}
 config-ns-record = {{ ns_record }}
 {% for parameter, value in slapparameter_dict.iteritems() -%}

software/powerdns/software.cfg

@@ -38,7 +38,7 @@ mode = 640
 [template-dns-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
-md5sum = 7cfc55bfb9821aad44272153f60450ef
+md5sum = 83cedc8fa923c59ca900b4600cc1c52f
 mode = 0644
 
 [iso-list]
@@ -64,5 +64,4 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/seleniumrunner/software.cfg

@@ -47,5 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/slapos-master/instance-erp5.cfg.in

@@ -6,16 +6,11 @@
 [request-common]
 recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
-sla = computer_guid
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
-config =
-  use-ipv6
-  ${:extra-config}
-extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
 {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }}
 {%   endif -%}
 {% endfor -%}
 sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
-extra-config = {{ ' '.join(config) }}
 {% for option, value in config.items() -%}
 config-{{ option }} = {{ dumps(value) }}
 {% endfor -%}
@@ -43,8 +37,13 @@ config-{{ option }} = {{ dumps(value) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
 {# Fail early if an unexpected value is provided -#}
-{% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%}
-{{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
+{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
+{% if {'zeo': 1, None: 0}[zodb_type] -%}
+{{   request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% else -%}
+{% do zodb_extern[0] %}
+{% endif -%}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}
@@ -67,34 +66,6 @@ recipe = slapos.cookbook:generate.password
 return =
   zope-address-list
   hosts-dict
-extra-config =
-  bt5
-  bt5-repository-url
-  cloudooo-url
-  deadlock-debugger-password
-  developer-list
-  hosts-dict
-  inituser-login
-  inituser-password
-  instance-count
-  kumofs-url
-  longrequest-logger-interval
-  longrequest-logger-timeout
-  memcached-url
-  mysql-test-url-list
-  mysql-url-list
-  name
-  port-base
-  site-id
-  smtp-url
-  thread-amount
-  tidstorage-ip
-  tidstorage-port
-  timerserver-interval
-  timezone
-  webdav
-  zodb-dict
-  zodb-storage-type
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator')) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
@@ -109,11 +80,16 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
 config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
+config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+{% if zodb_type -%}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-zodb-dict = ${request-zodb:connection-zodb-dict}
 config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
+{% endif -%}
+{% if zodb_extern -%}
+config-zodb-extern = {{ dumps(zodb_extern) }}
+{% endif -%}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
@@ -168,19 +144,6 @@ config-url = ${request-balancer:{{ family_name }}-v6}
 name = balancer
 software-type = balancer
 sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
-extra-config =
-  tcpv4-port
-  haproxy-server-check-path
-  backend-path-dict
-  ssl-authentication-dict
-  ssl
-  zope-family-dict
-  country-code
-  email
-  state
-  city
-  company
-  {{ zope_address_list_id_dict.values() | join(' ') }}
 return =
 {%- for family in zope_family_dict %}
   {{ family }}
@@ -207,7 +170,6 @@ config-company = {{ slapparameter_dict.get('company', 'Compagny') }}
 < = request-common
 software-url = {{ dumps(frontend_dict['software-url']) }}
 software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
-sla = instance_guid
 sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
 slave = true
 {% set config_dict = {
@@ -216,7 +178,6 @@ slave = true
 {%   if frontend_dict.get('domain') -%}
 {%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
 {%   endif -%}
-extra-config = url {{ config_dict.keys() | join(' ') }}
 {%   for name, value in config_dict.items() -%}
 config-{{ name }} = {{ value }}
 {%   endfor -%}

software/slapos-master/software.cfg

@@ -55,7 +55,7 @@ extra-paths +=
   ${vifib:location}/master
 
 [template-erp5]
-md5sum = 1b1ba4a3d57e6f91eb86e36b19f75463
+md5sum = 6ada1fd4af0a451516443bfb6d00b717
 
 [template-balancer]
 md5sum = 818ab59ae966114735866aecef7a8563

software/slapos-testing/instance.cfg

@@ -69,14 +69,6 @@ recipe = plone.recipe.command
 command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
 update-command = $${:command}
 
-[officejs]
-<= download-source
-repository = ${officejs-repository:location}
-
-[jio]
-<= download-source
-repository = ${jio-repository:location}
-
 [slapos-test-runner]
 recipe = slapos.cookbook:egg_test
 run-test-suite = $${create-directory:bin}/runTestSuite
@@ -92,8 +84,6 @@ test-list =
   $${slapos.recipe.build:location}
   $${slapos.recipe.cmmi:location}
   $${erp5-util:location}
-  $${jio:location}
-  $${officejs:location}
 prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
 environment = environment
 

software/slapos-testing/software.cfg

@@ -19,8 +19,6 @@ parts =
   slapos.recipe.build-repository
   slapos.recipe.cmmi-repository
   erp5-util-repository
-  officejs-repository
-  jio-repository
   eggs
   phantomjs
   template
@@ -33,6 +31,8 @@ eggs =
   slapos.cookbook
   collective.recipe.template
   plone.recipe.command
+  slapos.recipe.template
+  slapos.recipe.cmmi
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
 scripts =
@@ -72,18 +72,10 @@ repository = http://git.erp5.org/repos/slapos.recipe.cmmi.git
 <= git-clone-repository
 repository = http://git.erp5.org/repos/erp5.git
 
-[officejs-repository]
-<= git-clone-repository
-repository = http://git.erp5.org/repos/officejs.git
-
-[jio-repository]
-<= git-clone-repository
-repository = http://git.erp5.org/repos/jio.git
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 94bfc33413cdfbb6e243002f6d9b6928
+md5sum = 934df9a6b59fd7cc43195931b3ba4520
 output =  ${buildout:directory}/template.cfg
 mode = 640
 
@@ -91,5 +83,4 @@ mode = 640
 Pygments = 1.6
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/slaprunner/common.cfg

@@ -21,6 +21,9 @@ extends =
   ../../stack/resilient/buildout.cfg
   ../../stack/monitor/buildout.cfg
 
+# stacks are listed from most generic to most specific,
+# to avoid versioning issues
+
 parts =
   template
   eggs
@@ -43,14 +46,14 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = a4f3127bc82ff3a671a65a150f26c969
+md5sum = 8ebc0e59db33f0105968a813a3130236
 mode = 0644
 
 [template-runner-import-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
 download-only = true
-md5sum = d64529e263d42b0b56e5fe3a976d18c9
+md5sum = c0d05a26b06ce172efaad03c52ef92ca
 filename = runner-import.sh.jinja2
 mode = 0644
 
@@ -65,7 +68,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = e74a0cbda64182c7fdff42cc46f07a12
+md5sum = 0f290b46c0b89ff84aee5c10477e07ca
 filename = runner-export.sh.jinja2
 mode = 0644
 
@@ -87,7 +90,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = 01545742e97b4b95cd5e14d5d7cb1584
+md5sum = 105c7c427def7b2643b7480b2bada3a5
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -110,7 +113,7 @@ mode = 0644
 [template_launcher]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/launcher.in
-md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
+md5sum = 525e37ea8b2acf6209869999b15071a6
 filename = launcher.in
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -41,9 +41,7 @@ recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
 software-type = resilient
 name = Resilient Instance (Root Instance)
-{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
-config = resiliency-backup-periodicity resilient-clone-number {{ cluster_parameter_dict.keys() | join(' ') }}
-{% for key, value in cluster_parameter_dict.items() -%}
+{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-resiliency-backup-periodicity = */6 * * * *
@@ -51,7 +49,6 @@ config-resilient-clone-number = 2
 # XXX hardcoded
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 return = backend_url
 

software/slaprunner/instance-runner-input-schema.json

@@ -4,92 +4,92 @@
         "user-authorized-key": {
             "title": "User Authorized Key",
             "description": "SSH public key in order to connect to the SSH server of this runner instance.",
-            "type": "string",
+            "type": "string"
         },
         "instance-amount": {
             "title": "Partition Amount",
             "description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.",
             "type": "integer",
             "minimum": 1,
-            "maximum": 40,
+            "maximum": 40
         },
         "slapos-software": {
             "title": "Pre-selected Software Release",
             "description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
-            "type": "string",
+            "type": "string"
         },
         "auto-deploy": {
             "title": "Automatically Deploy Software",
             "description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "slapos-repository": {
             "title": "SlapOS Git Repository URL",
             "description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is http://git.erp5.org/repos/slapos.git)",
             "type": "string",
             "format": "uri",
-            "pattern": "^(http|https|ftp)://",
+            "pattern": "^(http|https|ftp)://"
         },
         "slapos-reference": {
             "title": "SlapOS Git Branch Name",
             "description": "Branch or hash on which the default repository will checkout (default is master)",
-            "type": "string",
+            "type": "string"
         },
         "auto-deploy-instance": {
             "title": "Automatically Deploy Instances",
             "description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "autorun": {
             "title": "Automatically Run Sofware/Instance",
             "description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "slapos-software-type": {
             "title": "Deployed Instance Software Type" ,
             "description": "Software type of your instance inside the runner",
-            "type": "string",
+            "type": "string"
         },
         "custom-frontend-backend-url": {
             "title": "Custom Frontend Backend URL",
             "description": "return an ipv4 frontend of the given ipv6(+optional port)",
             "type": "string",
-            "format": "uri",
+            "format": "uri"
         },
         "custom-frontend-backend-type": {
             "title": "Custom Frontend Backend Type",
             "description": "The type of the frontend slave instance to ask",
             "type": "string",
-            "enum": ["zope"],
+            "enum": ["zope"]
         },
         "custom-frontend-basic-auth": {
             "title": "Custom Frontend Basic Auth",
             "description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
-            "type": "boolean",
+            "type": "boolean"
         },
         "custom-frontend-instance-guid": {
             "title": "Custom Frontend Instance GUID",
             "description": "Instance guid of the frontend you whish to use",
-            "type": "string",
+            "type": "string"
         },
         "custom-frontend-software-type": {
             "title": "Custom Frontend Software Type",
             "description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)",
             "type": "string",
-            "enum": ["RootSoftwareInstance", "default"],
+            "enum": ["RootSoftwareInstance", "default"]
         },
         "custom-frontend-software-url": {
             "title": "Custom Frontend Software URL",
             "description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)",
             "type": "string",
-            "format": "uri",
+            "format": "uri"
         },
         "monitor-port": {
             "title": "Monitor Port",
             "description": "Allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server (default 9684)",
             "type": "integer",
             "minimum": 9683,
-            "exclusiveMinimum": true,
+            "exclusiveMinimum": true
         },
         "cpu-usage-ratio": {
             "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
@@ -100,7 +100,7 @@
             "title": "Number of Resilient Clones",
             "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
             "type": "integer",
-            "enum": [1, 2],
+            "enum": [1, 2]
         }
     }
 }

software/slaprunner/instance-runner-output-schema.json

@@ -15,7 +15,7 @@
       "type": "string"
     },
     "ssh_command": {
-      "description": "SSH command used to access your runner in ssh when you provided a ssh public key"
+      "description": "SSH command used to access your runner in ssh when you provided a ssh public key",
       "type": "string"
     },
     "monitor_url": {

software/slaprunner/instance-runner.cfg

@@ -78,16 +78,12 @@ computer-id = $${slap-connection:computer-id}
 partition-id = $${slap-connection:partition-id}
 
 {%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
-sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 {% endif -%}
 
-{% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type')%}
-{% if custom_frontend_backend_type %}
-config = url type
+{% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type') -%}
+{% if custom_frontend_backend_type -%}
 config-type = {{ custom_frontend_backend_type }}
-{% else %}
-config = url
 {% endif -%}
 config-url = {{ slapparameter_dict.get('custom-frontend-backend-url') }}
 return = site_url domain
@@ -281,6 +277,7 @@ path_log = $${directory:log}/nginx.log
 path_access_log = $${directory:log}/nginx.access.log
 path_error_log = $${directory:log}/nginx.error.log
 path_tmp = $${directory:tmp}/
+nginx_prefix = $${buildout:directory}
 # Config files
 path_nginx_conf = $${directory:etc}/nginx.conf
 # Executables
@@ -421,7 +418,6 @@ name = SlapRunner Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = $${slaprunner:access-url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain
@@ -433,7 +429,6 @@ name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain

software/slaprunner/launcher.in

@@ -3,4 +3,6 @@
 # BEWARE: It will be overwritten automatically
 
 # Run nginx
-exec {{ param_nginx_frontend['bin_nginx'] }} -c {{ param_nginx_frontend['path_nginx_conf'] }}
+exec {{ param_nginx_frontend['bin_nginx'] }} \
+    -p {{ param_nginx_frontend['nginx_prefix'] }} \
+    -c {{ param_nginx_frontend['path_nginx_conf'] }}

software/slaprunner/software.cfg

@@ -20,25 +20,28 @@ gitdb = 0.5.4
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-rdiff-backup = 1.0.5
 slapos.recipe.build = 0.12
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.cookbook==0.87
+jsonschema = 2.4.0
+
+# Required by:
+# slapos.toolbox==0.40.4
 paramiko = 1.15.1

software/slaprunner/template/runner-export.sh.jinja2

 #!{{ shell_binary }}
-LANG=en_US
-export $LANG
+LC_ALL=C
+export LC_ALL
 umask 077
 sync_element () {
   path=$1
@@ -11,7 +11,7 @@ sync_element () {
   do
     cd $path;
     if [ -f $element ] || [ -d $element ]; then
-       {{ rsync_binary }} -av --safe-links --delete $element  $backup_path;
+       {{ rsync_binary }} -rlptgov --safe-links --delete $element  $backup_path;
     fi
   done
 }
@@ -23,3 +23,5 @@ sync_element {{ directory['etc'] }}  {{ directory['backup'] }}/etc/ config.json
 if [ -d {{ directory['backup'] }}/runner/software ]; then
   rm {{ directory['backup'] }}/runner/software/*
 fi
+
+cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature

software/slaprunner/template/runner-import.sh.jinja2

 #!{{ shell_binary }}
+LC_ALL=C
+export LC_ALL
 umask 077
 restore_element () {
   backup_path=$1
@@ -13,6 +15,16 @@ restore_element () {
     fi
   done
 }
+
+write_backup_proof () {
+  cd {{ directory['backup'] }}
+  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
+  diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
+}
+
+# For now we just make the diff before 
+write_backup_proof
+
 restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner  instance project  proxy.db
 restore_element  {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
 cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};

software/test-slave-instance-deployment/instance-default.cfg

@@ -40,7 +40,6 @@ name = Test Root Instance - Slave
 software-url = $${slap-connection:software-release-url}
 software-type = test
 slave = true
-sla = instance_guid
 sla-instance_guid = $${request-test-instance:instance-guid}
 return = foo
 

software/test-slave-instance-deployment/software.cfg

@@ -58,7 +58,7 @@ eggs = collective.recipe.template
 [versions]
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
+slapos.recipe.build = 0.13
 slapos.recipe.template = 2.5
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.

software/unstable/glpi/instance.cfg

-[buildout]
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-parts = instance
-
-[instance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-source = ${application:location}
-
-httpd_binary = ${apache:location}/bin/httpd
-mysql_binary = ${mariadb:location}/bin/mysql
-mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
-mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld_binary = ${mariadb:location}/libexec/mysqld

software/unstable/glpi/software.cfg

 [buildout]
-versions = versions
-
-parts =
-  template
-  apache-php
-  mariadb
-  eggs
-  instance-recipe-egg
   
 extends =
-  ../../stack/lamp.cfg
+  ../../../stack/lamp/buildout.cfg
 
 [application]
-recipe = slapos.recipe.build:download-unpacked
+#url = https://forge.indepnet.net/attachments/download/1811/glpi-0.84.7.tar.gz
+#md5sum = 885d7893460979197888cefafa5a75d0
 url = https://forge.indepnet.net/attachments/download/1151/glpi-0.83.1.tar.gz
 md5sum = 3f9cfb11c09c457a0f0a8fa1563b45a7
 
-[instance-recipe]
-egg = slapos.cookbook
-module = lamp.simple
 
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-#${:_profile_base_location_}/instance.cfg
-#md5sum = Student shall put md5 of instance.cfg here
-output = ${buildout:directory}/template.cfg
+[application-template]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/dummy-config.php.in
+md5sum = d41d8cd98f00b204e9800998ecf8427e
+download-only = True
+filename = template.in
 mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs =
-    ${instance-recipe:egg}
+[application-configuration]
+location = dummy-config.php

software/unstable/hadoop-demo/gutenberg/data-download.sh

+#!/bin/bash
+
+. environment.sh
+
+DIR=var/gutenberg/raw-data
+
+mkdir -p $DIR
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/103/pg103.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/18857/pg18857.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/2488/pg2488.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/164/pg164.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/1268/pg1268.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/800/pg800.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/4791/pg4791.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/3526/pg3526.txt
+wget -P $DIR -c http://www.gutenberg.org/cache/epub/2083/pg2083.txt
+

software/unstable/hadoop-demo/gutenberg/mapper.py

+#!/usr/bin/env python
+# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
+
+import sys
+
+# input comes from STDIN (standard input)
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+    # split the line into words
+    words = line.split()
+    # increase counters
+    for word in words:
+        # write the results to STDOUT (standard output);
+        # what we output here will be the input for the
+        # Reduce step, i.e. the input for reducer.py
+        #
+        # tab-delimited; the trivial word count is 1
+        print '%s\t%s' % (word, 1)

software/unstable/hadoop-demo/gutenberg/put-files.sh.in

+#!/bin/bash
+
+# exit on error
+# set -e
+
+source environment.sh
+
+hdfs dfs -mkdir var/gutenberg/input
+
+RAW_DATA=var/gutenberg/raw-data
+
+for file in `ls $RAW_DATA`; do
+  hdfs dfs -put $RAW_DATA/$file var/gutenberg/input
+done
+
+
+

software/unstable/hadoop-demo/gutenberg/reducer.py

+#!/usr/bin/env python
+# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
+
+from operator import itemgetter
+import sys
+
+current_word = None
+current_count = 0
+word = None
+
+# input comes from STDIN
+for line in sys.stdin:
+    # remove leading and trailing whitespace
+    line = line.strip()
+
+    # parse the input we got from mapper.py
+    word, count = line.split('\t', 1)
+
+    # convert count (currently a string) to int
+    try:
+        count = int(count)
+    except ValueError:
+        # count was not a number, so silently
+        # ignore/discard this line
+        continue
+
+    # this IF-switch only works because Hadoop sorts map output
+    # by key (here: word) before it is passed to the reducer
+    if current_word == word:
+        current_count += count
+    else:
+        if current_word:
+            # write result to STDOUT
+            print '%s\t%s' % (current_word, current_count)
+        current_count = count
+        current_word = word
+
+# do not forget to output the last word if needed!
+if current_word == word:
+    print '%s\t%s' % (current_word, current_count)

software/unstable/hadoop-demo/gutenberg/run.sh.in

+#!/bin/bash
+
+# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
+
+. environment.sh
+hadoop jar software_release/parts/hadoop-streaming/*jar \
+    -mapper demo/gutenberg/mapper.py \
+    -reducer demo/gutenberg/reducer.py \
+    -input var/gutenberg/input/* \
+    -output var/gutenberg/output
+

software/unstable/hadoop-demo/instance.cfg.in

+[buildout]
+
+extends =
+  ${instance-stack:output}
+
+parts =
+  sh-environment
+  start-daemons
+  deploy-tar
+  gutenberg-data-download
+  gutenberg-mapper
+  gutenberg-reducer
+  gutenberg-run
+  gutenberg-put-files
+  wikipedia-data-download
+  wikipedia-mapper
+  wikipedia-reducer
+  wikipedia-run
+  wikipedia-put-files
+
+
+[directories]
+demo = $${buildout:directory}/demo
+gutenberg = $${:demo}/gutenberg
+wikipedia = $${:demo}/wikipedia
+
+
+[gutenberg-data-download]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/gutenberg/data-download.sh
+#md5sum = 
+output = $${directories:gutenberg}/data-download.sh
+mode = 0755
+
+
+[gutenberg-mapper]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/gutenberg/mapper.py
+#md5sum = 
+output = $${directories:gutenberg}/mapper.py
+mode = 0755
+
+
+[gutenberg-reducer]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/gutenberg/reducer.py
+#md5sum = 
+output = $${directories:gutenberg}/reducer.py
+mode = 0755
+
+
+[gutenberg-run]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/gutenberg/run.sh.in
+#md5sum = 
+output = $${directories:gutenberg}/run.sh
+mode = 0755
+
+
+[gutenberg-put-files]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/gutenberg/put-files.sh.in
+#md5sum = 
+output = $${directories:gutenberg}/put-files.sh
+mode = 0755
+
+
+[wikipedia-data-download]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/wikipedia/data-download.sh
+#md5sum = 
+output = $${directories:wikipedia}/data-download.sh
+mode = 0755
+
+
+[wikipedia-mapper]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/wikipedia/mapper.py
+#md5sum = 
+output = $${directories:wikipedia}/mapper.py
+mode = 0755
+
+
+[wikipedia-reducer]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/wikipedia/reducer.py
+#md5sum = 
+output = $${directories:wikipedia}/reducer.py
+mode = 0755
+
+
+[wikipedia-run]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/wikipedia/run.sh.in
+#md5sum = 
+output = $${directories:wikipedia}/run.sh
+mode = 0755
+
+
+[wikipedia-put-files]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/wikipedia/put-files.sh.in
+#md5sum = 
+output = $${directories:wikipedia}/put-files.sh
+mode = 0755
+
+

software/unstable/hadoop-demo/software.cfg

+[buildout]
+
+extends = 
+  ../../../stack/hadoop/buildout.cfg
+
+parts =
+  slapos-cookbook
+  libffi
+  python-cffi
+  eggs
+  java
+  hadoop
+  hadoop-streaming
+  instance-stack
+  instance
+
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+# md5sum = 
+mode = 0644
+
+
+[versions]
+collective.recipe.template = 1.11
+cp.recipe.cmd = 0.5nxd001
+plone.recipe.command = 1.1
+slapos.recipe.template = 2.5
+

software/unstable/hadoop-demo/wikipedia/data-download.sh

+#!/bin/bash
+
+. environment.sh
+
+DIR=var/wikipedia/raw-data
+
+mkdir -p $DIR
+
+# http://dumps.wikimedia.org/enwiki/20140203/
+
+# All pages, current versions only.
+
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current1.xml-p000000010p000010000.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current2.xml-p000010001p000025000.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current3.xml-p000025001p000055000.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current4.xml-p000055002p000104998.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current5.xml-p000105001p000184999.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current6.xml-p000185003p000305000.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current7.xml-p000305002p000464997.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current8.xml-p000465001p000665000.bz2
+wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current9.xml-p000665001p000925000.bz2
+
+# don't download the full dataset
+
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current10.xml-p000925001p001325000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current11.xml-p001325001p001825000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current12.xml-p001825001p002425000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current13.xml-p002425001p003124998.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current14.xml-p003125001p003924999.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current15.xml-p003925001p004825000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current16.xml-p004825002p006025000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current17.xml-p006025001p007524997.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current18.xml-p007525002p009225000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current19.xml-p009225001p011125000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current20.xml-p011125001p013324998.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current21.xml-p013325001p015725000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current22.xml-p015725003p018225000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current23.xml-p018225001p020925000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current24.xml-p020925002p023725000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current25.xml-p023725001p026624999.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current26.xml-p026625002p029625000.bz2
+# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current27.xml-p029625001p041836446.bz2

software/unstable/hadoop-demo/wikipedia/mapper.py

+#!/usr/bin/env python
+
+import bz2
+import os
+import sys
+import xml.sax
+
+class WikipediaTitleHandler(xml.sax.ContentHandler):
+    def startElement(self, name, attrs):
+        self.chars = []
+        self.tag = name
+
+    def characters(self, content):
+        if self.tag == 'title':
+            self.chars.append(content)
+
+    def endElement(self, name):
+        if self.tag == 'title':
+            title = ''.join(self.chars)
+            if title.startswith('Talk:'):
+                return
+            if title.startswith('User talk:'):
+                return
+            if title.startswith('Wikipedia:'):
+                return
+            if title.startswith('Wikipedia talk:'):
+                return
+            if title.startswith('User:'):
+                return
+            print title.encode('utf8')
+
+
+
+
+
+def process_xml(input):
+    
+    parser = xml.sax.make_parser()
+    parser.setContentHandler(WikipediaTitleHandler())
+    parser.parse(input)
+
+
+if __name__ == '__main__':
+  input = bz2.BZ2File('/dev/fd/0')
+  process_xml(input)
+

software/unstable/hadoop-demo/wikipedia/put-files.sh.in

+#!/bin/bash
+
+# exit on error
+# set -e
+
+source environment.sh
+
+hdfs dfs -mkdir var/wikipedia/input
+
+RAW_DATA=var/wikipedia/raw-data
+
+for file in `ls $RAW_DATA`; do
+  hdfs dfs -put $RAW_DATA/$file var/wikipedia/input
+done
+
+
+

software/unstable/hadoop-demo/wikipedia/reducer.py

+#!/usr/bin/env python
+
+import bz2
+import os
+import sys
+import xml.sax
+
+class WikipediaTitleHandler(xml.sax.ContentHandler):
+    def startElement(self, name, attrs):
+        self.chars = []
+        self.tag = name
+
+    def characters(self, content):
+        if self.tag == 'title':
+            self.chars.append(content)
+
+    def endElement(self, name):
+        if self.tag == 'title':
+            title = ''.join(self.chars)
+            if title.startswith('Talk:'):
+                return
+            if title.startswith('User talk:'):
+                return
+            if title.startswith('Wikipedia:'):
+                return
+            if title.startswith('Wikipedia talk:'):
+                return
+            if title.startswith('User:'):
+                return
+            print title.encode('utf8')
+
+
+
+
+
+def process_xml(input):
+    
+    parser = xml.sax.make_parser()
+    parser.setContentHandler(WikipediaTitleHandler())
+    parser.parse(input)
+
+
+
+
+if __name__ == '__main__':
+  input = bz2.BZ2File('/dev/fd/0')
+  process_xml(input)
+
+
+
+#  dirname = '/srv/slapgrid/slappart20/srv/runner/instance/slappart0/software_release/raw-data/'
+#  filenames = os.listdir(dirname)
+#  # ['enwiki-20140203-pages-meta-current1.xml-p000000010p000010000.bz2']
+#  for fname in filenames:
+#      process_xml(os.path.join(dirname, fname))
+#      input = bz2.BZ2File(process_xml(os.path.join(dirname, fname)))
+
+
+
+
+
+

software/unstable/hadoop-demo/wikipedia/run.sh.in

+#!/bin/bash
+
+. environment.sh
+hadoop jar software_release/parts/hadoop-streaming/*jar \
+    -mapper demo/wikipedia/mapper.py \
+    -reducer demo/wikipedia/reducer.py \
+    -input var/wikipedia/input/* \
+    -output var/wikipedia/output
+

software/unstable/hexaglobe-watermarking/instance-watermarking.cfg.in

@@ -165,7 +165,6 @@ name = Frontend for $${slap-connection:computer-id} $${slap-connection:partition
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url
 config-url = http://[$${instance-parameter:ipv6-random}]:5000
 return = site_url
 

software/unstable/lamp-generic/instance-apache-php.cfg.in

@@ -151,7 +151,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}

software/unstable/lamp-generic/instance-edge.cfg.in

@@ -20,7 +20,6 @@ return = url backend-url
 country-list = $${slap-parameter:country-list}
 software-url = $${slap-connection:software-release-url}
 software-type = default
-config = application-location
 config-application-location = $${slap-parameter:application-location}
 return = backend-url
 

software/unstable/lamp-generic/software.cfg

@@ -13,7 +13,7 @@ md5sum = 26ac6629a83869140189a85e581e822f
 
 [instance-apache-php]
 url = ${:_profile_base_location_}/instance-apache-php.cfg.in
-md5sum = fdc849c9f49b9d2fb043bf229d24076b
+md5sum = 062e77e6be6cda965cfdd0e6011df86e
 
 [instance-edge]
 recipe = slapos.recipe.template

software/unstable/maarch/software.cfg

 [buildout]
 
 extends =
-  ../../component/git/buildout.cfg
-  ../../stack/lapp/buildout.cfg
+  ../../../component/git/buildout.cfg
+  ../../../stack/lapp/buildout.cfg
 
 
 # += since we need rdiff-backup and friends

software/unstable/mioga/instance-apacheperl.cfg

@@ -75,7 +75,6 @@ name = Postgres
 software-url = $${slap-connection:software-release-url}
 software-type = postgres
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [postgres-urlparse]
@@ -139,7 +138,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url
 config-url = $${mioga-url:direct_url}
 return = site_url
 

software/unstable/trac-svn/instance-trac.cfg

@@ -181,7 +181,6 @@ name = MariaDB
 software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
@@ -437,7 +436,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = http://[$${slap-network-information:global-ipv6}]:8080
 return = site_url
 config-custom_domain = $${slap-parameter:domain}

software/unstable/trac-svn/software.cfg

@@ -84,7 +84,7 @@ md5sum = bc6ed91a1862a10af661713aa0691848
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-trac.cfg
 output = ${buildout:directory}/template-trac.cfg
-md5sum = 1546a60e086b4d083d58ad4fca6dcca8
+md5sum = 2dc7ebfa9b7d5117274b74bafed4cfd4
 mode = 0644
 
 [instance-mariadb]

software/unstable/zimbra-kvm/instance-kvm.cfg.in

@@ -196,11 +196,9 @@ partition-id = $${slap-connection:partition-id}
 name = VNC Frontend
 software-type = $${slap-parameter:frontend-software-type}
 slave = true
-config = host port
 config-host = $${novnc-instance:ip}
 config-port = $${novnc-instance:port}
 return = url resource port domainname
-sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 [request-web-frontend]
@@ -210,7 +208,6 @@ name = Web Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = https://[$${tunnel-ipv6-kvm-https:ipv6}]:$${tunnel-ipv6-kvm-https:ipv6-port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}
@@ -223,7 +220,6 @@ name = SMTP Frontend
 #software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/zimbra-kvm:/software/reverse-proxy-nginx/development.cfg
 software-url = /opt/slapdev/software/reverse-proxy-nginx/development.cfg
 slave = true
-config = smtp-upstream-host smtp-upstream-port
 config-smtp-upstream-host = $${tunnel-ipv6-kvm-smtp:ipv6}
 config-smtp-upstream-port = $${tunnel-ipv6-kvm-smtp:port}
 return = listening-ipv4

software/varnish/software.cfg

@@ -101,4 +101,4 @@ scripts =
 
 [versions]
 erp5.util = 0.4.41
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4

stack/boinc/buildout.cfg

@@ -34,7 +34,7 @@ eggs =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-boinc.cfg
 output = ${buildout:directory}/template-boinc.cfg
-md5sum = 6642942dde15ab5195d8d8533690ef29
+md5sum = 5eb6057311fb1f73a54b72a40ead47c2
 mode = 0644
 
 #Template for deploying MySQL Database Server
@@ -71,7 +71,7 @@ gitdb = 0.5.4
 plone.recipe.command = 1.1
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

stack/boinc/instance-boinc.cfg

@@ -302,7 +302,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}

stack/cloudooo.cfg

@@ -9,27 +9,28 @@ exec-sitecustomize = false
 
 extends =
   ../stack/slapos.cfg
+  ../component/cloudooo/buildout.cfg
+  ../component/coreutils/buildout.cfg
   ../component/cups/buildout.cfg
   ../component/dbus/buildout.cfg
+  ../component/dcron/buildout.cfg
+  ../component/ffmpeg/buildout.cfg
+  ../component/file/buildout.cfg
+  ../component/fonts/buildout.cfg
+  ../component/git/buildout.cfg
   ../component/glib/buildout.cfg
-  ../component/libreoffice-bin/buildout.cfg 
+  ../component/haproxy/buildout.cfg
+  ../component/imagemagick/buildout.cfg
   ../component/libffi/buildout.cfg
   ../component/libpng/buildout.cfg
+  ../component/libreoffice-bin/buildout.cfg
   ../component/lxml-python/buildout.cfg
-  ../component/python-2.7/buildout.cfg
-  ../component/xorg/buildout.cfg
-  ../component/fonts/buildout.cfg
-  ../component/git/buildout.cfg
+  ../component/mesa/buildout.cfg
   ../component/poppler/buildout.cfg
-  ../component/imagemagick/buildout.cfg
-  ../component/ffmpeg/buildout.cfg
-  ../component/file/buildout.cfg
-  ../component/stunnel/buildout.cfg
+  ../component/python-2.7/buildout.cfg
   ../component/rdiff-backup/buildout.cfg
-  ../component/dcron/buildout.cfg
-  ../component/coreutils/buildout.cfg
-  ../component/cloudooo/buildout.cfg
-  ../component/haproxy/buildout.cfg
+  ../component/stunnel/buildout.cfg
+  ../component/xorg/buildout.cfg
 
 versions = versions
 
@@ -57,8 +58,11 @@ parts =
   dbus
   dbus-glib
   glib
+  libexpat
   libffi
   libpng12
+  xdamage
+  xfixes
   imagemagick
   file
   poppler
@@ -77,7 +81,6 @@ WSGIUtils = 0.7
 plone.recipe.command = 1.1
 python-magic = 0.4.6
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
 
 # Required by:

stack/erp5/buildout.cfg

@@ -25,6 +25,8 @@ extends =
   ../../component/libpng/buildout.cfg
   ../../component/libreoffice-bin/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/mesa/buildout.cfg
+  ../../component/numpy/buildout.cfg
   ../../component/percona-toolkit/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/mysql-python/buildout.cfg
@@ -178,7 +180,7 @@ md5sum = 5ebf310655d5de27da039d71a63d209b
 [template-cloudooo]
 < = download-base
 filename = instance-cloudoo.cfg.in
-md5sum = 838e56f46ba749dfc53b8e69e21be008
+md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64
 
 [template-zope-conf]
 < = download-base
@@ -215,7 +217,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 99159fc1e32a0651b35f824709d561d0
+md5sum = 7c622aaa7873e04ec340383b22ede3be
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -226,8 +228,8 @@ extra-context =
     key cups_location cups:location
     key curl_location curl:location
     key dash_location dash:location
-    key dbus_location dbus:location
     key dbus_glib_location dbus-glib:location
+    key dbus_location dbus:location
     key dcron_location dcron:location
     key erp5_location erp5:location
     key file_location file:location
@@ -236,6 +238,7 @@ extra-context =
     key fonts_location fonts:location
     key freetype_location freetype:location
     key glib_location glib:location
+    key glu_location glu:location
     key gzip_location gzip:location
     key haproxy_location haproxy:location
     key jsl_location jsl:location
@@ -247,6 +250,7 @@ extra-context =
     key libXdmcp_location libXdmcp:location
     key libXext_location libXext:location
     key libXrender_location libXrender:location
+    key libexpat_location libexpat:location
     key libffi_location libffi:location
     key libpng12_location libpng12:location
     key libreoffice_bin_location libreoffice-bin:location
@@ -255,6 +259,7 @@ extra-context =
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
     key mariadb_location mariadb:location
+    key mesa_location mesa:location
     key openssl_location openssl:location
     key sixtunnel_location 6tunnel:location
     key stunnel_location stunnel:location
@@ -276,12 +281,14 @@ extra-context =
     key template_zope_conf template-zope-conf:target
     key userhosts_location userhosts:location
     key wget_location wget:location
+    key xdamage_location xdamage:location
+    key xfixes_location xfixes:location
     key zlib_location zlib:location
 
 [template-erp5]
 < = download-base
 filename = instance-erp5.cfg.in
-md5sum = 2ac3bbeb031eb74fbd6f9d033c71b677
+md5sum = 386f5f15d71636f855f22201d38367e5
 
 [template-zeo]
 < = download-base
@@ -439,6 +446,7 @@ ZODB3-patch-binary = ${patch:location}/bin/patch
 [eggs]
 recipe = zc.recipe.egg
 eggs =
+  ${numpy:egg}
   ${mysql-python:egg}
   ${lxml-python:egg}
   ${pillow-python:egg}
@@ -458,7 +466,6 @@ eggs =
   ipdb
   Jinja2
   mechanize
-  numpy
   paramiko
   ply
   pyflakes
@@ -563,7 +570,7 @@ setup = ${erp5:location}
 
 [cloudooo-repository]
 branch =
-revision = f67af2797f74d4212f7221978645552353f341c8
+revision = 8db3977b312e3cf8dbb64660c6f4f9e639b749c9
 
 [slapos.cookbook-repository]
 branch = erp5-cluster
@@ -595,9 +602,6 @@ PasteDeploy = 1.5.2
 Pygments = 1.6
 coverage = 3.7.1
 
-# Known version with works
-numpy = 1.6.2
-
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
 Products.PluggableAuthService = 1.9.0
 
@@ -632,10 +636,11 @@ zope.app.testing = 3.8.1
 
 # Pinned versions
 MySQL-python = 1.2.5
+Pillow = 2.6.1
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
-Products.GenericSetup = 1.7.4
+Products.GenericSetup = 1.7.5
 Products.LongRequestLogger = 1.1.0
 Products.MimetypesRegistry = 2.0.6
 Products.PluginRegistry = 1.3
@@ -648,7 +653,7 @@ WSGIUtils = 0.7
 apache-libcloud = 0.15.1
 astroid = 1.2.1
 async = 0.6.1
-chardet = 2.2.1
+chardet = 2.3.0
 csp-eventlet = 0.7.0
 ecdsa = 0.11
 elementtree = 1.2.7-20070827-preview
@@ -657,17 +662,17 @@ eventlet = 0.15.2
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
 gitdb = 0.5.4
-greenlet = 0.4.4
+greenlet = 0.4.5
 http-parser = 0.8.3
 httplib2 = 0.9
 huBarcode = 1.0.0
 interval = 1.0.0
 ipdb = 0.8
-ipython = 2.2.0
-logilab-common = 0.62.1
+ipython = 2.3.0
+logilab-common = 0.63.0
 minitage.paste = 1.4.6
 minitage.recipe.egg = 1.107
-pillow = 2.5.3
+numpy = 1.9.1
 plone.recipe.command = 1.1
 ply = 3.4
 polib = 1.0.5
@@ -675,13 +680,12 @@ pprofile = 1.7.2
 pycountry = 1.8
 pyflakes = 0.8.1
 pylint = 1.3.1
-python-ldap = 2.4.15
+python-ldap = 2.4.17
 python-magic = 0.4.6
 python-memcached = 1.53
-qrcode = 5.0.1
+qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
 slapos.toolbox = 0.43.0
 smmap = 0.8.2

stack/erp5/instance-cloudoo.cfg.in

@@ -33,7 +33,7 @@ wrapper = ${directory:service}/cloudooo
 data-directory = ${directory:cloudooo-data}
 
 environment =
-  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib
+  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
 
 # Binary information

stack/erp5/instance-erp5.cfg.in

@@ -6,16 +6,11 @@
 [request-common]
 recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
-sla = computer_guid
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
-config =
-  use-ipv6
-  ${:extra-config}
-extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
 {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }}
 {%   endif -%}
 {% endfor -%}
 sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
-extra-config = {{ ' '.join(config) }}
 {% for option, value in config.items() -%}
 config-{{ option }} = {{ dumps(value) }}
 {% endfor -%}
@@ -72,30 +66,6 @@ recipe = slapos.cookbook:generate.password
 return =
   zope-address-list
   hosts-dict
-{% set extra_config = '
-  bt5
-  bt5-repository-url
-  cloudooo-url
-  deadlock-debugger-password
-  developer-list
-  hosts-dict
-  inituser-login
-  inituser-password
-  instance-count
-  kumofs-url
-  longrequest-logger-interval
-  longrequest-logger-timeout
-  memcached-url
-  mysql-test-url-list
-  mysql-url-list
-  name
-  port-base
-  site-id
-  smtp-url
-  thread-amount
-  timerserver-interval
-  timezone
-  webdav' -%}
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
@@ -116,19 +86,11 @@ config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
 config-zodb-dict = ${request-zodb:connection-zodb-dict}
 config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
-{% set extra_config = extra_config ~ '
-  tidstorage-ip
-  tidstorage-port
-  zodb-dict
-  zodb-storage-type' -%}
 {% endif -%}
 {% if zodb_extern -%}
 config-zodb-extern = {{ dumps(zodb_extern) }}
-{% set extra_config = extra_config ~ '
-  zodb-extern' -%}
 {% endif -%}
 software-type = zope
-extra-config = {{ extra_config }}
 
 {% set zope_family_dict = {} -%}
 {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
@@ -177,13 +139,6 @@ config-url = ${request-balancer:{{ family_name }}-v6}
 name = balancer
 software-type = balancer
 sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
-extra-config =
-  tcpv4-port
-  haproxy-server-check-path
-  backend-path
-  ssl
-  zope-family-dict
-  {{ zope_address_list_id_dict.values() | join(' ') }}
 return =
 {%- for family in zope_family_dict %}
   {{ family }}
@@ -204,7 +159,6 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
 < = request-common
 software-url = {{ dumps(frontend_dict['software-url']) }}
 software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
-sla = instance_guid
 sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
 slave = true
 {% set config_dict = {
@@ -213,7 +167,6 @@ slave = true
 {%   if frontend_dict.get('domain') -%}
 {%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
 {%   endif -%}
-extra-config = url {{ config_dict.keys() | join(' ') }}
 {%   for name, value in config_dict.items() -%}
 config-{{ name }} = {{ value }}
 {%   endfor -%}

stack/erp5/instance.cfg.in

@@ -32,16 +32,21 @@ file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
 freetype = {{ freetype_location }}
 glib = {{ glib_location }}
-libffi = {{ libffi_location }}
+glu = {{ glu_location }}
 libICE = {{ libICE_location }}
-libpng12 = {{ libpng12_location }}
 libSM = {{ libSM_location }}
 libX11 = {{ libX11_location }}
 libXau = {{ libXau_location }}
 libXdmcp = {{ libXdmcp_location }}
 libXext = {{ libXext_location }}
-libxcb = {{ libxcb_location }}
 libXrender = {{ libXrender_location }}
+libexpat = {{ libexpat_location }}
+libffi = {{ libffi_location }}
+libpng12 = {{ libpng12_location }}
+libxcb = {{ libxcb_location }}
+mesa = {{ mesa_location }}
+xdamage = {{ xdamage_location }}
+xfixes = {{ xfixes_location }}
 zlib = {{ zlib_location }}
 libreoffice-bin = {{ libreoffice_bin_location }}
 fonts = {{ fonts_location }}

stack/hadoop/buildout.cfg

+[buildout]
+
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/libffi/buildout.cfg
+  ../../component/pkgconfig/buildout.cfg
+  ../../component/python-cffi/buildout.cfg
+  ../../component/java/buildout.cfg
+
+
+parts =
+  slapos-cookbook
+  libffi
+  python-cffi
+  eggs
+  java
+  hadoop
+  hadoop-streaming
+  instance-stack
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs = 
+  slapos.cookbook
+  collective.recipe.template
+  cp.recipe.cmd
+  plone.recipe.command
+
+
+[hadoop]
+recipe = hexagonit.recipe.download
+version = 2.5.1
+filename = hadoop-${:version}.tar.gz
+url = http://apache.mirrors.spacedump.net/hadoop/common/hadoop-${:version}/${:filename}
+md5sum = 015b12d790701deb4de9a647013f72a9
+download-only = true
+mode = 0644
+strip-top-level-dir = true
+
+
+[hadoop-streaming]
+recipe = hexagonit.recipe.download
+url = http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-streaming/0.20.203.0/hadoop-streaming-0.20.203.0.jar
+download-only = true
+md5sum = 792c7df75b63e477fef12988b7cced6f
+mode = 0644
+
+
+[instance-stack]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-stack.cfg.in
+output = ${buildout:directory}/instance-stack.cfg
+md5sum =  3c846dd1269aeef43cd2d5448463563b
+mode = 0644
+
+

stack/hadoop/instance-stack.cfg.in

+[buildout]
+
+parts =
+  sh-environment
+  start-daemons
+  deploy-tar
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+
+[sh-environment]
+# environment needed for running/debugging the applications, exported to a shell script to be sourced.
+recipe = collective.recipe.template
+output = $${buildout:directory}/environment.sh
+input = inline:
+  export JAVA_HOME="${java:location}"
+  export HADOOP_PREFIX="$${directories:hadoop-prefix}"
+  export PATH="$PATH:$HADOOP_PREFIX/bin"
+  export HADOOP_HOME="$HADOOP_PREFIX"
+  export HADOOP_COMMON_HOME="$HADOOP_PREFIX"
+  export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/"
+  export HADOOP_HDFS_HOME="$HADOOP_PREFIX"
+  export HADOOP_MAPRED_HOME="$HADOOP_PREFIX"
+  export HADOOP_YARN_HOME="$HADOOP_PREFIX"
+
+
+[deploy-tar]
+recipe = plone.recipe.command
+update = true
+command =
+    [ -d $${directories:hadoop-prefix}/bin} ] || tar xf ${hadoop:location}/${hadoop:filename} -C $${directories:hadoop-prefix}  --strip-components=1
+
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+hadoop-prefix = $${buildout:directory}/hadoop
+services = $${directories:etc}/service
+promises = $${directories:etc}/promise
+
+
+[start-daemons]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template/start-daemons.sh.in
+output = $${directories:bin}/start-daemons.sh
+md5sum = 35a51a7f826d29e47c564c57e954db80
+mode = 0755
+

stack/hadoop/template/start-daemons.sh.in

+#!/bin/bash
+## Start HDFS daemons
+# Format the namenode directory (DO THIS ONLY ONCE, THE FIRST TIME)
+#$HADOOP_PREFIX/bin/hdfs namenode -format
+# Start the namenode daemon
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh start namenode
+# Start the datanode daemon
+$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode
+ 
+## Start YARN daemons
+# Start the resourcemanager daemon
+$HADOOP_PREFIX/sbin/yarn-daemon.sh start resourcemanager
+# Start the nodemanager daemon
+$HADOOP_PREFIX/sbin/yarn-daemon.sh start nodemanager

stack/lamp.cfg

@@ -44,10 +44,9 @@ apache-libcloud = 0.9.1
 async = 0.6.1
 gitdb = 0.5.4
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.7
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.3
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

stack/lamp/apache/instance-apache-php.cfg.in

@@ -179,7 +179,6 @@ name = MariaDB
 software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
@@ -250,7 +249,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-network-configuration:listening-ip}]:$${apache-network-configuration:listening-port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}

stack/lamp/buildout.cfg

@@ -74,7 +74,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
 output = ${buildout:directory}/instance-apache-php.cfg
-md5sum = 3e2d71d3684aac3e52d2f55794df96bf
+md5sum = b2564a719fdde097f402ef80e0991903
 mode = 0644
 
 [instance-apache-import]
@@ -88,6 +88,7 @@ mode = 0644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
 rendered = ${buildout:directory}/instance-apache-export.cfg
+extensions = jinja2.ext.do
 
 context = key templateapache instance-apache-php:output
           key templatepbsreadyexport pbsready-export:output
@@ -102,6 +103,7 @@ mode = 0644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
 rendered = ${buildout:directory}/instance-resilient.cfg
+extensions = jinja2.ext.do
 
 context = key buildout buildout:bin-directory
           key develop_eggs_directory buildout:develop-eggs-directory
@@ -110,7 +112,7 @@ context = key buildout buildout:bin-directory
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = f6dc9290194ad1fbe1ced553f5668922
+md5sum = 5e2f917b032e43af6a736532966ae1f0
 mode = 0644
 
 [instance-mariadb]
@@ -185,10 +187,9 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.11.6
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

stack/lamp/instance-resilient.cfg.jinja2

@@ -19,7 +19,6 @@ parts +=
 [request-apache]
 return = url ssh-public-key ssh-url notification-id ip url backend_url
 # XXX: hardcoded values
-config = domain number authorized-key notify ip-list namebase mariadb1-computer-guid pbs-mariadb1-computer-guid mariadb2-computer-guid pbs-mariadb2-computer-guid mariadb3-computer-guid pbs-mariadb3-computer-guid
 config-mariadb1-computer-guid = ${slap-parameter:mariadb1-computer-guid}
 config-pbs-mariadb1-computer-guid = ${slap-parameter:pbs-mariadb1-computer-guid}
 config-mariadb2-computer-guid = ${slap-parameter:mariadb2-computer-guid}

stack/lapp/apache/instance-apache-php.cfg.in

@@ -179,7 +179,6 @@ name = Postgres
 software-url = $${slap-connection:software-release-url}
 software-type = postgres
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [postgres-urlparse]
@@ -235,7 +234,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}

stack/lapp/buildout.cfg

@@ -35,8 +35,8 @@ extends =
   ../../component/rdiff-backup/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/dropbear/buildout.cfg
-  ../resilient/buildout.cfg
   ../slapos.cfg
+  ../resilient/buildout.cfg
 
 
 #----------------
@@ -62,7 +62,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
 output = ${buildout:directory}/instance-apache-php.cfg
-md5sum = 823257dda6f3068a38c6b69c771cf307
+md5sum = 7a0670c747f2f21a79c240bfbfbf9f6e
 mode = 0644
 
 [instance-apache-import]
@@ -76,6 +76,7 @@ mode = 0644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
 rendered = ${buildout:directory}/instance-apache-export.cfg
+extensions = jinja2.ext.do
 
 context = key templateapache instance-apache-php:output
           key templatepbsreadyexport pbsready-export:output
@@ -90,6 +91,7 @@ mode = 0644
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
 rendered = ${buildout:directory}/instance-resilient.cfg
+extensions = jinja2.ext.do
 
 context = key buildout buildout:bin-directory
           key develop_eggs_directory buildout:develop-eggs-directory
@@ -98,7 +100,7 @@ context = key buildout buildout:bin-directory
 import-list = file parts template-parts:destination
               file replicated template-replicated:destination
 
-md5sum = ef38aa9810ce20960382261f235abfcd
+md5sum = 35bbf8cb40fe7185664641d82b406334
 mode = 0644
 
 [instance-postgres]
@@ -182,11 +184,9 @@ cp.recipe.cmd = 0.4
 gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6
-rdiff-backup = 1.0.5
-slapos.recipe.build = 0.11.6
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

stack/lapp/instance-resilient.cfg.jinja2

@@ -19,7 +19,6 @@ parts +=
 [request-apache]
 return = url ssh-public-key ssh-url notification-id ip url backend_url
 # XXX: hardcoded values
-config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid maarch-sql-data-file
 config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid}
 config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid}
 config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid}

stack/resilient/buildout.cfg

@@ -34,11 +34,11 @@ eggs = collective.recipe.template
 [pbsready]
 # Common parts for pbsready-import and pbsready-export.
 # Provides rdiff-backup, notification queue, ssh authentication,
-# dropbear server, and the bully script.
+# dropbear server, and the takeover script.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = 28e9bc354cc146d1bec31c8cac263ee2
+md5sum = e89d8378cc610704b518a89b095d3a19
 mode = 0644
 
 [pbsready-import]
@@ -47,7 +47,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = 0f953067aac3e0132f72fc7e1ed38bd4
+md5sum = a5570ecfeff7a9d1b5f8be08db4feefe
 mode = 0644
 
 [pbsready-export]
@@ -56,7 +56,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 135638b8c513c7723efb51e3d9182ae9
+md5sum = 879fff114d1dbf1f58774ccbce9bdd22
 mode = 0644
 
 [template-pull-backup]
@@ -69,14 +69,14 @@ mode = 0644
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = 771e1ab7e7e77b35116c57bbae56ba62
+md5sum = a21d751078dbf1fc80ecb93a6f499287
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
 [template-parts]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-parts.cfg.in
-md5sum = a3f55a20881c3f1ec4416662146c06f7
+md5sum = 071b1034ee8f5cc14f79b16fdeba2813
 mode = 0644
 destination = ${buildout:directory}/template-parts.cfg.in
 
@@ -116,8 +116,16 @@ md5sum = 19ee9055de961acf402e2dfe5b9581d2
 filename = monitor-check-resilient-feed.in
 mode = 0644
 
+[rdiff-backup-build]
+# use our own version
+find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
+
 [versions]
 # Pin Jinja2 to 2.6, as 2.7 breaks current code
 Jinja2 = 2.6
 # ... And newer s.r.template requires Jinja2 >= 2.7
 slapos.recipe.template = 2.4.2
+rdiff-backup = 1.3.4nxd2
+slapos.cookbook = 0.91
+
+

stack/resilient/pbsready-export.cfg.in

@@ -54,7 +54,7 @@ pidfile = $${resilient-directory:pid}/$${:name}.pid
 recipe = slapos.cookbook:cron.d
 name = backup
 frequency = $${slap-parameter:resiliency-backup-periodicity}
-command = $${notifier-exporter:wrapper}
+command = $${notifier-exporter:wrapper} --transaction-id `date +%s`
 
 [slap-parameter]
 # In cron.d format (i.e things like */15 * * * * are accepted).

stack/resilient/pbsready-import.cfg.in

@@ -36,6 +36,10 @@ takeover-password = $${resilient-web-takeover-password:passwd}
 [dropbear-server]
 port = 22220
 
+# Define port of notifier (same reason)
+[notifier]
+port = 65533
+
 [import-on-notification]
 # notifier.callback runs a script when a notification (sent by a parent PBS)
 # is received

stack/resilient/pbsready.cfg.in

@@ -128,14 +128,10 @@ rotate-num = 30
 
 #----------------
 #--
-#-- Resiliency script for the bully algorithm
+#-- Resiliency script for the node takeover
 
 [resiliency]
-# If enable-bully-service is true, the scripts will be run automatically.
-# If false, they can be run with bin/bullly for all the PBSReady instances.
-enable-bully-service = False
 recipe = slapos.cookbook:addresiliency
-wrapper-bully = bully
 wrapper-takeover = takeover
 services = $${basedirectory:services}
 bin = $${rootdirectory:bin}

stack/resilient/template-parts.cfg.in

@@ -3,12 +3,10 @@
   {% macro replicate(namebase, nbbackup) %}
 
   request-{{namebase}}
-  request-{{namebase}}-2
   resilient-request-{{namebase}}-public-key-promise
   
   {% for id in range(1,nbbackup|int) %}
   request-{{namebase}}-pseudo-replicating-{{id}}
-  request-{{namebase}}-pseudo-replicating-{{id}}-2
   resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise
   {% endfor %}
   

stack/resilient/template-replicated.cfg.in

@@ -24,10 +24,6 @@ recipe = slapos.cookbook:request
 config-namebase = {{namebase}}
 software-url = ${slap-connection:software-release-url}
 
-## Every request is double to provide the 3 IPs.
-## First, we request with ip-list parameter hardcoded.
-## Then, we request the same thing without this.
-## XXX-Cedric: is it useful?
 [request-{{namebase}}]
 <= resilient
    slap-connection
@@ -35,24 +31,15 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{typeexport}}
 name = {{namebase}}0
 return = ssh-public-key ssh-url notification-id ip
-config =
-# Resilient related parameters
-  number authorized-key notify ip-list namebase
-{% if slapparameter_dict is defined %}
-# Software Instance related parameters
-  {% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
-{% endif %}
 config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
-config-ip-list =
 # Bubble up all the instance parameters to the requested export instance.
 {% if slapparameter_dict is defined %}
 {% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
 {% endfor %}
 {% endif %}
 {% if sla_parameter_dict == {} -%}
-sla = mode
 sla-mode = unique_by_network
 {% else %}
 {%   set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%}
@@ -67,13 +54,10 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   if sla_dict %}
-sla = {{ ' '.join(sla_dict.keys()) }}
 {%   for key, value in sla_dict.iteritems() -%}
 sla-{{ key }} = {{ value }}
 {%   endfor -%}
 {% endif -%}
-{% endif -%}
 
 {% for id in range(1,nbbackup|int) %}
 
@@ -90,13 +74,10 @@ return = ssh-public-key ssh-url notification-url ip
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
 
-config = number authorized-key on-notification ip-list namebase
 config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
-config-ip-list =
 {% if sla_parameter_dict == {} -%}
-sla = mode
 sla-mode = unique_by_network
 {% else %}
 {%   set sla_key_main = "-sla-%s%s-" % (namebase, id) -%}
@@ -111,23 +92,17 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   if sla_dict %}
-sla = {{ ' '.join(sla_dict.keys()) }}
 {%   for key, value in sla_dict.iteritems() -%}
 sla-{{ key }} = {{ value }}
 {%   endfor -%}
 {% endif %}
-{% endif %}
 
-{% endfor -%}
+[publish-connection-informations]
+feed-url-{{namebase}}-{{id}}-push = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
 
+{% endfor -%}
 
-[iplist]
-config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
 
-[request-{{namebase}}-2]
-<= request-{{namebase}}
-   iplist
 
 [resilient-request-{{namebase}}-public-key-promise]
 # Check that public-key-value parameter exists and is not empty
@@ -135,7 +110,7 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
 recipe = collective.recipe.template
 # XXX: don't use system executable
 input = inline:#!/bin/sh
-  PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})"
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}:connection-ssh-public-key})"
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
@@ -143,9 +118,6 @@ output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-ke
 mode = 700
 
 {% for id in range(1,nbbackup|int) %}
-[request-{{namebase}}-pseudo-replicating-{{id}}-2]
-<= request-{{namebase}}-pseudo-replicating-{{id}}
-   iplist
 
 [resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
 # Check that public-key-value parameter exists and is not empty
@@ -153,7 +125,7 @@ mode = 700
 recipe = collective.recipe.template
 # XXX: don't use system executable
 input = inline:#!/bin/sh
-  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})"
+  PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key})"
   if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
@@ -185,7 +157,6 @@ name = PBS ({{namebase}} / {{id}})
 return = ssh-key notification-url feeds-url
 slave = false
 {% if sla_parameter_dict == {} -%}
-sla = mode
 sla-mode = unique_by_network
 {% else %}
 {%   set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%}
@@ -200,13 +171,10 @@ sla-mode = unique_by_network
 {%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
 {%     endif -%}
 {%   endfor -%}
-{%   if sla_dict %}
-sla = {{ ' '.join(sla_dict.keys()) }}
 {%   for key, value in sla_dict.iteritems() -%}
 sla-{{ key }} = {{ value }}
 {%   endfor %}
 {% endif %}
-{% endif %}
 
 [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
 # Check that public-key-value parameter exists and is not empty
@@ -225,7 +193,6 @@ mode = 700
 [request-pull-backup-server-{{namebase}}-{{id}}]
 <= request-pbs-common
 name = PBS {{id}} pulling from ${request-{{namebase}}:name}
-config = url name type server-key on-notification notify notification-id title remove-backup-older-than
 config-url = ${request-{{namebase}}:connection-ssh-url}
 config-type = pull
 config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
@@ -236,13 +203,15 @@ config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{n
 config-title = Pulling from {{namebase}}
 config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '2W') }}
 slave = true
-sla = instance_guid
 sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 
+[publish-connection-informations]
+feed-url-{{namebase}}-{{id}}-pull = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
+
+
 [request-pull-backup-server-{{namebase}}-backup-{{id}}]
 <= request-pbs-common
 name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
-config = url name type server-key on-notification notify notification-id title
 config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
 config-type = push
 config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
@@ -252,7 +221,6 @@ config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-no
 config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
 config-title = Pushing to {{namebase}} backup {{id}}
 slave = true
-sla = instance_guid
 sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 {% endfor %}
 

stack/slapos.cfg

@@ -16,6 +16,7 @@ download-cache =
 extensions +=
   buildout-versions
   mr.developer
+  slapos.extension.strip
 
 # Use shacache and lxml
 extends =
@@ -111,34 +112,36 @@ Werkzeug = 0.9.6
 buildout-versions = 1.7
 cffi = 0.8.6
 cmd2 = 0.6.7
-cryptography = 0.5.4
+cryptography = 0.6
 itsdangerous = 0.24
 lxml = 3.4.0
 meld3 = 1.0.0
-mr.developer = 1.30
+mr.developer = 1.31
 prettytable = 0.7.2
 pyOpenSSL = 0.14
-pyparsing = 2.0.2
-setuptools = 5.8
+pyparsing = 2.0.3
+setuptools = 7.0
 six = 1.8.0
+simplejson = 3.6.5
 slapos.cookbook = 0.87
-slapos.core = 1.2.4
-slapos.libnetworkcache = 0.14.1
+slapos.core = 1.3.3
+slapos.libnetworkcache = 0.14.2
+slapos.recipe.build = 0.14
 slapos.recipe.cmmi = 0.2
-stevedore = 1.0.0
+stevedore = 1.1.0
 
 # Required by:
-# slapos.core==1.2.4
+# slapos.core==1.3.3
 Flask = 0.10.1
 
 # Required by:
-# cliff==1.7.0
-# stevedore==1.0.0
+# cliff==1.8.0
+# stevedore==1.1.0
 argparse = 1.2.1
 
 # Required by:
-# slapos.core==1.2.4
-cliff = 1.7.0
+# slapos.core==1.3.3
+cliff = 1.8.0
 
 # Required by:
 # slapos.cookbook==0.87
@@ -153,12 +156,21 @@ lock-file = 2.0
 netaddr = 0.7.12
 
 # Required by:
-# slapos.core==1.2.4
+# slapos.core==1.3.3
 netifaces = 0.10.4
 
 # Required by:
-# slapos.core==1.2.4
-psutil = 2.1.2
+# cliff==1.8.0
+# stevedore==1.1.0
+pbr = 0.10.0
+
+# Required by:
+# pbr==0.10.0
+pip = 1.5.6
+
+# Required by:
+# slapos.core==1.3.3
+psutil = 2.1.3
 
 # Required by:
 # cffi==0.8.6
@@ -166,22 +178,26 @@ pycparser = 2.10
 
 # Required by:
 # slapos.cookbook==0.87
-pytz = 2014.7
+pytz = 2014.9
+
+# Required by:
+# slapos.core==1.3.3
+requests = 2.4.3
 
 # Required by:
-# slapos.core==1.2.4
-requests = 2.4.1
+# slapos.core==1.3.3
+supervisor = 3.1.3
 
 # Required by:
-# slapos.core==1.2.4
-supervisor = 3.1.2
+# slapos.core==1.3.3
+uritemplate = 0.6
 
 # Required by:
 # slapos.cookbook==0.87
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.core==1.2.4
+# slapos.core==1.3.3
 zope.interface = 4.1.1
 
 [networkcache]