Merge remote-tracking branch 'origin/master' into erp5-cluster

Conflicts:
	stack/erp5/instance-tidstorage.cfg.in

.gitignore

@@ -9,3 +9,5 @@ eggs/
 parts/
 slapos.cookbook.egg-info
 .*.swp
+*~
+\#*\#
\ No newline at end of file

CHANGES.txt

 Changes
 =======
 
-0.78.1 (2013-05-31)
+0.78.5 (2013-08-06)
+-----------------------
+
+ * check_url_available: add option to check secure links [6cbce4d8231]
+
+0.78.4 (2013-08-06)
+-----------------------
+
+ * slapos.cookbook:slaprunner: Update to use https. [Cedric Le Ninivin]
+
+
+0.78.3 (2013-07-18)
+-------------------
+
+ * slapos.cookbook:publish: Add support to publish information for slaves. [Cedric Le Ninivin]
+
+0.78.2 (2013-07-18)
 -------------------
 
  * Fix slapos.cookbook:request: Add backward compatiblity about getInstanceGuid(). [Cedric de Saint Martin]
  * slapos.cookbook:check_* promises: Add timeout to curl that is not otherwise killed by slapos promise subsystem. [Cedric de Saint Martin]
+ * Cloudooo: Allow any environment variables. [Yusei Tahara]
+ * ERP5: disable MariaDB query cache completely by 'query_cache_type = 0' for ERP5. [Kazuhiko Shiozaki]
+ * ERP5: enable haproxy admin socket and install haproxyctl script. [Kazuhiko Shiozaki]
+ * ERP5: increase the maximum number of open file descriptors before starting mysqld. [Kazuhiko Shiozaki]
+ * python 2.7: updated to 2.7.5 [Cedric de Saint Martin]
+
+0.78.1 (2013-05-31)
+-------------------
+
  * Add boinc recipe: Allow to deploy an empty BOINC project. [Alain Takoudjou]
  * Add boinc.app recipe: Allow to deploy and update a BOINC application into existing BOINC server instance . [Alain Takoudjou]
  * Add boinc.client recipe: Allow to deploy a BOINC Client instance on SlapOS. [Alain Takoudjou]
@@ -15,10 +40,6 @@ Changes
  * Add trac recipe: for deploying Trac and manage project with support of SVN and GIT. [Alain Takoudjou]
  * Add bonjourgrid recipe: for deploying BonjourGrid Master and submit BOINC or Condor project. [Alain Takoudjou]
  * Add bonjourgrid.client recipe: for deploying BonjourGrid Worker instance and execute BOINC or Condor Jobs. [Alain Takoudjou]
- * Cloudooo: Allow any environment variables. [Yusei Tahara]
- * ERP5: disable MariaDB query cache completely by 'query_cache_type = 0' for ERP5. [Kazuhiko Shiozaki]
- * ERP5: enable haproxy admin socket and install haproxyctl script. [Kazuhiko Shiozaki]
- * ERP5: increase the maximum number of open file descriptors before starting mysqld. [Kazuhiko Shiozaki]
 
 0.78.0 (2013-04-28)
 -------------------

component/apache/buildout.cfg

@@ -120,8 +120,9 @@ make-targets =
 [apache-2.2]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.24.tar.gz
-md5sum = 64a3392018ad60583209a16d728180d3
+version = 2.2.25
+url = http://mir2.ovh.net/ftp.apache.org/dist/httpd/httpd-${:version}.tar.bz2
+md5sum = 9ebe3070c0bb4311f21a0cd0e34f0045
 patch-options = -p1
 configure-options = --disable-static
                     --enable-authn-alias
@@ -182,6 +183,8 @@ environment =
 recipe = slapos.recipe.cmmi
 url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download
 md5sum = 66862bf10e9be3a023e475604a28a0b4
+depends =
+  ${apache-2.2:version}
 configure-command = ${apache-2.2:location}/bin/apxs
 configure-options = -c mod_antiloris.c
 make-binary = ${:configure-command}

component/cloud9/buildout.cfg

@@ -44,6 +44,13 @@ filename = cloud9-session-directory.patch
 download-only = true
 md5sum = 5dc8cc28447ed3747b8a53c768d872aa
 
+[cloud9-socket.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = cloud9-socket.patch
+download-only = true
+#md5sum = 5dc8cc28447ed3747b8a53c768d872aa
+
 [cloud9-git]
 # Online IDE written in javascript/node.js
 # URL : c9.io
@@ -55,7 +62,7 @@ commit = f7d102bc225c922f116d2cea52a746d64343ea59
 repository = https://github.com/ajaxorg/cloud9.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
-command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1)
+command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-socket.patch:location}/${cloud9-socket.patch:filename}) || (rm -fr ${:location}; exit 1)
 update-command =
 executable = ${:location}/server.js
 

component/cloud9/cloud9-socket.patch

+diff --git a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+index fa7e54a..14b8e67 100644
+--- a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
++++ b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+@@ -2126,7 +2126,7 @@ Polling.prototype.uri = function () {
+     query = '?' + query;
+   }
+ 
+-  return schema + '://' + this.host + port + this.path + query;
++  return this.path + query;
+ };
+ 
+ });require.register("transports/websocket.js", function(module, exports, require, global){

component/coreutils/buildout.cfg

@@ -2,7 +2,7 @@
 extends =
   ../xz-utils/buildout.cfg
 parts =
-  coreutils
+  coreutils-output
 
 [coreutils]
 recipe = slapos.recipe.cmmi
@@ -13,3 +13,13 @@ configure-options =
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   LDFLAGS =-Wl,--as-needed
+
+[coreutils-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm}
+test = ${coreutils:location}/bin/test
+cat = ${coreutils:location}/bin/cat
+rm = ${coreutils:location}/bin/rm

component/dash/buildout.cfg

 [buildout]
+extends = 
+  ../coreutils/buildout.cfg
 
-parts = dash
+parts = dash-output
 
 [dash]
 recipe = slapos.recipe.cmmi
@@ -10,3 +12,11 @@ configure-options =
   --disable-static
   --disable-fnmatch
   --disable-glob
+
+[dash-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:dash}
+dash = ${dash:location}/bin/dash

component/dcron/buildout.cfg

 [buildout]
-parts = dcron
+extends = 
+  ../coreutils/buildout.cfg
+
+parts = dcron-output
+
+[dcron-hooks-download]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 860e914dff4108b47565965fe5ebc7b5
+download-only = true
+filename = dcron-hooks.py
 
 [dcron-patch-nonroot]
 recipe = hexagonit.recipe.download
-md5sum = 2f5b22dc1cbe81060a9c28e6f5c06e8b
+md5sum = d5408ab682b65cc1eda40d588fcd7db8
 url = ${:_profile_base_location_}/${:filename}
 filename = dcron-4.4.noroot.no.globals.patch
 download-only = true
@@ -18,3 +28,13 @@ patches =
 patch-options = -p1
 make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+post-make-hook = ${dcron-hooks-download:location}/${dcron-hooks-download:filename}:post_make_hook
+
+[dcron-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:crond} -a -x ${:crontab} -a ! -u ${:crontab}
+crond = ${dcron:location}/sbin/crond
+crontab = ${dcron:location}/bin/crontab

component/dcron/dcron-4.4.noroot.no.globals.patch

-# Patch for making dcron usable without root user, as a local service
 diff -ru dcron-4.4.org/chuser.c dcron-4.4/chuser.c
 --- dcron-4.4.org/chuser.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/chuser.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/chuser.c	2013-07-18 18:17:16.342147418 +0200
 @@ -14,47 +14,6 @@
  int
  ChangeUser(const char *user, char *dochdir)
@@ -53,7 +52,16 @@ diff -ru dcron-4.4.org/chuser.c dcron-4.4/chuser.c
  
 diff -ru dcron-4.4.org/crontab.c dcron-4.4/crontab.c
 --- dcron-4.4.org/crontab.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/crontab.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/crontab.c	2013-07-18 18:18:07.768535485 +0200
+@@ -88,7 +88,7 @@
+ 				break;
+ 			case 'c':
+ 				/* getopt guarantees optarg != 0 here */
+-				if (*optarg != 0 && getuid() == geteuid()) {
++				if (*optarg != 0) {
+ 					CDir = optarg;
+ 				} else {
+ 					printlogf(0, "-c option: superuser only");
 @@ -316,9 +316,6 @@
  
  	close(filedes[0]);
@@ -75,7 +83,7 @@ diff -ru dcron-4.4.org/crontab.c dcron-4.4/crontab.c
  				ptr = PATH_VI;
 diff -ru dcron-4.4.org/job.c dcron-4.4/job.c
 --- dcron-4.4.org/job.c	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/job.c	2011-04-01 11:19:19.000000000 +0200
++++ dcron-4.4/job.c	2013-07-18 18:17:16.342147418 +0200
 @@ -62,14 +62,6 @@
  		 * Change running state to the user in question
  		 */
@@ -108,7 +116,7 @@ diff -ru dcron-4.4.org/job.c dcron-4.4/job.c
  		/*
 diff -ru dcron-4.4.org/Makefile dcron-4.4/Makefile
 --- dcron-4.4.org/Makefile	2010-01-18 16:27:31.000000000 +0100
-+++ dcron-4.4/Makefile	2011-04-01 11:19:35.000000000 +0200
++++ dcron-4.4/Makefile	2013-07-18 18:17:16.342147418 +0200
 @@ -3,7 +3,6 @@
  
  # these variables can be configured by e.g. `make SCRONTABS=/different/path`

component/dcron/dcron-hooks.py

+import os
+import shutil
+def post_make_hook(options, buildout):
+  crontab_path = os.path.join(options['location'], 'bin', 'crontab')
+  os.chmod(crontab_path, 0750)

component/dropbear/buildout.cfg

@@ -7,9 +7,10 @@
 [buildout]
 extends = 
   ../zlib/buildout.cfg
+  ../coreutils/buildout.cfg
 
 parts =
-  dropbear
+  dropbear-output
 
 [dropbear-userspace-patch]
 recipe = hexagonit.recipe.download
@@ -63,3 +64,12 @@ patches=
 
 patch-options=
   -p1
+
+[dropbear-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:ssh} -a -x ${:keygen}
+ssh = ${dropbear:location}/bin/dbclient
+keygen = ${dropbear:location}/bin/dropbearkey

component/ffmpeg/buildout.cfg

@@ -2,6 +2,7 @@
 extends = 
   ../bzip2/buildout.cfg
   ../libpng/buildout.cfg
+  ../patch/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
 
@@ -28,16 +29,25 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig
 
+[libtheora-png_sizeof.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = libtheora-png_sizeof.patch
+md5sum = eaa1454081b50f05b59495a12f52b0d5
+download-only = true
+
 [libtheora]
 recipe = slapos.recipe.cmmi
 url = http://downloads.xiph.org/releases/theora/libtheora-1.1.1.tar.bz2
 md5sum = 292ab65cedd5021d6b7ddd117e07cd8e
 depends =
   ${libpng:so_version}
+patches = ${libtheora-png_sizeof.patch:location}/${libtheora-png_sizeof.patch:filename}
+patch-options = -p1
 configure-options =
   --disable-static
 environment = 
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${libogg:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libvorbis:location}/lib/pkgconfig
 
 [yasm]

component/ffmpeg/libtheora-png_sizeof.patch

+--- libtheora-1.1.1/examples/png2theora.c.orig        2009-08-23 03:14:04.000000000 +0900
++++ libtheora-1.1.1/examples/png2theora.c     2013-07-16 12:40:07.629087870 +0900
+@@ -462,9 +462,9 @@
+   png_set_strip_alpha(png_ptr);
+ 
+   row_data = (png_bytep)png_malloc(png_ptr,
+-    3*height*width*png_sizeof(*row_data));
++    3*height*width*sizeof(*row_data));
+   row_pointers = (png_bytep *)png_malloc(png_ptr,
+-    height*png_sizeof(*row_pointers));
++    height*sizeof(*row_pointers));
+   for(y = 0; y < height; y++) {
+     row_pointers[y] = row_data + y*(3*width);
+   }

component/gcc/buildout.cfg

@@ -33,7 +33,7 @@ environment =
 [ppl]
 recipe = slapos.recipe.cmmi
 # we should use version 0.10.x for gcc-4.5
-url = http://www.cs.unipr.it/ppl/Download/ftp/releases/0.10.2/ppl-0.10.2.tar.bz2
+url = http://bugseng.com/products/ppl/download/ftp/releases/0.10.2/ppl-0.10.2.tar.bz2
 md5sum = 5667111f53150618b0fa522ffc53fc3e
 configure-options =
   --with-libgmp-prefix=${gmp:location}
@@ -85,9 +85,6 @@ patches =
   ${gcc-multiarch.patch:location}/${gcc-multiarch.patch:filename}
 patch-options = -p2
 configure-command = make clean \\; make distclean \\; ./configure
-# GMP does not correctly detect achitecture so it have to be given
-# as slapos.recipe.cmmi is using shell expansion in subproceses
-# backticks are working
 configure-options =
   --disable-bootstrap
   --enable-languages="c,c++"
@@ -97,12 +94,30 @@ configure-options =
   --with-mpc=${mpc:location}
   --with-ppl=${ppl:location}
   --with-cloog=${cloog-ppl:location}
-  --with-ecj-jar=${ecj:location}/${ecj:filename}
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
-
 environment =
   LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
-  PATH=${zip:location}/bin:%(PATH)s
+# make install does not work when several core are used
+make-targets = install -j1
+
+[gcc-minimal]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/gcc/gcc-4.5.4/gcc-core-4.5.4.tar.bz2
+md5sum = ca62e442629a9a7710f5d797bf1b521c
+patches =
+  ${gcc-multiarch.patch:location}/${gcc-multiarch.patch:filename}
+patch-options = -p2
+configure-options =
+  --disable-bootstrap
+  --enable-languages=c
+  --disable-multilib
+  --with-gmp=${gmp:location}
+  --with-mpfr=${mpfr:location}
+  --with-mpc=${mpc:location}
+  --without-ppl
+  --without-cloog
+environment =
+  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib
 # make install does not work when several core are used
 make-targets = install -j1
 

component/git/buildout.cfg

@@ -4,6 +4,7 @@
 [buildout]
 extends =
   ../curl/buildout.cfg
+  ../gettext/buildout.cfg
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
   ../zlib/buildout.cfg

component/glib/buildout.cfg

@@ -17,3 +17,7 @@ environment =
   CPPFLAGS=-I${zlib:location}/include -I${gettext:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
   PATH=${gettext:location}/bin:%(PATH)s
+
+[x86-cygwin-glib]
+patches =
+  ${:_profile_base_location_}/cygwin-glib-2.28.8-src.patch

component/glib/cygwin-glib-2.28.8-src.patch

+--- glib/gstrfuncs.c~	2012-12-30 14:51:30.000000000 +0800
++++ glib/gstrfuncs.c	2012-12-30 14:51:50.203125000 +0800
+@@ -1423,7 +1423,7 @@
+ #ifdef HAVE_STRSIGNAL
+   const char *msg_locale;
+ 
+-#if defined(G_OS_BEOS) || defined(G_WITH_CYGWIN)
++#if defined(G_OS_BEOS) 
+ extern const char *strsignal(int);
+ #else
+   /* this is declared differently (const) in string.h on BeOS */

component/gnutls/buildout.cfg

 [buildout]
 extends =
-  ../readline/buildout.cfg
+  ../gmp/buildout.cfg
+  ../nettle/buildout.cfg
   ../ncurses/buildout.cfg
+  ../readline/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = gnutls
@@ -22,14 +24,13 @@ environment =
   LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
 
 [gnutls]
-# XXX-Cedric : update to latest gnutls
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2
-md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
+md5sum = e0cba4ddd923420026ff9739b3bc069a
 configure-options =
   --with-libgcrypt-prefix=${gcrypt:location}
   --disable-static
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
-  LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
-  PKG_CONFIG=${zlib:location}/lib/pkgconfig
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
+  LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
+

component/grep/buildout.cfg

 [buildout]
 extends =
   ../pcre/buildout.cfg
+  ../coreutils/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
-  grep
+  grep-output
 
 [grep]
 recipe = slapos.recipe.cmmi
@@ -13,3 +14,11 @@ environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include
   LDFLAGS=-L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib
+
+[grep-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:grep}
+grep = ${grep:location}/bin/grep

component/mariadb/buildout.cfg

@@ -25,10 +25,10 @@ download-only = true
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 5.5.31
+version = 5.5.32
 revision = 1
 url = http://downloads.askmonty.org/f/mariadb-${:version}/kvm-tarbake-jaunty-x86/mariadb-${:version}.tar.gz/from/http://ftp.osuosl.org/pub/mariadb
-md5sum = 3fe756bc76f0e7a3af2757e48ce0f3f4
+md5sum = 565c2dce6a2fb027c9d0ffbae4934135
 # compile directory is required to build mysql plugins.
 keep-compile-dir = true
 patch-options = -p0

component/nettle/buildout.cfg

+[buildout]
+extends =
+  ../gmp/buildout.cfg
+  ../m4/buildout.cfg
+
+[nettle-lib-location.patch]
+recipe = hexagonit.recipe.download
+download-only = true
+filename = ${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 41dd0ce2a73487929bdc637b75dd62c9
+
+[nettle]
+recipe = slapos.recipe.cmmi
+url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
+md5sum = 003d5147911317931dd453520eb234a5
+patches =
+  ${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
+configure-option =
+  --disable-static
+  --disable-assembler
+  --disable-openssl
+environment =
+  PATH=${m4:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gmp:location}/include
+  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/nettle/nettle-lib-location.patch

+--- configure.orig	2013-07-05 15:37:28.000000000 +0200
++++ configure	2013-07-05 15:47:48.000000000 +0200
+@@ -4680,52 +4680,6 @@
+ if test "x$ABI" != xstandard ; then
+   { $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
+ $as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
+-  if test "$libdir" = '${exec_prefix}/lib' ; then
+-    # Try setting a better default
+-    case "$host_cpu:$host_os:$ABI" in
+-      *:solaris*:32|*:sunos*:32)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *:solaris*:64|*:sunos*:64)
+-	libdir='${exec_prefix}/lib/64'
+-	;;
+-      # Linux conventions are a mess... According to the Linux File
+-      # Hierarchy Standard, all architectures except IA64 puts 32-bit
+-      # libraries in lib, and 64-bit in lib64. Some distributions,
+-      # e.g., Fedora and Gentoo, adhere to this standard, while at
+-      # least Debian has decided to put 64-bit libraries in lib and
+-      # 32-bit libraries in lib32.
+-
+-      # We try to figure out the convention, except if we're cross
+-      # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
+-      # appears to not be a symlink to a different name.
+-      *:linux*:32|*:linux*:64)
+-	if test "$cross_compiling" = yes ; then
+-          { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
+-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; 	else
+-	  # The dash builtin pwd tries to be "helpful" and remember
+-	  # symlink names. Use -P option, and hope it's portable enough.
+-	  test -d /usr/lib${ABI} \
+-	    && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
+-	    && libdir='${exec_prefix}/'"lib${ABI}"
+-	fi
+-	;;
+-      # On freebsd, it seems 32-bit libraries are in lib32,
+-      # and 64-bit in lib. Don't know about "kfreebsd", does
+-      # it follow the Linux fhs conventions?
+-      *:freebsd*:32)
+-	libdir='${exec_prefix}/lib32'
+-	;;
+-      *:freebsd*:64)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *)
+-        { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
+-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
+-    esac
+-    { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
+-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
+-  fi
+ fi
+ 
+ # Select assembler code

component/nginx/buildout.cfg

@@ -6,8 +6,9 @@ extends =
 
 [nginx]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.2.7.tar.gz
-md5sum = d252f5c689a14a668e241c744ccf5f06
+#url = http://nginx.org/download/nginx-1.2.7.tar.gz
+url = http://nginx.org/download/nginx-1.4.2.tar.gz
+#md5sum = d252f5c689a14a668e241c744ccf5f06
 configure-options=
   --with-ipv6
   --with-http_ssl_module

component/nodejs/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 [nodejs-0.8]
 # Server-side Javascript.
 recipe = slapos.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
-md5sum = 284fd2c7578064c339d9cf6a3a475ac7
+url = http://nodejs.org/dist/v0.8.23/node-v0.8.23.tar.gz
+md5sum = 22fe54ac365f52d3d80ecf748e7323d5
 configure-options =
   --openssl-includes=${openssl:location}/include
   --openssl-libpath=${openssl:location}/lib

component/openssl/buildout.cfg

@@ -51,4 +51,4 @@ configure-options =
 make-options =
   -j1
 make-targets =
-  install && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
+  all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true

component/popt/buildout.cfg

@@ -8,3 +8,6 @@ url = ftp://anduin.linuxfromscratch.org/BLFS/svn/p/popt-1.16.tar.gz
 md5sum = 3743beefa3dd6247a73f8f7a32c14c33
 configure-options =
   --disable-static
+
+[x86-cygwin-popt]
+patches = ${:_profile_base_location_}/cygwin-popt-1.16-src.patch
\ No newline at end of file

component/popt/cygwin-popt-1.16-src.patch

+--- poptconfig.c~	2011-09-10 22:49:30.802250000 +0800
++++ poptconfig.c	2011-09-10 23:46:30.572048000 +0800
+@@ -45,8 +45,9 @@ extern int glob_pattern_p (const char *_
+ #if !defined(__GLIBC__)
+ /* Return nonzero if PATTERN contains any metacharacters.
+    Metacharacters can be quoted with backslashes if QUOTE is nonzero.  */
++#define glob_pattern_p glob_pattern_p_s
+ static int
+-glob_pattern_p (const char * pattern, int quote)
++glob_pattern_p_s(const char * pattern, int quote)
+ 	/*@*/
+ {
+     const char * p;

component/python-openssl/buildout.cfg

@@ -8,6 +8,8 @@ parts =
 [python-openssl]
 recipe = zc.recipe.egg:custom
 egg = pyOpenSSL
+include-dirs =
+  ${openssl:location}/include/
 library-dirs =
   ${openssl:location}/lib/
 rpath =

component/qemu-kvm/buildout.cfg

@@ -3,22 +3,26 @@ extends =
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
+  ../../component/pkgconfig/buildout.cfg
   ../../component/xorg/buildout.cfg
   ../../component/zlib/buildout.cfg
 
-[kvm]
-# Backward compatibility
-<= qemu-kvm
+# XXX Change all reference to kvm section to qemu section, then
+# use qemu as main name section.
+[qemu]
+<= kvm
 
-[qemu-kvm]
+[kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.4.1.tar.bz2
-md5sum = eb2d696956324722b5ecfa46e41f9a75
+url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
+md5sum = b56e73bdcfdb214d5c68e13111aca96f
 depends =
   ${libpng:so_version}
 configure-options =
-  --target-list=""
+  --target-list=x86_64-softmmu
+  --enable-system
+  --with-system-pixman
   --disable-sdl
   --disable-xen
   --enable-vnc-tls
@@ -36,4 +40,27 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
+  LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
+
+# The following is only available in buildout2, which we don't use yet.
+[kvm-bits64]
+configure-options =
+  --target-list=x86_64-softmmu
+  ${kvm:configure-options}
+
+[kvm-bits32]
+configure-options =
+  --target-list=i386-softmmu
+  ${kvm:configure-options}
+
+
+[debian-amd64-netinst.iso]
+# Download the installer of Debian 7 (Wheezy)
+recipe = slapos.recipe.download
+url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
+filename = ${:_buildout_section_name_}
+md5sum = 80f498a1f9daa76bc911ae13692e4495
+download-only = true
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 

component/rdiff-backup/buildout.cfg

 [buildout]
 extends =
   ../librsync/buildout.cfg
+  ../coreutils/buildout.cfg
 parts =
-  rdiff-backup
+  rdiff-backup-output
 
 [rdiff-backup-build]
 recipe = zc.recipe.egg:custom
@@ -24,3 +25,11 @@ eggs =
 entry-points =
   rdiff-backup=rdiff_backup.Main:Main
 arguments = sys.argv[1:]
+
+[rdiff-backup-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:rdiff-backup}
+rdiff-backup = ${buildout:directory}/bin/rdiff-backup

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.78.2.dev'
+version = '0.78.5'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -165,8 +165,6 @@ setup(name=name,
           'publish.serialised = slapos.recipe.publish:Serialised',
           'publishsection = slapos.recipe.publish:PublishSection',
           'publishurl = slapos.recipe.publishurl:Recipe',
-          'pwgen = slapos.recipe.pwgen:Recipe',
-          'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
           'redis.server = slapos.recipe.redis:Recipe',
           'request = slapos.recipe.request:Recipe',
           'request.serialised = slapos.recipe.request:Serialised',
@@ -188,6 +186,8 @@ setup(name=name,
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
           'slaprunner.test = slapos.recipe.slaprunner:Test',
+          'slaprunner.export = slapos.recipe.slaprunner.backup:ExportRecipe',
+          'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

slapos/recipe/README.generic_varnish.txt

@@ -12,15 +12,14 @@ On slap console, you can instanciate varnish like this:
 instance = request(
   software_type='varnish',
   partition_parameter_kw={
-     'tidstorage-url':'http://[your tidstrage address]:your tid strage port',
+     'backend-url':'https://[your_backend_address]:your_backend_port',
      'web-checker-frontend-url':'http://www.example.com',
      'web-checker-mail-address':'web-checker-result@example.com',
      'web-checker-smtp-host':'mail.example.com',
   }
 )
 
-tidstrage-url is the backend url that varnish will cache. It is expected that
-the backend is created by tidstorage recipe.
+backend-url is the backend url that varnish will cache.
 
 web-checker-frontend-url is the entry-point-url that web checker will check
 the HTTP headers of all the pages in the web site.

slapos/recipe/README.kvm.txt

@@ -31,10 +31,10 @@ KVM with Remote and gzipped Image
     gzip = true
     
     # Use -hda instead -drive arg
-    # Default is drive (see Options bellow)
+    # Default is drive (see Options below)
     image_type = hda
 
-    ### Common Configuration bellow. ###
+    ### Common Configuration below. ###
     
     # VNC is optional
     kvm_vnc = <SOME-IP>:<VNC-DISPLAY>

slapos/recipe/addresiliency/renamer.py

@@ -59,8 +59,6 @@ class Renamer(object):
 
         cp_broken.rename(new_name=broken_new_ref)
 
-        cp_broken.stopped()
-
         log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
 
         # update name (and later, software type) for the partition that will take over

slapos/recipe/check_url_available/__init__.py

@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
       'url': self.options['url'],
       'shell_path': self.options['dash_path'],
       'curl_path': self.options['curl_path'],
+      'check_secure': self.options.get('check-secure', 0)
     }
 
     # XXX-Cedric in this script, curl won't check certificate

slapos/recipe/check_url_available/template/check_url.in

@@ -31,6 +31,13 @@ if [ $CODE -eq 000 ]; then
   exit 1
 fi
 
+if [ %(check_secure)s -eq 1 ]; then
+  if [ $CODE -eq 401 ]; then
+    echo "$URL is protected (returned $CODE)." >&2
+    exit 0
+  fi
+fi
+
 if ! [ $CODE -eq 200 ]; then
   echo "$URL is not available (returned $CODE)." >&2
   exit 2

slapos/recipe/davstorage/__init__.py

@@ -27,6 +27,8 @@
 import subprocess
 import httplib
 import base64
+import os
+import shutil
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -50,13 +52,29 @@ class Recipe(GenericBaseRecipe):
                            user, password
                           ])
 
+    htdocs_location = self.options['htdocs']
+    if not (os.path.exists(htdocs_location) and os.listdir(htdocs_location)):
+      try:
+        os.rmdir(htdocs_location)
+      except:
+        pass
+      shutil.copytree(self.options['source'], htdocs_location)
+
+    # Install php.ini
+    php_ini = self.createFile(os.path.join(self.options['php-ini-dir'],
+                                             'php.ini'),
+                                self.substituteTemplate(self.getTemplateFilename('php.ini.in'),
+                                dict(tmp_directory=self.options['tmp-dir']))
+                )
+    path_list.append(php_ini)
     
     apache_config = dict(
       pid_file=self.options['pid-file'],
       lock_file=self.options['lock-file'],
       davlock_db=self.options['davdb-lock'],
       ip=self.options['ip'],
-      port=self.options['port'],
+      port_webdav=self.options['port_webdav'],
+      port_ajax=self.options['port_ajax'],
       error_log=self.options['error-log'],
       access_log=self.options['access-log'],
       document_root=self.options['htdocs'],
@@ -67,6 +85,7 @@ class Recipe(GenericBaseRecipe):
       htpasswd_file=htpasswd_file,
       ssl_certificate=self.options['cert-file'],
       ssl_key=self.options['key-file'],
+      php_ini_dir=self.options['php-ini-dir']
     )
 
     # Create logfiles
@@ -86,7 +105,7 @@ class Recipe(GenericBaseRecipe):
 
     promise = self.createPythonScript(self.options['promise'],
       __name__ + '.promise',
-      dict(host=self.options['ip'], port=int(self.options['port']),
+      dict(host=self.options['ip'], port=int(self.options['port_webdav']),
            user=self.options['user'], password=self.options['password'])
                                      )
     path_list.append(promise)

slapos/recipe/davstorage/template/httpd.conf.in

 ServerRoot "%(server_root)s"
 
-Listen [%(ip)s]:%(port)s
+Listen [%(ip)s]:%(port_webdav)s
+Listen [%(ip)s]:%(port_ajax)s
+
+NameVirtualHost [%(ip)s]:%(port_webdav)s
+NameVirtualHost [%(ip)s]:%(port_ajax)s
 
 # Needed modules
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule unixd_module "%(modules_dir)s/mod_unixd.so"
+LoadModule access_compat_module "%(modules_dir)s/mod_access_compat.so"
+LoadModule authn_core_module "%(modules_dir)s/mod_authn_core.so"
+LoadModule authz_core_module "%(modules_dir)s/mod_authz_core.so"
 LoadModule authn_file_module "%(modules_dir)s/mod_authn_file.so"
 LoadModule authz_host_module "%(modules_dir)s/mod_authz_host.so"
 LoadModule authz_user_module "%(modules_dir)s/mod_authz_user.so"
 LoadModule auth_basic_module "%(modules_dir)s/mod_auth_basic.so"
-LoadModule auth_digest_module "%(modules_dir)s/mod_auth_digest.so"
+# Comment auth_digest since we don't use it
+#LoadModule auth_digest_module "%(modules_dir)s/mod_auth_digest.so"
 LoadModule log_config_module "%(modules_dir)s/mod_log_config.so"
 LoadModule headers_module "%(modules_dir)s/mod_headers.so"
 LoadModule setenvif_module "%(modules_dir)s/mod_setenvif.so"
-LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+LoadModule socache_shmcb_module "%(modules_dir)s/mod_socache_shmcb.so"
 LoadModule ssl_module "%(modules_dir)s/mod_ssl.so"
 LoadModule mime_module "%(modules_dir)s/mod_mime.so"
 LoadModule dav_module "%(modules_dir)s/mod_dav.so"
 LoadModule dav_fs_module "%(modules_dir)s/mod_dav_fs.so"
 LoadModule dir_module "%(modules_dir)s/mod_dir.so"
+LoadModule php5_module "%(modules_dir)s/libphp5.so"
 
 ServerAdmin %(email_address)s
 
 # Quiet Server header (if not, Apache give its life history)
 # It's safer
 ServerTokens ProductOnly
+PidFile "%(pid_file)s"
+PHPINIDir "%(php_ini_dir)s"
+
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+
+<VirtualHost [%(ip)s]:%(port_ajax)s>
+  #ServerName www.example.com
   
+  # Directory protection
+  <Directory />
+      Options FollowSymLinks
+      AllowOverride None
+      Require all denied
+  </Directory>
+  
+  <Directory %(document_root)s>
+    Options FollowSymLinks
+    AllowOverride All
+    Require all granted
+  </Directory>
+ 
+  DocumentRoot "%(document_root)s"
+  DirectoryIndex index.html index.php
+
+SSLEngine on
+SSLCertificateFile "%(ssl_certificate)s"
+SSLCertificateKeyFile "%(ssl_key)s"
+
+</VirtualHost>
+
+<VirtualHost [%(ip)s]:%(port_webdav)s>
 DocumentRoot "%(document_root)s"
-PidFile "%(pid_file)s"
 DavLockDB "%(davlock_db)s"
 
 <Directory />
@@ -66,6 +103,12 @@ DavLockDB "%(davlock_db)s"
 
 </Directory>
 
+SSLEngine on
+SSLCertificateFile "%(ssl_certificate)s"
+SSLCertificateKeyFile "%(ssl_key)s"
+
+</VirtualHost>
+
 ErrorLog "%(error_log)s"
 LogLevel warn
 
@@ -77,9 +120,5 @@ DefaultType text/plain
 TypesConfig "%(mime_types)s"
 AddType application/x-compress .Z
 AddType application/x-gzip .gz .tgz
-
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
+AddType application/x-httpd-php .php .phtml .php5 .php4
+AddType application/x-httpd-php-source .phps
\ No newline at end of file
-SSLEngine on
-SSLCertificateFile "%(ssl_certificate)s"
-SSLCertificateKeyFile "%(ssl_key)s"

slapos/recipe/davstorage/template/php.ini.in

+[PHP]
+engine = On
+safe_mode = Off
+expose_php = Off
+error_reporting = E_ALL & ~(E_DEPRECATED|E_NOTICE|E_WARNING)
+display_errors = On
+display_startup_errors = Off
+log_errors = On
+log_errors_max_len = 1024
+ignore_repeated_errors = Off
+ignore_repeated_source = Off
+session.save_path = "%(tmp_directory)s"
+session.auto_start = 0
+date.timezone = Europe/Paris
+file_uploads = On
+upload_max_filesize = 8M
+post_max_size = 8M
+magic_quotes_gpc=Off
+output_buffering=Off
\ No newline at end of file

slapos/recipe/generatepassword.py

@@ -26,27 +26,69 @@
 #
 ##############################################################################
 
-import binascii
+import errno
 import os
+import random
+import string
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+def generatePassword(length):
+  return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
 
 
-class Recipe(GenericBaseRecipe):
+class Recipe(object):
+  """Generate a password that is only composed of lowercase letters
+
+    This recipe only makes sure that ${:passwd} does not end up in `.installed`
+    file, which is world-readable by default. So be careful not to spread it
+    throughout the buildout configuration by referencing it directly: see
+    recipes like slapos.recipe.template:jinja2 to safely process the password.
+
+    Options:
+    - bytes: password length (default: 8 characters)
+    - storage-path: plain-text persistent storage for password,
+                    that can only be accessed by the user
+      (default: ${buildout:parts-directory}/${:_buildout_section_name_})
+  """
 
   def __init__(self, buildout, name, options):
-    if os.path.exists(options['storage-path']):
-      open_file = open(options['storage-path'], 'r')
-      options['passwd'] = open_file.read()
-      open_file.close()
+    options_get = options.get
+    try:
+      self.storage_path = options['storage-path']
+    except KeyError:
+      self.storage_path = options['storage-path'] = os.path.join(
+        buildout['buildout']['parts-directory'], name)
+    try:
+      with open(self.storage_path) as f:
+        passwd = f.read()
+    except IOError, e:
+      if e.errno != errno.ENOENT:
+        raise
+      passwd = None
+    if not passwd:
+      passwd = self.generatePassword(int(options_get('bytes', '8')))
+      self.update = self.install
+    self.passwd = passwd
+    # Password must not go into .installed file, for 2 reasons:
+    # security of course but also to prevent buildout to always reinstall.
+    options.get = lambda option, *args, **kw: passwd \
+      if option == 'passwd' else options_get(option, *args, **kw)
 
-    if options.get('passwd', '') == '':
-      options['passwd'] = binascii.hexlify(os.urandom(
-                            int(options.get('bytes', '24'))))
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+  generatePassword = staticmethod(generatePassword)
 
   def install(self):
-    with open(self.options['storage-path'], 'w') as fout:
-      fout.write(self.options['passwd'])
-    return [self.options['storage-path']]
+    if self.storage_path:
+      try:
+        os.unlink(self.storage_path)
+      except OSError, e:
+        if e.errno != errno.ENOENT:
+          raise
+      fd = os.open(self.storage_path,
+        os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
+      try:
+        os.write(fd, self.passwd)
+      finally:
+        os.close(fd)
+    return self.storage_path
 
+  def update(self):
+    return ()

slapos/recipe/generic_varnish/__init__.py

@@ -37,7 +37,12 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     ip = self.options['ip']
-    backend_url = self.parameter_dict['tidstorage-url']
+    backend_url = self.options.get('backend-url',
+      # BBB: Peeking in partition parameters directly. Eew.
+      self.parameter_dict.get('backend-url',
+        self.parameter_dict.get('tidstorage-url') # BBB
+      )
+    )
     backend_server, backend_port = self._getBackendServer(backend_url)
     path_list = []
     if backend_url.startswith('https://'):
@@ -70,6 +75,7 @@ class Recipe(GenericSlapRecipe):
       varnishd_pid_file=self.options['pid-file'],
       varnish_instance_name=self.options['varnish-instance-name'],
       varnish_data=self.options['varnish-data'],
+      gcc_location=self.options['gcc-location'],
       shell_path=self.options['shell-path'],
       vcl_file=self.options['vcl-file'],
       backend_port=backend_port,

slapos/recipe/generic_varnish/template/varnishd.in

@@ -4,11 +4,12 @@ DAEMON_OPTS="-F \
              -a %(varnish_ip)s:%(varnishd_server_port)s \
              -T %(varnish_ip)s:%(varnishd_manager_port)s \
              -t 0 \
+             -p nuke_limit=500 \
              -n %(varnish_instance_name)s \
              -f %(vcl_file)s \
              -s file,%(varnish_data)s/varnish_storage.bin,1G"
 
 PIDFILE=%(varnishd_pid_file)s
 # exporting PATH here so that we will pass the PATH variable to the subprocess
-export PATH
+export PATH="%(gcc_location)s:$PATH"
 exec %(varnishd_binary)s -P ${PIDFILE} ${DAEMON_OPTS} 2>&1

slapos/recipe/generic_zope/template/zope.conf.in

@@ -25,6 +25,9 @@ rest-input-encoding utf-8
 rest-output-encoding utf-8
 default-zpublisher-encoding utf-8
 
+# Disable ownership checking to execute codes generated by alarm
+skip-ownership-checking on
+
 # Temporary storage database (for sessions)
 <zodb_db temporary>
      <temporarystorage>

slapos/recipe/generic_zope_zeo_client/template/zope.conf.in

@@ -25,6 +25,9 @@ rest-input-encoding utf-8
 rest-output-encoding utf-8
 default-zpublisher-encoding utf-8
 
+# Disable ownership checking to execute codes generated by alarm
+skip-ownership-checking on
+
 # Temporary storage database (for sessions)
 <zodb_db temporary>
      <temporarystorage>

slapos/recipe/haproxy/__init__.py

@@ -87,7 +87,11 @@ class Recipe(GenericBaseRecipe):
       'haproxy-listen-snippet.cfg.in')
     server_snippet = ""
     ip = self.options['ip']
-    server_check_path = self.options['server-check-path']
+    server_check_path = self.options.get('server-check-path', None)
+    if server_check_path:
+      httpchk = 'option httpchk GET %s' % server_check_path
+    else:
+      httpchk = ''
     # FIXME: maxconn must be provided per-backend, not globally
     maxconn = self.options['maxconn']
     i = 0
@@ -97,7 +101,7 @@ class Recipe(GenericBaseRecipe):
           'name': name,
           'ip': ip,
           'port': port,
-          'server_check_path': server_check_path,
+          'httpchk': httpchk,
         })
       for address in backend_list:
         i += 1

slapos/recipe/haproxy/template/haproxy-listen-snippet.cfg.in

 listen %(name)s %(ip)s:%(port)s
   cookie  SERVERID insert
   balance roundrobin
-  option httpchk GET %(server_check_path)s
+  %(httpchk)s
   stats uri /haproxy
   stats realm Global\ statistics

slapos/recipe/kvm/__init__.py

@@ -61,7 +61,8 @@ class Recipe(GenericBaseRecipe):
       shell_path=self.options['shell-path'],
       qemu_path=self.options['qemu-path'],
       qemu_img_path=self.options['qemu-img-path'],
-      vnc_passwd=self.options['passwd']
+      vnc_passwd=self.options['passwd'],
+      default_disk_image=self.options['default-disk-image'],
     )
 
     # Runners

slapos/recipe/kvm/template/kvm_run.in

@@ -7,6 +7,9 @@ import os
 import socket
 import subprocess
 
+# XXX: give all of this through parameter, don't use this as template
+default_disk_image = '%(default_disk_image)s'
+
 def getSocketStatus(host, port):
   s = None
   for res in socket.getaddrinfo(host, port,
@@ -26,7 +29,8 @@ def getSocketStatus(host, port):
     break
   return s
 
-# create disk if doesn't exist
+# Create disk if doesn't exist
+# XXX: move to Buildout profile
 disk_path = '%(disk_path)s'
 if not os.path.exists(disk_path):
   subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
@@ -58,4 +62,10 @@ for nbd_ip, nbd_port in (
           '-drive',
           'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
 
+# If no NBD is specified/available: use internal disk image
+else:
+  kvm_argument_list.extend([
+      '-drive', 'file=%%s,media=cdrom' %% default_disk_image
+  ])
+
 os.execv('%(qemu_path)s', kvm_argument_list)

slapos/recipe/librecipe/generic.py

@@ -183,17 +183,13 @@ class GenericBaseRecipe(object):
         'template/%s' % template_name)
 
   def generatePassword(self, len_=32):
-    """
-    The purpose of this method is to generate a password which doesn't change
-    from one execution to the next, so the generated password doesn't change
-    on each slapgrid-cp execution.
-
-    Currently, it returns a hardcoded password because no decision has been
-    taken on where a generated password should be kept (so it is generated
-    once only).
-    """
-    # TODO: implement a real password generator which remember the last
-    # call.
+    # TODO: Consider having generate.password recipe inherit this class,
+    #       so that it can be easily inheritable.
+    #       In the long-term, it's probably better that passwords are provided
+    #       by software requesters, to avoid keeping unhashed secrets in
+    #       partitions when possible.
+    self.logger.warning("GenericBaseRecipe.generatePassword is deprecated."
+                        " Use generate.password recipe instead.")
     return "insecure"
 
   def isTrueValue(self, value):
@@ -247,7 +243,8 @@ class GenericBaseRecipe(object):
       destination = self.location
     if os.path.exists(destination):
         # leftovers from a previous failed attempt, removing it.
-        log.warning('Removing already existing directory %s' % destination)
+        self.logger.warning('Removing already existing directory %s',
+                            destination)
         shutil.rmtree(destination)
     os.mkdir(destination)
 

slapos/recipe/neoppod.py

@@ -50,8 +50,6 @@ class NeoBaseRecipe(GenericBaseRecipe):
       #'-n', options['name'],
       '-c', options['cluster'],
     ]
-    if options['verbose']:
-      option_list.append('-v')
     option_list.extend(self._getOptionList())
     return [self.createPythonScript(
       options['wrapper'],

slapos/recipe/publish.py

@@ -35,20 +35,22 @@ class Recipe(GenericSlapRecipe):
     publish_dict = dict()
     options = self.options.copy()
     del options['recipe']
-
+    slave_reference = options.pop('-slave-reference', None)
     for k, v in options.iteritems():
+      if k[:1] == '-':
+        continue
       publish_dict[k] = v
-    self._setConnectionDict(publish_dict)
+    self._setConnectionDict(publish_dict, slave_reference)
     return []
 
-  def _setConnectionDict(self, publish_dict):
-    return self.setConnectionDict(publish_dict)
+  def _setConnectionDict(self, publish_dict, slave_reference=None):
+    return self.setConnectionDict(publish_dict, slave_reference)
 
 SERIALISED_MAGIC_KEY = '_'
 
 class Serialised(Recipe):
-  def _setConnectionDict(self, publish_dict):
-    return super(Serialised, self)._setConnectionDict(wrap(publish_dict))
+  def _setConnectionDict(self, publish_dict, slave_reference=None):
+    return super(Serialised, self)._setConnectionDict(wrap(publish_dict), slave_reference)
 
 
 

slapos/recipe/pwgen.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import subprocess
-import os
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-class Recipe(GenericBaseRecipe):
-
-  def _options(self, options):
-    if not os.path.exists(self.options['file']):
-      password = subprocess.check_output([self.options['pwgen-binary'], '-1']).strip()
-      with open(self.options['file'], 'w') as password_file:
-        password_file.write(password)
-    else:
-      with open(self.options['file'], 'r') as password_file:
-        password = password_file.read()
-    options['password'] = password
-
-  def install(self):
-    os.chmod(self.options['file'], 0600)
-    return []
-
-class StablePasswordGeneratorRecipe(GenericBaseRecipe):
-  """
-  The purpose of this class is to generate a password which doesn't change
-  from one execution to the next (hence "stable"), so the generated password
-  doesn't change on each slapgrid-cp execution.
-
-  See GenericBaseRecipe.generatePassword .
-  """
-
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-
-  update = install = lambda self: []

slapos/recipe/slaprunner/__init__.py

@@ -40,8 +40,8 @@ class Recipe(GenericBaseRecipe):
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options.get('cloud9-url', '').strip()
     self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
-    # Set slaprunner access URL
-    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+    # Set slaprunner access URL, CLN Beware ipv6 access is made throught nginx
+    options['access-url'] = 'https://[%s]:%s' % (self.ipv6, self.runner_port)
 
   def install(self):
     path_list = []
@@ -62,7 +62,7 @@ class Recipe(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['run_dir'],
         log_dir=self.options['log_dir'],
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,
@@ -132,7 +132,7 @@ class Test(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['etc_dir'],
         log_dir=self.workdir,
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,

slapos/recipe/slaprunner/backup.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import textwrap
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an exporter script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """\
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                sync_element () {
+                  path=$1
+                  backup_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --safe-links --delete $element  $backup_path;
+                    fi
+                  done
+                }
+                sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db softwareLink
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh
+                if [ -d %(backup-directory)s/runner/software ]; then
+                  rm %(backup-directory)s/runner/software/*
+                fi
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+    """\
+    This recipe creates an importer script for using with the resilient stack.
+
+    Required options:
+        backup-directory
+            folder that will contain the dump file.
+        srv-directory
+            folder that contain the runner directory.
+        wrapper
+            full path of the exporter script to create.
+    """
+
+    def install(self):
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """\
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!%(shell-binary)s
+                umask 077
+                restore_element () {
+                  backup_path=$1
+                  restore_path=$2
+                  shift 2
+                  element_list=$*
+                  for element in $element_list
+                  do
+                    cd $backup_path;
+                    if [ -f $element ] || [ -d $element ]; then
+                       %(rsync-binary)s -avz --delete $backup_path/$element $restore_path;
+                    fi
+                  done
+                }
+                restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db softwareLink
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh
+                ifs=$IFS IFS=';'
+                read user pass remaining < %(etc-directory)s/.users
+                IFS=$ifs
+                %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
+                %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
+                rm -f login_cookie
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+

slapos/recipe/softwaretype.py

@@ -87,8 +87,6 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
-    self.logger.info('Deploying instance with software type %s' % \
-        software_type)
 
     # Raise if request software_type does not exist ...
     if software_type not in self.options:

slapos/recipe/web_checker/__init__.py

@@ -34,11 +34,15 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     path_list = []
+    try:
+      web_checker_mail_address = self.options['mail-address']
+      web_checker_smtp_host = self.options['smtp-host']
+      web_checker_frontend_url = self.options['frontend-url']
+    except KeyError:
+      # BBB
       web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
       web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
-    web_checker_frontend_url = self.parameter_dict.get(
-                                 'web-checker-frontend-url',
-                                 self.options['frontend-url'])
+      web_checker_frontend_url = self.parameter_dict['web-checker-frontend-url']
     web_checker_working_directory = \
       self.options['web-checker-working-directory']
     config = dict(

software/backupserver/instance-pullrdiffbackup.cfg.in

+{% set part_list = [] -%}
+{% set crontab_line_list = [] -%}
+###############################
+#
+# Instanciate dcron
+#
+###############################
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+# scripts = $${:etc}/run
+services = $${:etc}/service
+cron-entries = $${:etc}/cron.d
+cron-lines = $${:etc}/cron.lines
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+backup = $${:srv}/backup
+backupscript = $${:etc}/backup
+home = $${:etc}/home
+ssh = $${:home}/.ssh
+
+#################################
+# Cron service
+#################################
+[dcron-service]
+recipe = slapos.recipe.template
+url = ${template-dcron-service:output}
+output = $${directory:services}/crond
+mode = 0700
+logfile = $${directory:log}/crond.log
+
+#################################
+# Slave backup scripts and crontab
+#################################
+# Go throught slave list to set their configuration
+{% for slave_instance in slave_instance_list -%}
+{%   set slave_reference = slave_instance.get('slave_reference') -%}
+{%   set frequency = slave_instance.get('frequency', '') -%}
+{%   set hostname = slave_instance.get('hostname', '') -%}
+{%   set connection = slave_instance.get('connection', '') -%}
+{%   set include = slave_instance.get('include', '') -%}
+{%   set include_string = "' --include='".join(include.split(' ')) -%}
+{%   set exclude = slave_instance.get('exclude', '') -%}
+{%   set exclude_string = '' -%}
+{%   set sudo = slave_instance.get('sudo', 'False') -%}
+{%   set remote_schema = 'rdiff-backup --server --restrict-read-only / -- "$@"' -%}
+
+{%   if (exclude != '') -%}
+{%     set exclude_string = "' --exclude='".join(exclude.split(' ')) -%}
+{%     set exclude_string = "--exclude='" + exclude_string + "'" -%}
+{%   endif -%}
+
+{%   if (sudo == 'True') -%}
+{%     set remote_schema = 'sudo backupagent_rdiff-backup' -%}
+{%   endif -%}
+
+{%   if (frequency != '') and (hostname != '') and (connection != '') and (include != '') -%}
+
+[{{ slave_reference }}-backup-directory]
+recipe = slapos.cookbook:mkdirectory
+directory = $${directory:backup}/$${:_buildout_section_name_}
+
+[{{ slave_reference }}-backup-private_key]
+recipe = plone.recipe.command
+stop-on-error = true
+command = ${dropbear-output:keygen} -t $${:type} -s 2048 -f $${:key} 
+key = $${directory:ssh}/$${:_buildout_section_name_}
+location = $${:key}
+type = rsa
+
+[{{ slave_reference }}-backup-public_key]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = ${coreutils-output:rm} -f $${:key} && ${dropbear-output:keygen} -y -f {{ '$${' ~ slave_reference }}-backup-private_key:key} | ${grep-output:grep} {{ '$${' ~ slave_reference }}-backup-private_key:type} > $${:key}
+key = {{ '$${' ~ slave_reference }}-backup-private_key:key}.pub
+location = $${:key}
+
+[{{ slave_reference }}-backup-check-public_key]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = grep ssh-{{ '$${' ~ slave_reference }}-backup-private_key:type} {{ '$${' ~ slave_reference }}-backup-public_key:key}
+
+[{{ slave_reference }}-backup-read-public_key]
+recipe = slapos.cookbook:generate.password
+storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
+bytes = 8
+
+# Publish slave {{ slave_reference }} information
+[{{ slave_reference }}-backup-publish]
+recipe = slapos.cookbook:publish
+-slave-reference = {{ slave_reference }}
+authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:passwd}
+{%   do part_list.append("%s-backup-publish" % slave_reference) -%}
+
+[{{ slave_reference }}-backup-script]
+recipe = slapos.recipe.template
+url = ${template-backup-script:output}
+output = $${directory:backupscript}/$${:_buildout_section_name_}
+mode = 0700
+datadirectory = {{ '$${' ~ slave_reference }}-backup-directory:directory}
+sshkey = {{ '$${' ~ slave_reference }}-backup-private_key:key}
+connection = {{ connection }}
+hostname = {{ hostname }}
+include = {{ include_string }}
+exclude_string = {{ exclude_string }}
+remote_schema = {{ remote_schema }}
+
+[{{ slave_reference }}-backup-crontab-line]
+recipe = slapos.recipe.template
+url = ${template-crontab-line:output}
+output = $${directory:cron-lines}/$${:_buildout_section_name_}
+mode = 0600
+script = {{ '$${' ~ slave_reference }}-backup-script:output}
+frequency = {{ frequency }}
+{%   do crontab_line_list.append("$${%s-backup-crontab-line:output}" % slave_reference) -%}
+
+{%   endif -%}
+{% endfor -%}
+
+#################################
+# Generate crontab file
+#################################
+
+{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
+[activate-crontab-file]
+# XXX File is never removed
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = $${:command}
+command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
+
+
+# Add parts generated by template
+[buildout]
+parts =
+  dcron-service
+  activate-crontab-file
+{% for part in part_list -%}
+{{ '    %s' % part }}
+{% endfor -%}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true

software/backupserver/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[dynamic-template-pullrdiffbackup]
+recipe = slapos.recipe.template:jinja2
+template = ${template-pullrdiffbackup:output}
+rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
+filename = instance-pullrdiffbackup.cfg
+extensions = jinja2.ext.do
+context =
+    key slave_instance_list instance-parameter:slave-instance-list
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:pullrdiffbackup}
+# pullrdiffbackup = ${template-pullrdiffbackup:output}
+pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+# [slap-parameter]
+# slave-instance-list = []
+
+[instance-parameter]
+# Fetches parameters defined in SlapOS Master for this instance.
+# Always the same.
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}

software/backupserver/software.cfg

+[buildout]
+extends =
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+#   ../../component/nginx/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+#   ../../component/duplicity/buildout.cfg
+#   ../../component/git/buildout.cfg
+#   ../../component/subversion/buildout.cfg
+  ../../component/rsync/buildout.cfg
+  ../../component/dropbear/buildout.cfg
+  ../../component/grep/buildout.cfg
+#   ../../stack/flask.cfg
+  ../../stack/slapos.cfg
+
+parts =
+  rdiff-backup
+#   duplicity
+  dcron
+  logrotate
+#   nginx
+#   git
+#   subversion
+  rsync
+#   flask-egg
+  template
+  template-pullrdiffbackup
+  template-backup-script
+  template-crontab-line
+  slapos-cookbook
+
+[networkcache]
+# signature certificates of the following uploaders.
+#   Romain Courteaud
+#   Sebastien Robin
+#   Kazuhiko Shiozaki
+#   Cedric de Saint Martin
+#   Yingjie Xu
+#   Gabriel Monnerat
+#   Łukasz Nowak
+#   Test Agent (Automatic update from tests)
+#   Aurélien Calonne
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+  QUUGLQ==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+  BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+  DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+  YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+  XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+  L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+  /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+  LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+  FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+  R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+  hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV
+  BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL
+  BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH
+  vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk
+  3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso
+  67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE
+  RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v
+  PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj
+  z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL
+  KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY
+  vaZhjNYKWQf79l6zXfOvphzJ
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT
+  MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3
+  i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9
+  mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O
+  BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg
+  62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK
+  YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W
+  it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+
+  TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx
+  EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn
+  oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0
+  NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O
+  BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB
+  JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6
+  baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA
+  Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n
+  yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAMNZBmoIOXPBMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMTMyMCAXDTEyMDUwMjEyMDQyNloYDzIxMTIwNDA4MTIwNDI2WjAT
+  MREwDwYDVQQDDAhDT01QLTEzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  6peZQt1sAmMAmSG9BVxxcXm8x15kE9iAplmANYNQ7z2YO57c10jDtlYlwVfi/rct
+  xNUOKQtc8UQtV/fJWP0QT0GITdRz5X/TkWiojiFgkopza9/b1hXs5rltYByUGLhg
+  7JZ9dZGBihzPfn6U8ESAKiJzQP8Hyz/o81FPfuHCftsCAwEAAaNQME4wHQYDVR0O
+  BBYEFNuxsc77Z6/JSKPoyloHNm9zF9yqMB8GA1UdIwQYMBaAFNuxsc77Z6/JSKPo
+  yloHNm9zF9yqMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAl4hBaJy1
+  cgiNV2+Z5oNTrHgmzWvSY4duECOTBxeuIOnhql3vLlaQmo0p8Z4c13kTZq2s3nhd
+  Loe5mIHsjRVKvzB6SvIaFUYq/EzmHnqNdpIGkT/Mj7r/iUs61btTcGUCLsUiUeci
+  Vd0Ozh79JSRpkrdI8R/NRQ2XPHAo+29TT70=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
+  MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  o7aipd6MbnuGDeR1UJUjuMLQUariAyQ2l2ZDS6TfOwjHiPw/mhzkielgk73kqN7A
+  sUREx41eTcYCXzTq3WP3xCLE4LxLg1eIhd4nwNHj8H18xR9aP0AGjo4UFl5BOMa1
+  mwoyBt3VtfGtUmb8whpeJgHhqrPPxLoON+i6fIbXDaUCAwEAAaNQME4wHQYDVR0O
+  BBYEFEfjy3OopT2lOksKmKBNHTJE2hFlMB8GA1UdIwQYMBaAFEfjy3OopT2lOksK
+  mKBNHTJE2hFlMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAaNRx6YN2
+  M/p3R8/xS6zvH1EqJ3FFD7XeAQ52WuQnKSREzuw0dsw12ClxjcHiQEFioyTiTtjs
+  5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
+  x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB+DCCAWGgAwIBAgIJAKGd0vpks6T/MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCUNPTVAtMTU4NDAgFw0xMzA2MjAxMjE5MjBaGA8yMTEzMDUyNzEyMTkyMFow
+  FDESMBAGA1UEAwwJQ09NUC0xNTg0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+  gQDZTH9etPUC+wMZQ3UIiOwyyCfHsJ+7duCFYjuo1uZrhtDt/fp8qb8qK9ob+df3
+  EEYgA0IgI2j/9jNUEnKbc5+OrfKznzXjrlrH7zU8lKBVNCLzQuqBKRNajZ+UvO8R
+  nlqK2jZCXP/p3HXDYUTEwIR5W3tVCEn/Vda4upTLcPVE5wIDAQABo1AwTjAdBgNV
+  HQ4EFgQU7KXaNDheQWoy5uOU01tn1M5vNkEwHwYDVR0jBBgwFoAU7KXaNDheQWoy
+  5uOU01tn1M5vNkEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQASmqCU
+  Znbvu6izdicvjuE3aKnBa7G++Fdp2bdne5VCwVbVLYCQWatB+n4crKqGdnVply/u
+  +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
+  m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
+  -----END CERTIFICATE-----
+
+##########################################################
+# Service startup scripts and configuration files
+##########################################################
+[template-dcron-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-dcron-service.sh.in
+md5sum = 1372441dac23e4fa7d2dc773a74725ea
+output = ${buildout:directory}/template-dcron-service.sh.in
+mode = 0644
+
+[template-backup-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-backup-script.sh.in
+md5sum = 19c3a72913cc285bf9745ab2f869018e
+output = ${buildout:directory}/template-backup-script.sh.in
+mode = 0644
+
+[template-crontab-line]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-crontab-line.in
+md5sum = 5cbd64f04da0601ba4286516a6161f5e
+output = ${buildout:directory}/template-crontab-line.in
+mode = 0644
+
+[template-crontab]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-crontab.in
+md5sum = 072be0fd04896880c931d44d8eabde37
+output = ${buildout:directory}/template-crontab.in
+mode = 0644
+
+##########################################################
+# Buildout instance.cfg templates
+##########################################################
+[template-pullrdiffbackup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
+md5sum = 62c236773dadecac11eb9a47dbca9351
+output = ${buildout:directory}/template-pullrdiffbackup.cfg
+mode = 0644
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 42021b325159dff29e4bd4e33b8ff2f3
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[versions]
+rdiff-backup = 1.0.5
+Jinja2 = 2.7
+MarkupSafe = 0.18
+Werkzeug = 0.9.1
+buildout-versions = 1.7
+gunicorn = 17.5
+itsdangerous = 0.22
+meld3 = 0.6.10
+plone.recipe.command = 1.1
+slapos.cookbook = 0.78.3
+slapos.recipe.build = 0.11.6
+slapos.recipe.cmmi = 0.1.1
+slapos.recipe.template = 2.4.2
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.35.1
+Flask = 0.10.1
+
+# Required by:
+# slapos.cookbook==0.78.1
+inotifyx = 0.2.0-1
+
+# Required by:
+# slapos.cookbook==0.78.1
+lock-file = 2.0
+
+# Required by:
+# slapos.cookbook==0.78.1
+# slapos.core==0.35.1
+# xml-marshaller==0.9.7
+lxml = 3.2.1
+
+# Required by:
+# slapos.cookbook==0.78.1
+netaddr = 0.7.10
+
+# Required by:
+# slapos.core==0.35.1
+netifaces = 0.8-1
+
+# Required by:
+# slapos.core==0.35.1
+pyflakes = 0.7.3
+
+# Required by:
+# slapos.cookbook==0.78.1
+pytz = 2013b
+
+# Required by:
+# slapos.cookbook==0.78.1
+# slapos.core==0.35.1
+# zc.buildout==1.6.0-dev-SlapOS-010
+# zc.recipe.egg==1.3.2
+setuptools = 0.9.5
+
+# Required by:
+# slapos.cookbook==0.78.1
+slapos.core = 0.35.1
+# Required by:
+# slapos.core==0.35.1
+supervisor = 3.0b2
+
+# Required by:
+# slapos.core==0.35.1
+unittest2 = 0.5.1
+
+# Required by:
+# slapos.cookbook==0.78.1
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.core==0.35.1
+zope.interface = 4.0.5
+
+cliff = 1.4
+cmd2 = 0.6.5.1
+prettytable = 0.7.2
+requests = 1.2.3

software/backupserver/template-backup-script.sh.in

+#!${dash-output:dash}
+
+# trap "echo Backing up failed for $${:hostname}" ERR
+
+export HOME=$${directory:home}
+
+set -e
+cd $${:datadirectory}
+${rdiff-backup-output:rdiff-backup} \
+  $${:exclude_string} \
+  --include='$${:include}' \
+  --exclude='**' \
+  --remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
+  $${:connection}::/ ./
+
+# python scripts/verify_with_sudo.py ./ $${:connection}:/
+

software/backupserver/template-crontab-line.in

+# $${:_buildout_section_name_}
+$${:frequency} $${:script}

software/backupserver/template-crontab.in

+# min(0-59)  hours(0-23) day(1-31) month(1-12) dow(0-7)   command
+MAILTO=admins@erp5.org
+

software/backupserver/template-dcron-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+exec ${dcron-output:crond} \
+  -s $${directory:cron-entries} \
+  -c $${directory:crontabs} \
+  -t $${directory:cronstamps} \
+  -f -l 5 \
+  -L $${dcron-service:logfile}
+  # -M cron_simplelogger

software/cloudooo/instance-cloudoo-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "extends": "./schema-definitions.json#",
+  "properties": {
+    "tcpv4-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
+        "description": "Start allocating ports at this value, going upward",
+        "default": 23000
+      }]
+    },
+    "font-url-list": {
+      "description": "List of URLs from which fonts are to be downloaded",
+      "default": [],
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "backend-count": {
+      "description": "Number of backend cloudooo instances",
+      "default": 1,
+      "type": "integer"
+    }
+  }
+}

software/cloudooo/instance-cloudoo-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Cloudooo instanciation",
+  "properties": {
+    "url": {
+      "description": "Conversion service access information",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/cloudooo/instance-cloudoo.cfg.in

+{% set ipv4 = (ipv4_set | list)[0] -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+{% set section_list = [] -%}
+{% macro section(name) %}{% do section_list.append(name) %}{{ name }}{% endmacro -%}
 [buildout]
 parts =
   publish-cloudooo-connection-information
@@ -9,35 +12,53 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [publish-cloudooo-connection-information]
-recipe = slapos.cookbook:publishurl
-url = cloudooo://${cloudooo-instance:ip}:${cloudooo-instance:port}/
+recipe = slapos.cookbook:publish.serialised
+url = cloudooo://${haproxy:ip}:${haproxy:port}/
 
-[cloudooo-instance]
+[cloudooo-base]
 recipe = slapos.cookbook:generic.cloudooo
-
-# Network options
-ip = ${slap-network-information:local-ipv4}
-port = 23000
-openoffice-port = 23060
-
-# Paths
-configuration-file = ${rootdirectory:etc}/cloudooo.cfg
-wrapper = ${basedirectory:services}/cloudooo
-
-# Paths: Data
-data-directory = ${directory:cloudooo-data}
-
+ip = {{ ipv4 }}
 environment =
   LD_LIBRARY_PATH = {{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
   PATH = ${binary-link:target-directory}
-
 # Binary information
 # cloudooo specific configuration
 ooo-binary-path = {{ parameter_dict['libreoffice-bin'] }}/program
 ooo-paster = {{ bin_directory }}/cloudooo_paster
 ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
 
+{% set cloudooo_port = slapparameter_dict.get('tcpv4_port', 23000) | int -%}
+{% set backend_count = slapparameter_dict.get('backend-count', 1) | int -%}
+{% for index in range(backend_count) -%}
+{%   set name = 'cloudooo-' ~ index -%}
+[{{ section(name) }}]
+< = cloudooo-base
+port = {{ cloudooo_port }}
+openoffice-port = {{ cloudooo_port + 1 }}
+configuration-file = ${directory:etc}/{{ name }}.cfg
+data-directory = ${directory:srv}/{{ name }}
+wrapper = ${directory:services}/{{ name }}
+
+{%   set cloudooo_port = cloudooo_port + 2 -%}
+{% endfor -%}
+
+[haproxy]
+recipe = slapos.cookbook:haproxy
+name = cloudooo
+conf-path = ${directory:etc}/haproxy.cfg
+socket-path = ${directory:run}/haproxy.sock
+ip = {{ ipv4 }}
+port = 8001
+maxconn = 1
+wrapper-path = ${directory:services}/haproxy
+binary-path = {{ parameter_dict['haproxy'] }}/sbin/haproxy
+ctl-path = ${directory:bin}/haproxy-ctl
+backend-list =
+{%- for section_name in section_list %}
+  {{ "${" ~ section_name ~ ":ip}:${" ~ section_name ~ ":port}" }}
+{%- endfor %}
+
 [cloudooo-test-runner]
 recipe = slapos.cookbook:cloudooo.test
 
@@ -45,24 +66,25 @@ prepend-path = ${buildout:bin-directory}
 run-unit-test = ${buildout:bin-directory}/runUnitTest
 run-test-suite = ${buildout:bin-directory}/runTestSuite
 
-ooo-paster = ${cloudooo-instance:ooo-paster}
-configuration-file = ${cloudooo-instance:configuration-file}
+ooo-paster = ${cloudooo-0:ooo-paster}
+configuration-file = ${cloudooo-0:configuration-file}
 
 run-unit-test-binary = {{ bin_directory }}/runCloudoooUnitTest
 run-test-suite-binary = {{ bin_directory }}/runCloudoooTestSuite
 
 [fontconfig-instance]
 recipe = slapos.cookbook:fontconfig
-conf-path = ${rootdirectory:etc}/font.conf
+conf-path = ${directory:etc}/font.conf
 font-system-folder = {{ parameter_dict['fonts'] }}
 font-folder = ${directory:font}
-url-list =
-service-folder = ${basedirectory:services}
+{# XXX: violates "instanciation happens offline" rule -#}
+url-list = {{ slapparameter_dict.get('font-url-list', []) | join(' ') }}
+service-folder = ${directory:services}
 onetimedownload_path = {{ bin_directory }}/onetimedownload
 
 [binary-link]
 recipe = slapos.cookbook:symbolic.link
-target-directory = ${rootdirectory:bin}
+target-directory = ${directory:bin}
 link-binary =
   {{ parameter_dict['coreutils'] }}/bin/basename
   {{ parameter_dict['coreutils'] }}/bin/cat
@@ -78,17 +100,12 @@ link-binary =
   {{ parameter_dict['poppler'] }}/bin/pdftohtml
 
 # rest of parts are candidates for some generic stuff
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-services = ${rootdirectory:etc}/run
-
 [directory]
 recipe = slapos.cookbook:mkdirectory
-cloudooo-data = ${rootdirectory:srv}/cloudooo
-font = ${rootdirectory:srv}/font
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
 etc = ${buildout:directory}/etc
+font = ${:srv}/font
+run = ${:var}/run
+services = ${:etc}/run
 srv = ${buildout:directory}/srv
-bin = ${buildout:directory}/bin
+var = ${buildout:directory}/var

software/cloudooo/instance.cfg.in

@@ -6,6 +6,14 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
@@ -13,12 +21,14 @@ extra-context =
 context =
     key eggs_directory buildout:eggs-directory
     key develop_eggs_directory buildout:develop-eggs-directory
+    key slapparameter_dict slap-parameters:configuration
     ${:extra-context}
 
 [dynamic-template-cloudooo-parameters]
 file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
 freetype = {{ freetype_location }}
+haproxy = {{ haproxy_location }}
 libICE = {{ libICE_location }}
 libpng12 = {{ libpng12_location }}
 libSM = {{ libSM_location }}
@@ -40,8 +50,10 @@ buildout-bin-directory = {{ buildout_bin_directory }}
 < = jinja2-template-base
 template = {{ template_cloudooo }}
 filename = instance-cloudoo.cfg
+extensions = jinja2.ext.do
 extra-context =
     section parameter_dict dynamic-template-cloudooo-parameters
+    key ipv4_set slap-parameters:ipv4
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype

software/cloudooo/software.cfg

@@ -16,6 +16,7 @@ parts +=
   check-recipe
 # Create instance template
   template
+  slapos-cookbook
 
 # XXX: Workaround of SlapOS limitation
 # Unzippig of eggs is required, as SlapOS do not yet provide nicely working
@@ -24,11 +25,10 @@ unzip = true
 
 # Local development
 [slapos.cookbook-repository]
-recipe = plone.recipe.command
-stop-on-error = true
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = ${git:location}/bin/git clone --branch cloudooo --quiet http://git.erp5.org/repos/slapos.git ${:location}
-update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = master
+git-executable = ${git:location}/bin/git
 
 [check-recipe]
 recipe = plone.recipe.command
@@ -36,6 +36,14 @@ stop-on-error = true
 update-command = ${:command}
 command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}.in
@@ -54,7 +62,7 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 694205787e78c5d615d72d7b4b26d174
+md5sum = 425cb2e76d46d53bb0b0eebdb8c1aa95
 extra-context =
     key buildout_bin_directory buildout:bin-directory
     key dcron_location dcron:location
@@ -63,6 +71,7 @@ extra-context =
     key fonts_location fonts:location
     key freetype_location freetype:location
     key git_location git:location
+    key haproxy_location haproxy:location
     key imagemagick_location imagemagick:location
     key libICE_location libICE:location
     key libSM_location libSM:location
@@ -83,10 +92,9 @@ extra-context =
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = 4c8608f9525be0f01a09d60b240315a9
+md5sum = bbe84b4c9022db62c926e8a8a4bf02a1
 mode = 640
 
-
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud

software/davstorage/common.cfg

+[buildout]
+
+versions = versions
+
+extends =
+  ../../component/apache-php/buildout.cfg  
+  ../../component/apache/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../stack/shacache-client.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/gzip/buildout.cfg
+
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+  http://www.nexedi.org/static/packages/source/
+  http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
+
+# Use only quite well working sites.
+allow-hosts +=
+  *.googlecode.com
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  alastairs-place.net
+  bitbucket.org
+  dist.repoze.org
+  effbot.org
+  github.com
+  launchpad.net
+  peak.telecommunity.com
+  sourceforge.net
+  www.dabeaz.com
+  www.owlfish.com
+
+parts =
+  apache-php
+  application
+  template
+  lxml-python
+  eggs
+  instance-recipe-egg
+
+unzip= true
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+
+[instance-recipe]
+egg = slapos.cookbook
+module = davstorage
+
+[instance-recipe-egg]
+recipe = zc.recipe.egg
+python = python2.7
+eggs = ${instance-recipe:egg}
+
+[application]
+recipe = hexagonit.recipe.download
+url = http://garr.dl.sourceforge.net/project/ajaxplorer/ajaxplorer/dev-channel/4.3.4/ajaxplorer-core-4.3.4.tar.gz
+md5sum = 2f2ff8bda7bbe841ef0e870c724eb74f
+strip-top-level-dir = true
+
+[template]
+# Default template for the instance.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = bed788dee6daf05349c4577e7a7f1299
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[instance-davstorage]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-davstorage.cfg
+md5sum = 699ecf4678386667f58a3391bab7af0f
+output = ${buildout:directory}/template-davstorage.cfg
+mode = 0644
+
+[lxml-python]
+python = python2.7

software/davstorage/development.cfg

+[buildout]
+extends =
+  ../../component/git/buildout.cfg
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = davstorage-ajaxplorer
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+[versions]
+zc.buildout = 1.6.0-dev-SlapOS-002
+Jinja2 = 2.6
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+hexagonit.recipe.cmmi = 1.6
+hexagonit.recipe.download = 1.6nxd002
+meld3 = 0.6.10
+openssl = 1.0.1c
+# Required by:
+# slapos.core
+Flask = 0.9
+
+# Required by:
+# slapos.cookbook
+PyXML = 0.8.4
+
+slapos.recipe.template = 2.4.2
+
+# Required by:
+# slapos.cookbook==0.24
+# slapos.core==0.14
+# xml-marshaller==0.9.7
+lxml = 3.1.0
+
+# Required by:
+# slapos.cookbook==0.24
+netaddr = 0.7.10
+
+# Required by:
+# slapos.core==0.14
+netifaces = 0.8
+
+# Required by:
+# slapos.cookbook==0.24
+# slapos.core==0.14
+# zc.buildout==1.5.3-dev-SlapOS-009
+# zc.recipe.egg==1.3.2
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.73.1
+slapos.core = 0.35.1
+
+# Required by:
+# slapos.core==0.35.1
+supervisor = 3.0b1
+
+# Required by:
+# slapos.cookbook==0.73.1
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.24
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.35.1
+zope.interface = 4.0.5
+

software/davstorage/instance-davstorage.cfg

 [buildout]
 parts =
   davstorage
-  url
+  publish-connection-informations
   certificate-authority
   ca-davstorage
   cron
   cron-entry-logrotate
   logrotate
   logrotate-entry-davstorage
+  request-frontend
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[url]
-recipe = slapos.cookbook:publishurl
-scheme = webdavs
-user = $${davstorage:user}
-port = $${davstorage:port}
-host = $${davstorage:ip}
-password = $${davstorage:password}
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+webdav_access = https://$${request-frontend:connection-domain}
+url = https://$${request-frontend-ajaxupload:connection-domain}
+webdav_user = $${davstorage:user}
+webdav_password = $${davstorage:password}
 
 [davstorage]
 recipe = slapos.cookbook:davstorage
 user = user
-port = 8080
+password = insecure
+port_webdav = 8080
+port_ajax = 8070
 ip = $${slap-network-information:global-ipv6}
 
 # Path
@@ -42,12 +44,15 @@ root = $${buildout:directory}
 email-address = admin+davstorage@vifib.net
 htpasswd-file = $${directory:davstorage-conf}/davstorage.htpasswd
 promise = $${basedirectory:promises}/davstorage
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
 
 # Binaries
 apache-binary = ${apache:location}/bin/httpd
 apache-modules-dir = ${apache:location}/modules/
 apache-mime-file = ${apache:location}/conf/mime.types
 apache-htpasswd = ${apache:location}/bin/htpasswd
+source = ${application:location}
 
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
@@ -66,7 +71,7 @@ dcrond-binary = ${dcron:location}/sbin/crond
 cron-entries = $${directory:cron-entries}
 crontabs = $${directory:crontabs}
 cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:binary}
+catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
 [logrotate]
@@ -91,8 +96,8 @@ command = $${logrotate:wrapper}
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
-binary = $${rootdirectory:bin}/cron_simplelogger
-output = $${directory:cronoutput}
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
 
 
 [logrotate-entry-davstorage]
@@ -128,6 +133,7 @@ etc = $${buildout:directory}/etc/
 var = $${buildout:directory}/var/
 srv = $${buildout:directory}/srv/
 bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -149,3 +155,38 @@ cron-entries = $${rootdirectory:etc}/cron.d/
 crontabs = $${rootdirectory:etc}/crontabs/
 cronstamps = $${rootdirectory:etc}/cronstamps/
 cronoutput = $${basedirectory:log}/cron/
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+
+# Request frontend
+[request-frontend-ajaxupload]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend Ajax
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url https-only
+config-https-only = true
+config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
+return = domain
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend Webdav
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url https-only
+config-https-only = true
+config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
+return = domain
+
+# XXX Vivien: promise not working for now
+#[frontend-ajaxupload-promise]
+#recipe = slapos.cookbook:check_url_available
+#path = $${basedirectory:promises}/frontend-ajaxupload
+#url = $${request-frontend-ajaxupload:connection-site_url}
+#dash_path = ${dash:location}/bin/dash
+#curl_path = ${curl:location}/bin/curl
\ No newline at end of file

software/davstorage/software.cfg

 [buildout]
-
-find-links +=
-    http://www.nexedi.org/static/packages/source/slapos.buildout/
-
-versions = versions
-
 extends =
-  ../../component/apache/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../stack/shacache-client.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/python-2.7/buildout.cfg
-  ../../component/gzip/buildout.cfg
-
-# Use only quite well working sites.
-allow-hosts =
-  *.nexedi.org
-  *.python.org
-  *.sourceforge.net
-  dist.repoze.org
-  effbot.org
-  github.com
-  peak.telecommunity.com
-  psutil.googlecode.com
-  www.dabeaz.com
-  alastairs-place.net
-
-
-parts =
-  template
-  lxml-python
-  apache
-  logrotate
-  dcron
-  eggs
-  gzip
-  instance-recipe-egg
-
-unzip= true
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-
-[instance-recipe]
-egg = slapos.cookbook
-module = davstorage
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-python = python2.7
-eggs = ${instance-recipe:egg}
-
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = bed788dee6daf05349c4577e7a7f1299
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[instance-davstorage]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-davstorage.cfg
-md5sum = 49c8e049a78e233d3a553e64c8914592
-output = ${buildout:directory}/template-davstorage.cfg
-mode = 0644
-
-[lxml-python]
-python = python2.7
+  ../../component/git/buildout.cfg
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = davstorage-ajaxplorer
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
 
 [versions]
 zc.buildout = 1.6.0-dev-SlapOS-002
 Jinja2 = 2.6
-Werkzeug = 0.7.1
-buildout-versions = 1.6
-hexagonit.recipe.cmmi = 1.5.0
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+hexagonit.recipe.cmmi = 1.6
 hexagonit.recipe.download = 1.6nxd002
-meld3 = 0.6.7
-slapos.cookbook = 0.26
-
+meld3 = 0.6.10
+openssl = 1.0.1c
 # Required by:
-# slapos.core==0.14
-Flask = 0.7.2
+# slapos.core
+Flask = 0.9
 
 # Required by:
-# slapos.cookbook==0.24
+# slapos.cookbook
 PyXML = 0.8.4
 
-# Required by:
-# slapos.recipe.template==1.1
-collective.recipe.template = 1.9
+slapos.recipe.template = 2.4.2
 
 # Required by:
 # slapos.cookbook==0.24
 # slapos.core==0.14
 # xml-marshaller==0.9.7
-lxml = 2.3
+lxml = 3.1.0
 
 # Required by:
 # slapos.cookbook==0.24
-netaddr = 0.7.6
+netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.14
-#netifaces = 0.4
+netifaces = 0.8
 
 # Required by:
 # slapos.cookbook==0.24
@@ -115,15 +73,15 @@ netaddr = 0.7.6
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.24
-slapos.core = 0.14
+# slapos.cookbook==0.73.1
+slapos.core = 0.35.1
 
 # Required by:
-# slapos.core==0.14
-supervisor = 3.0a10
+# slapos.core==0.35.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.24
+# slapos.cookbook==0.73.1
 xml-marshaller = 0.9.7
 
 # Required by:
@@ -131,5 +89,6 @@ xml-marshaller = 0.9.7
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.14
-zope.interface = 3.7.0
+# slapos.core==0.35.1
+zope.interface = 4.0.5
+

software/davstorage/todo.txt

+TODO:
+
++ Move Ajaxplorer to a separate directory (currently in the www/ directory
+  accessed by webdav client, and so might delete files)
+
++ Make configuration part disappear ! (configuration when instance deploy, 
+  no need for user to do it at each start of a new instance)

software/erp5testnode/instance-default.cfg

@@ -7,7 +7,6 @@ offline = true
 parts =
   connection-dict
   testnode
-  pwgen
   shell
   shellinabox
   certificate-authority
@@ -16,12 +15,11 @@ parts =
 [connection-dict]
 recipe = slapos.cookbook:publish
 url = http://[$${shellinabox:ipv6}]:$${shellinabox:port}/
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 
 [pwgen]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
+storage-path = $${buildout:directory}/.password
 
 [testnode]
 recipe = slapos.cookbook:erp5testnode
@@ -82,7 +80,7 @@ port = 8080
 shell = $${shell:wrapper}
 wrapper = $${rootdirectory:bin}/shellinaboxd
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 directory = $${buildout:directory}/
 login-shell = $${rootdirectory:bin}/login
 certificate-directory = $${directory:shellinabox}

software/erp5testnode/software.cfg

@@ -20,7 +20,6 @@ extends =
   ../../component/zip/buildout.cfg
   ../../component/busybox/buildout.cfg
   ../../component/shellinabox/buildout.cfg
-  ../../component/pwgen/buildout.cfg
 
 # Local development
 develop =

software/etherpad-lite/common.cfg

+[buildout]
+
+versions = versions
+
+extends =
+  ../../stack/slapos.cfg
+  ../../component/gcc/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/nodejs/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+
+parts =
+  postgresql
+  nodejs
+  etherpad-lite-repository
+  install-deps
+  template
+  lxml-python
+  eggs
+  instance-recipe-egg
+
+unzip= true
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+
+[instance-recipe]
+egg = slapos.cookbook
+module = etherpad-lite
+
+[instance-recipe-egg]
+recipe = zc.recipe.egg
+python = python2.7
+eggs = ${instance-recipe:egg}
+
+[etherpad-lite-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://github.com/ether/etherpad-lite.git
+branch = develop
+git-executable = ${git:location}/bin/git
+
+[template]
+# Default template for the instance.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = 7ab2a242df988bf5c10bf8002acac3bd
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[instance-etherpad-lite]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
+md5sum = fd7249be8988155110234c7bb877abb9
+output = ${buildout:directory}/template-etherpad-lite.cfg
+mode = 0644
+
+[template-conf]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/templates/${:filename}
+mode = 0644
+filename = settings.json.in
+md5sum = 19ab39e6b3256c82fd54ce074488b136
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[template-run-script]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/templates/${:filename}
+mode = 0644
+filename = run.sh.in
+md5sum = eac870b5f30e735e109a48913af2fae3
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[template-deps-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/templates/${:filename}
+etherpad-location = ${etherpad-lite-repository:location}
+nodejs-location = ${nodejs:location}
+curl-location = ${curl:location}
+postgre-location = ${postgresql:location}
+mode = 0755
+md5sum = 53d0d53d419bd9ee592d3e1a1c84c758
+filename = installDeps.sh.in
+output = ${etherpad-lite-repository:location}/bin/installDeps.sh
+
+[install-deps]
+recipe = plone.recipe.command
+command = ${template-deps-script:output}
+update-command = command
+
+[lxml-python]
+python = python2.7

software/etherpad-lite/development.cfg

+[buildout]
+extends =
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = etherpad-lite
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
\ No newline at end of file

software/etherpad-lite/instance-etherpad-lite.cfg

+[buildout]
+parts =
+  etherpad-lite
+  publish-connection-informations
+#  frontend-etherpad
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:etc}/run/
+backup = $${rootdirectory:srv}/backup/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etherpad-conf = $${rootdirectory:etc}/etherpad/
+etherpad-repository-location = $${buildout:directory}/parts/etherpad-lite-repository
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+url = $${request-frontend:connection-site_url}
+
+[etherpad-conf-generation]
+recipe = slapos.recipe.template
+url = ${template-conf:location}/${template-conf:filename}
+ip = $${slap-network-information:global-ipv6}
+dirtydb-location = $${rootdirectory:var}/dirty.db
+port = 9001
+mode = 0644
+output = $${directory:etherpad-conf}/settings.json
+
+[etherpad-run-script]
+recipe = slapos.recipe.template
+url = ${template-run-script:location}/${template-run-script:filename}
+etherpad-location = ${etherpad-lite-repository:location}
+etherpad-repository-location = $${directory:etherpad-repository-location}
+nodejs-location = ${nodejs:location}
+etherpad-deps-script-location = ${template-deps-script:output}
+etherpad-conf-location = $${etherpad-conf-generation:output}
+etherpad-conf-name = settings-$${slap-connection:partition-id}.json
+mode = 0755
+output = $${rootdirectory:bin}/run.sh
+
+# Command line comes from the run script of etherpad-lite
+[etherpad-lite]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${basedirectory:run}/etherpad-lite 
+command-line = $${etherpad-run-script:output} -s $${etherpad-run-script:etherpad-conf-name}
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url
+config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
+return = site_url
+
+[frontend-etherpad]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend-etherpad
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
\ No newline at end of file

software/etherpad-lite/instance.cfg

+[buildout]
+
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-etherpad-lite:output}

software/etherpad-lite/software.cfg

+[buildout]
+extends =
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = etherpad-lite
+git-executable = ${git:location}/bin/git
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
\ No newline at end of file

software/etherpad-lite/templates/installDeps.sh.in

+#!/bin/sh
+
+# Ugly setup
+cd ${:etherpad-location}
+export PATH=$PATH:${:postgre-location}/bin
+
+#Is gnu-grep (ggrep) installed on SunOS (Solaris)
+if [ $(uname) = "SunOS" ]; then
+  hash ggrep > /dev/null 2>&1 || { 
+    echo "Please install ggrep (pkg install gnu-grep)" >&2
+    exit 1 
+  }
+fi
+
+#Is wget installed?
+hash ${:curl-location}/bin/curl > /dev/null 2>&1 || { 
+  echo "Please install curl" >&2
+  exit 1 
+}
+
+#Is node installed?
+hash ${:nodejs-location}/bin/node > /dev/null 2>&1 || { 
+  echo "Please install node.js ( http://nodejs.org )" >&2
+  exit 1 
+}
+
+#Is npm installed?
+hash ${:nodejs-location}/bin/npm > /dev/null 2>&1 || { 
+  echo "Please install npm ( http://npmjs.org )" >&2
+  exit 1 
+}
+
+echo "Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient."
+(
+  mkdir -p node_modules
+  cd node_modules
+  [ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
+  cd ep_etherpad-lite
+  ${:nodejs-location}/bin/npm install --loglevel warn
+) || { 
+  rm -rf node_modules
+  exit 1 
+}
+
+echo "Ensure jQuery is downloaded and up to date..."
+DOWNLOAD_JQUERY="true"
+NEEDED_VERSION="1.9.1"
+
+if [ $DOWNLOAD_JQUERY = "true" ]; then
+  ${:curl-location}/bin/curl -lo src/static/js/jquery.js http://code.jquery.com/jquery-$NEEDED_VERSION.js || exit 1
+fi
+
+#Remove all minified data to force node creating it new
+echo "Clear minfified cache..."
+rm -f var/minified*
+
+echo "ensure custom css/js files are created..."
+
+for f in "index" "pad" "timeslider"
+do
+  if [ ! -f "src/static/custom/$f.js" ]; then
+    cp "src/static/custom/js.template" "src/static/custom/$f.js" || exit 1
+  fi
+  
+  if [ ! -f "src/static/custom/$f.css" ]; then
+    cp "src/static/custom/css.template" "src/static/custom/$f.css" || exit 1
+  fi
+done
+
+exit 0

software/etherpad-lite/templates/run.sh.in

+#!/bin/sh
+
+# Copy repository content from the software release
+mkdir ${:etherpad-repository-location}
+cp -R ${:etherpad-location}/* ${:etherpad-repository-location}
+
+#Move to the folder where ep-lite is installed
+cd ${:etherpad-repository-location}
+
+# XXX Vivien: very ugly, definitely need to find a cleaner way
+cp ${:etherpad-conf-location} ${:etherpad-repository-location}/${:etherpad-conf-name}
+
+#Move to the node folder and start
+echo "start..."
+${:nodejs-location}/bin/node node_modules/ep_etherpad-lite/node/server.js $*

software/etherpad-lite/templates/settings.json.in

+/*
+  This file must be valid JSON. But comments are allowed
+
+  Please edit settings.json, not settings.json.template
+*/
+{
+  // Name your instance!
+  "title": "Etherpad",
+
+  // favicon default name
+  // alternatively, set up a fully specified Url to your own favicon
+  "favicon": "favicon.ico",
+  
+  //IP and port which etherpad should bind at
+  "ip": "${:ip}",
+  "port" : "${:port}",
+
+  // Session Key, used for reconnecting user sessions
+  // Set this to a secure string at least 10 characters long.  Do not share this value.
+  "sessionKey" : "",
+
+  /*  
+  // Node native SSL support
+  // this is disabled by default
+  //
+  // make sure to have the minimum and correct file access permissions set
+  // so that the Etherpad server can access them
+
+  "ssl" : {
+            "key"  : "/path-to-your/epl-server.key",
+            "cert" : "/path-to-your/epl-server.crt"
+          },
+
+  */
+
+  //The Type of the database. You can choose between dirty, postgres, sqlite and mysql
+  //You shouldn't use "dirty" for for anything else than testing or development
+  "dbType" : "dirty",
+  //the database specific settings
+  "dbSettings" : {
+                   "filename" : "${:dirtydb-location}"
+                 },
+                 
+  /* An Example of MySQL Configuration
+   "dbType" : "mysql",
+   "dbSettings" : {
+                    "user"    : "root", 
+                    "host"    : "localhost", 
+                    "password": "", 
+                    "database": "store"
+                  },
+  */
+  
+  //the default text of a pad
+  "defaultPadText" : "Welcome to Etherpad!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nGet involved with Etherpad at http:\/\/etherpad.org\n",
+  
+  /* Users must have a session to access pads. This effectively allows only group pads to be accessed. */
+  "requireSession" : false,
+
+  /* Users may edit pads but not create new ones. Pad creation is only via the API. This applies both to group pads and regular pads. */
+  "editOnly" : false,
+  
+  /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, 
+     but makes it impossible to debug the javascript/css */
+  "minify" : true,
+
+  /* How long may clients use served javascript code (in seconds)? Without versioning this
+     may cause problems during deployment. Set to 0 to disable caching */
+  "maxAge" : 21600, // 60 * 60 * 6 = 6 hours
+  
+  /* This is the path to the Abiword executable. Setting it to null, disables abiword.
+     Abiword is needed to advanced import/export features of pads*/  
+  "abiword" : null,
+ 
+  /* This setting is used if you require authentication of all users.
+     Note: /admin always requires authentication. */
+  "requireAuthentication": false,
+
+  /* Require authorization by a module, or a user with is_admin set, see below. */
+  "requireAuthorization": false,
+
+  /* Users for basic authentication. is_admin = true gives access to /admin.
+     If you do not uncomment this, /admin will not be available! */
+  /*
+  "users": {
+    "admin": {
+      "password": "changeme1",
+      "is_admin": true
+    },
+    "user": {
+      "password": "changeme1",
+      "is_admin": false
+    }
+  },
+  */
+
+  // restrict socket.io transport methods
+  "socketTransportProtocols" : ["xhr-polling"],
+  
+  /* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */
+  "loglevel": "INFO",
+  
+  //Logging configuration. See log4js documentation for further information
+  // https://github.com/nomiddlename/log4js-node
+  // You can add as many appenders as you want here:
+  "logconfig" :
+	{ "appenders": [
+	    { "type": "console",
+      //  "category": "access"// only logs pad access
+	    }
+    /*
+	  , { "type": "file"
+      , "filename": "your-log-file-here.log"
+      , "maxLogSize": 1024
+      , "backups": 3 // how many log files there're gonna be at max
+      //, "category": "test" // only log a specific category
+      }*/
+    
+	    /*{ "type": "logLevelFilter",
+	      "level": "warn" // filters out all log messages that have a lower level than "error"
+	    , "appender":
+	      {  Use whatever appender you want here  }
+      }*/
+    /*
+	  , { "type": "logLevelFilter"
+	    , "level": "error" // filters out all log messages that have a lower level than "error"
+	    , "appender":
+	      { "type": "smtp"
+        , "subject": "An error occured in your EPL instance!"
+        , "recipients": "bar@blurdybloop.com, baz@blurdybloop.com"
+        , "sendInterval": 60*5 // in secs -- will buffer log messages; set to 0 to send a mail for every message
+        , "transport": "SMTP", "SMTP": { // see https://github.com/andris9/Nodemailer#possible-transport-methods
+            "host": "smtp.example.com", "port": 465,
+            "secureConnection": true,
+            "auth": {
+                "user": "foo@example.com",
+                "pass": "bar_foo"
+            }
+        }
+        }
+      }*/
+	] }
+}

software/etherpad-lite/todo.txt

+TODO:
+
++ Modify etherpad-lite to read absolute path correctly, so that we aren't
+  forced to move everything to the software release directory for it to work

software/gitrepo/instance-gitrepo.cfg

@@ -13,14 +13,13 @@ parts =
   gitdaemon
   git-http-backend-cgi
   htpasswd
-  pwgen
   git-repos
 
 [publish]
 recipe = slapos.cookbook:publish
 url = http://[$${slap-network-information:global-ipv6}]:$${httpd-conf:port}/
 user = $${pwgen:user}
-password = $${pwgen:password}
+password = $${pwgen:passwd}
 
 [httpd]
 recipe = slapos.cookbook:wrapper
@@ -79,14 +78,12 @@ output = $${basedirectory:services}/git-daemon
 recipe = collective.recipe.cmd
 output = $${rootdirectory:etc}/httpd.htpasswd
 on_install = true
-on_udptae = true
+on_update = true
 cmds =
-  ${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:password}
+  ${apache:location}/bin/htpasswd -cb $${:output} $${pwgen:user} $${pwgen:passwd}
 
 [pwgen]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
 user = slapos
 
 [rootdirectory]

software/gitrepo/software.cfg

@@ -4,7 +4,6 @@ extends =
   ../../component/apache/buildout.cfg
   ../../component/perl/buildout.cfg
   ../../component/git/buildout.cfg
-  ../../component/pwgen/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =

software/kvm/common.cfg

@@ -9,13 +9,18 @@ extends =
   ../../component/logrotate/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
+  ../../component/dcron/buildout.cfg
   ../../stack/nodejs.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
   template
   eggs
 
+# XXX: we have to manually add this for resilience
+  rdiff-backup
+
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
 #             Problem is : no native crypto support in web browsers. So we have to disable ssl
@@ -67,13 +72,58 @@ command =
   ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
 
 
+# Create all templates that will be used to deploy instances
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 0e84223169661462f439c164d62c2a6a
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg.in
-md5sum = 87197471aa93863c310204e8865b5ac1
+md5sum = f7c0e2172dac4ee70daae50f38d610ef
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
+[template-kvm-resilient]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
+mode = 644
+md5sum = 3ee64c654aae503b93b39e9ccd6d3643
+
+[template-kvm-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
+md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
+output = ${buildout:directory}/template-kvm-import.cfg
+mode = 0644
+
+[template-kvm-import-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-import.sh.in
+filename = kvm-import.sh.in
+md5sum = e03ed049cddd8d157228b09e1ebc071a
+download-only = true
+mode = 0755
+
+[template-kvm-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
+md5sum = 64a1a505aff9fde52afac46240811047
+output = ${buildout:directory}/template-kvm-export.cfg
+mode = 0644
+
+[template-kvm-export-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-export.sh.in
+filename = kvm-export.sh.in
+md5sum = 08cd8da2221f09095b14e35e6acd2a56
+download-only = true
+mode = 0755
+
 [template-nbd]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-nbd.cfg.in
@@ -87,10 +137,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 0a98e34aaec7097a84066c0665e3a49a
-output = ${buildout:directory}/template.cfg
-mode = 0644

software/kvm/development.cfg

@@ -31,3 +31,6 @@ update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+[versions]
+websockify = 0.3.0
\ No newline at end of file

software/kvm/instance-kvm-export.cfg.in

+[buildout]
+extends = ${template-kvm:output}
+          ${pbsready-export:output}
+
+parts +=
+  cron-entry-backup
+
+  certificate-authority
+  publish-connection-information
+  kvm-promise
+  websockify-sighandler
+  novnc-promise
+  cron
+  frontend-promise
+
+# Create the exporter executable, which is a simple shell script
+[exporter]
+recipe = slapos.recipe.template
+url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-exporter
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+# Extends publish section with resilient parameters
+[publish-connection-information]
+<= resilient-publish-connection-parameter
+

software/kvm/instance-kvm-import.cfg.in

+[buildout]
+# Here, we don't need KVM to run to import data, so we don't
+# even extend the kvm instance profile.
+extends = ${pbsready-import:output}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+novnc-conf = $${:etc}/novnc
+run = $${:var}/run
+ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+
+[importer]
+recipe = slapos.recipe.template
+url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-importer
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+disk-path = $${directory:srv}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("kvm", "2") }}
+  publish-connection-informations
+
+{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
+
+# Bubble down the parameters of the requested instance to the user
+[request-kvm]
+# Note: += doesn't work.
+return =
+# Resilient related parameters
+  url ssh-public-key ssh-url notification-id ip
+# KVM related parameters
+  backend-url url
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend-url = ${request-kvm:connection-backend-url}
+url = ${request-kvm:connection-url}
+

software/kvm/instance-kvm.cfg.in

@@ -6,10 +6,13 @@
 [buildout]
 parts =
   certificate-authority
-  publish-kvm-connection-information
+  publish-connection-information
   kvm-promise
   websockify-sighandler
   novnc-promise
+#  kvm-monitor
+  cron
+#  cron-entry-monitor
   frontend-promise
 
 eggs-directory = ${buildout:eggs-directory}
@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc
 bin = $${buildout:directory}/bin
 srv = $${buildout:directory}/srv
 var = $${buildout:directory}/var
+log = $${:var}/log
 scripts = $${:etc}/run
 services = $${:etc}/service
 promises = $${:etc}/promise
 novnc-conf = $${:etc}/novnc
 run = $${:var}/run
 ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
 
 [create-mac]
 recipe = slapos.cookbook:generate.mac
@@ -44,6 +51,7 @@ bytes = 8
 recipe = slapos.cookbook:kvm
 vnc-ip = $${slap-network-information:local-ipv4}
 vnc-port = 5901
+default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
 nbd-host = $${slap-parameter:nbd-host}
 nbd-port = $${slap-parameter:nbd-port}
 nbd2-host = $${slap-parameter:nbd2-host}
@@ -122,11 +130,43 @@ hostname = $${novnc-instance:ip}
 port = $${novnc-instance:port}
 
 
-[kvm-monitor]
-recipe = slapos.cookbook:generic.slapmonitor
-db-path = $${directory:srv}/slapmonitor_database
+#[kvm-monitor]
+#recipe = slapos.cookbook:wrapper
+#wrapper-path = $${directory:services}/kvm_monitor
+#command-line = ${buildout:bin-directory}/kvm.monitor.test
+#             $${buildout:directory}/buildout-switch-softwaretype.cfg
+#             $${buildout:directory}/report.xml
+#             -s slap-parameter
+#             -opts disk-size ram-size cpu-count
 
 
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/crond.log
+
+
+#[cron-entry-monitor]
+#<= cron
+#recipe = slapos.cookbook:cron.d
+#name = kvm_monitor
+#frequency = 0 0 * * *
+#command = $${kvm-monitor:wrapper-path}
+
 [request-slave-frontend]
 recipe = slapos.cookbook:requestoptional
 software-url = $${slap-parameter:frontend-software-url}
@@ -146,17 +186,16 @@ sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 
-[publish-kvm-connection-information]
+[publish-connection-information]
 recipe = slapos.cookbook:publish
-backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
-password = $${kvm-instance:passwd}
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
+backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:passwd}
+url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:passwd}
 
 
 [frontend-promise]
 recipe = slapos.cookbook:check_url_available
 path = $${directory:promises}/frontend_promise
-url = $${publish-kvm-connection-information:url}
+url = $${publish-connection-information:url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
@@ -164,9 +203,9 @@ curl_path = ${curl:location}/bin/curl
 # Default values if not specified
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
-
+frontend-instance-guid =
 nbd-port = 1024
-nbd-host = debian.nbd.vifib.net
+nbd-host =
 nbd2-port = 1024
 nbd2-host =
 

software/kvm/instance.cfg.in

@@ -4,7 +4,6 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
@@ -13,11 +12,33 @@ kvm = ${template-kvm:output}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
 
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
+kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
+kvm-import = ${template-kvm-import:output}
+kvm-export = ${template-kvm-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[dynamic-template-kvm-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
+md5sum = 1b7a2fcc884649b4d08f238e828899c1
+rendered = $${buildout:directory}/template-kvm-resilient.cfg
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+

software/kvm/software.cfg

@@ -47,98 +47,96 @@ signature-certificate-list =
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
 
-
 [versions]
-numpy = 1.6.2
-Jinja2 = 2.6
-Werkzeug = 0.8.3
-apache-libcloud = 0.12.1
+Werkzeug = 0.9.3
+apache-libcloud = 0.13.0
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
-hexagonit.recipe.cmmi = 1.6
-lxml = 3.1.0
+itsdangerous = 0.22
+lxml = 3.2.3
 meld3 = 0.6.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.73.1
+rdiff-backup = 1.0.5
+slapos.cookbook = 0.78.3
+slapos.recipe.cmmi = 0.2
+slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.33.1
+slapos.toolbox = 0.35.0
 smmap = 0.8.2
-websockify = 0.3.0
+websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-Flask = 0.9
+# slapos.toolbox==0.35.0
+Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 feedparser = 5.1.3
 
 # Required by:
-# hexagonit.recipe.cmmi==1.6
-hexagonit.recipe.download = 1.6nxd002
+# slapos.cookbook==0.78.3
+inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.73.1
-inotifyx = 0.2.0
+# slapos.cookbook==0.78.3
+lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.73.1
+# slapos.cookbook==0.78.3
 netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.35.1
-netifaces = 0.8
+netifaces = 0.8-1
 
 # Required by:
-# slapos.toolbox==0.33.1
-paramiko = 1.10.0
+# websockify==0.5.1
+numpy = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.33.1
-psutil = 0.6.1
+# slapos.toolbox==0.35.0
+paramiko = 1.11.0
 
 # Required by:
-# slapos.core==0.35.1
-pyflakes = 0.6.1
+# slapos.toolbox==0.35.0
+psutil = 1.0.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-pytz = 2012j
+# slapos.core==0.35.1
+pyflakes = 0.7.3
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-setuptools = 0.6c12dev-r88846
+# slapos.cookbook==0.78.3
+pytz = 2013b
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 slapos.core = 0.35.1
 
 # Required by:
 # slapos.core==0.35.1
-supervisor = 3.0b1
+supervisor = 3.0b2
 
 # Required by:
 # slapos.core==0.35.1
 unittest2 = 0.5.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 xml-marshaller = 0.9.7
 
 # Required by:

software/kvm/template/kvm-export.sh.in

+#!/bin/bash
+# Create a backup of the disk image of the virtual machine
+QEMU_IMG=${kvm-instance:qemu-img-path}
+SNAPSHOT_NAME=$(date +%s)
+DISK_PATH=${kvm-instance:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+if [ ! -f $DISK_PATH ]; then
+  echo "Nothing to backup, disk image doesn't exist yet."
+  exit 0;
+fi
+
+$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
+if [ -f $BACKUP_PATH ]; then
+  rm $BACKUP_PATH
+fi
+$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
+$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH

software/kvm/template/kvm-import.sh.in

+#!/bin/bash
+DISK_PATH=${:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+# TODO: Use rdiff
+rm $DISK_PATH
+cp $BACKUP_PATH $DISK_PATH

software/lxc/instance-lxc.cfg

@@ -68,9 +68,8 @@ bridge = !!BRIDGE_NAME!!
 interface = lxc$${slap-network-information:network-interface}
 
 [passwd]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
+storage-path = $${buildout:directory}/.password
 
 [shellinabox]
 recipe = slapos.cookbook:shellinabox
@@ -79,7 +78,7 @@ port = 8080
 shell = ${lxc:location}/bin/lxc-console -n $${uuid:uuid}
 wrapper = $${rootdirectory:bin}/shellinaboxd_raw
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
-password = $${passwd:password}
+password = $${passwd:passwd}
 directory = $${buildout:directory}/
 login-shell = $${rootdirectory:bin}/login
 certificate-directory = $${directory:shellinabox}

software/lxc/software.cfg

@@ -10,7 +10,6 @@ extends =
   ../../component/xz-utils/buildout.cfg
   ../../component/tar/buildout.cfg
   ../../component/shellinabox/buildout.cfg
-  ../../component/pwgen/buildout.cfg
   ../../component/bash/buildout.cfg
   ../../component/coreutils/buildout.cfg
 
@@ -23,7 +22,6 @@ parts =
   slapos-toolbox
   lxc
   shellinabox
-  pwgen
 
 [template]
 recipe = slapos.recipe.template

software/neoppod/README.txt

@@ -37,11 +37,6 @@ Defautls to 12.
 Number of replicates.
 Defaults to 0 (no resilience).
 
-'verbose' (int, optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~
-Non-zero to increase verbosity.
-Defaults to 0.
-
 'mysql-storage-count' (int, optional)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Number of MySQL-based storage nodes to deploy. One master node is deployed

software/neoppod/cluster.cfg.in

@@ -19,10 +19,9 @@ partition-id = ${slap-connection:partition-id}
 config-cluster = {{ slapparameter_dict['cluster'] }}
 config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
 config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
-config-verbose = {{ dumps(slapparameter_dict.get('verbose', 0)) }}
 
 [request-config]
-base = cluster verbose
+base = cluster
 master = partitions replicas
 node = masters
 

software/neoppod/instance-neo-admin.cfg.in

@@ -16,7 +16,6 @@ ip = ${publish:ip}
 port = ${publish:port}
 cluster = ${slap-parameter:cluster}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 
 [publish]
 recipe = slapos.cookbook:publish
@@ -33,6 +32,7 @@ link-binary = ${neo-admin:wrapper}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
+promises = ${buildout:directory}/etc/promises
 run = ${buildout:directory}/etc/run
 log = ${buildout:directory}/var/log
 bin = ${buildout:directory}/bin

software/neoppod/instance-neo-master.cfg.in

@@ -17,7 +17,6 @@ cluster = ${slap-parameter:cluster}
 partitions = ${slap-parameter:partitions}
 replicas = ${slap-parameter:replicas}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 
 [publish]
 recipe = slapos.cookbook:publish
@@ -34,6 +33,7 @@ link-binary = ${neo-master:wrapper}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
+promises = ${buildout:directory}/etc/promises
 run = ${buildout:directory}/etc/run
 log = ${buildout:directory}/var/log
 bin = ${buildout:directory}/bin

software/neoppod/instance-neo-storage-mysql.cfg.in

@@ -10,6 +10,7 @@ database = neo
 conf-file = ${directory:etc}/mariadb.cnf
 # No networking (ip & port options)
 data-directory = ${directory:srv_mariadb}
+bin-directory = ${directory:bin}
 pid-file = ${directory:var_run}/mariadb.pid
 socket = ${directory:var_run}/mariadb.sock
 error-log = ${directory:log}/mariadb_error.log
@@ -33,7 +34,6 @@ logfile = ${directory:log}/neostorage.log
 ip = [${slap-network-information:global-ipv6}]
 cluster = ${slap-parameter:cluster}
 masters = ${slap-parameter:masters}
-verbose = ${slap-parameter:verbose}
 database-adapter = MySQL
 database-parameters = ${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:database}${mariadb-instance:socket}
 wait-database = 60

software/neoppod/software.cfg

-# Note on COOKBOOK/END COOKBOOK: the areas delimited by those tags should be
-# dropped when a cookbook version containing neoppod recipe is released as an
-# egg on pypi.
 # Note on LXML/END LXML: they delimit areas where lxml magic is needed. lxml is
 # a slapos.cookbook dependency, so it should be fetched automatically. But when
 # automatically fetched, it gets built against system headers/libs, which is
@@ -10,9 +7,6 @@
 [buildout]
 extends =
     ../../stack/slapos.cfg
-#COOKBOOK
-    ../../component/git/buildout.cfg
-#END COOKBOOK
 #LXML
     ../../component/lxml-python/buildout.cfg
 #END LXML
@@ -20,19 +14,9 @@ extends =
     ../../component/mariadb/buildout.cfg
     ../../component/mysql-python/buildout.cfg
 
-develop =
-#COOKBOOK
-# Note: as we develop a path we fetch, first buildout execution will fail as
-# develop couldn't find the clone. Second (and later) runs will succeed.
-    ${:parts-directory}/slapos.cookbook-repository
-#END COOKBOOK
-
 parts =
-#COOKBOOK
-    slapos.cookbook-repository
-    check-recipe
-#END COOKBOOK
     slapos-deps-eggs
+    slapos-cookbook
 # NEO & dependencies
     python2.7
     mariadb
@@ -47,29 +31,12 @@ eggs =
 #LXML
     ${lxml-python:egg}
 #END LXML
-#COOKBOOK
-    slapos.cookbook
-#END COOKBOOK
 
 #LXML
 [lxml-python]
 python = python2.7
 #END LXML
 
-#COOKBOOK
-[slapos.cookbook-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.git
-branch = neoppod
-git-executable = ${git:location}/bin/git
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = test -e ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-#END COOKBOOK
-
 [base-template]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}.in
@@ -85,12 +52,12 @@ context =
 [template-neo-master]
 < = base-template
 filename = instance-neo-master.cfg
-md5sum = b48e9446fb8fdce24d954d7d4f4c7460
+md5sum = 8947620b15535b95163c11a0efab546e
 
 [template-neo-storage-mysql]
 < = base-template
 filename = instance-neo-storage-mysql.cfg
-md5sum = e767c66e7de58232de30bdf26188efe1
+md5sum = 518c8c41ce73f0b608fbb1c3b889ca1b
 extra-context =
     key mariadb_location mariadb:location
     key master_cfg template-neo-master:rendered
@@ -98,12 +65,12 @@ extra-context =
 [template-neo-admin]
 < = base-template
 filename = instance-neo-admin.cfg
-md5sum = fcf1a5c937a609a934c8ccd7c1326726
+md5sum = 987f69333c563898cab21455509f8024
 
 [template-cluster]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/cluster.cfg.in
-md5sum = fe37b2ff36fee65b44945393f63e51c3
+md5sum = 7177d129e171b20044bf435ed549af43
 
 [template-instance]
 < = base-template

software/slaprunner/TODO.txt

+- Use https to connect to the runner. !!! Critical
+- resilient sr: Cloned instances should not launch slapgrid-sr if it was not launched on export instance

software/slaprunner/common.cfg

 [buildout]
 extends =
+  ../../component/bash/buildout.cfg
   ../../component/cloud9/buildout.cfg
+  ../../component/curl/buildout.cfg
   ../../component/dash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/rsync/buildout.cfg
   ../../stack/flask.cfg
   ../../stack/shacache-client.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
+  rdiff-backup
   template
-  slapos-cookbook
   eggs
+  nginx
+  simple-proxy
+  node-frontend-template
+  http-proxy
+  npm-modules
+  instance-runner-import
+  instance-runner-export
+  slapos-cookbook
+
+####################
+## Node JS proxy
+####################
+[simple-proxy]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/simple-proxy.js
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+md5sum = 86e2231b3f65587b56d9be63e21a4e05
+filename = simple-proxy.js
+mode = 0644
+
+[node-frontend-template]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/node-frontend.in
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = node-frontend.in
+md5sum = 72904152860dddb30ca936dac5bbf4cd
+mode = 0644
+
+[http-proxy]
+# https://github.com/nodejitsu/node-http-proxy
+recipe = slapos.recipe.build:download-unpacked
+#XXX-Cedric : use upstream when merged
+url = https://github.com/desaintmartin/node-http-proxy/archive/20120621.zip
+md5sum = 621e5fca448cbea137c5d847d780d84d
+
+[npm-modules]
+recipe = plone.recipe.command
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command =
+  export HOME=${:location};
+  rm -fr ${:destination} &&
+  mkdir -p ${:destination} &&
+  cd ${:destination} &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install colors@0.6.0-1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install socket.io-client@0.8.7 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install optimist@0.3.1 &&
+  ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
+
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
+md5sum = b6ed8c30cc2ec51244796ce57315089f
 mode = 0644
-md5sum = 5307e4200f044ae57b504ad68444491c
 
 [template-runner]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg
-md5sum = 91d6550c43b7a43a999724af4650ae40
+md5sum = 4877c808972b6b04bfac1de94c14a301
+mode = 0644
+
+[instance-runner-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-import.cfg.in
+output = ${buildout:directory}/instance-runner-import.cfg
+md5sum = f16cb60bb16632e652bea69cd5cdd9b7
 mode = 0644
 
+[instance-runner-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner-export.cfg.in
+output = ${buildout:directory}/instance-runner-export.cfg
+md5sum = 7e71622c09271790b5cef21c8613b8ac
+mode = 0644
+
+[template-resilient]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
+md5sum = fc253453da66583e7d515a6e2eb1475e
+filename = instance-resilient.cfg.jinja2
+mode = 0644
+
+[template_nginx_conf]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/nginx_conf.in
+md5sum = 09b7677dfc6b23c1f58e67fd06a7625e
+filename = nginx_conf.in
+mode = 0644
+
+[template_launcher]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/launcher.in
+md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
+filename = launcher.in
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs =
   ${lxml-python:egg}
+  cns.recipe.symlink
+  hexagonit.recipe.download
+  inotifyx
+  lock-file
+  netaddr
   slapos.libnetworkcache
   slapos.toolbox[flask_auth]
   slapos.core
-  cns.recipe.symlink
+  xml_marshaller
+  pytz

software/slaprunner/development.cfg

@@ -10,20 +10,21 @@ extends = common.cfg
 
 parts +=
   slapos.cookbook-repository
-  slapos.core-repository
   slapos.toolbox-repository
-  check-recipe
+
+#  slapos.core-repository
+#  check-recipe
 
 develop =
-  ${:parts-directory}/slapos.cookbook-repository
-  ${:parts-directory}/slapos.core-repository
 	${:parts-directory}/slapos.toolbox-repository
+	${:parts-directory}/slapos.cookbook-repository
+#  ${:parts-directory}/slapos.core-repository
 
 
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = slaprunner
+branch = slaprunner-resiliency
 git-executable = ${git:location}/bin/git
 
 [slapos.cookbook-repository]
@@ -32,17 +33,18 @@ repository = http://git.erp5.org/repos/slapos.git
 branch = slaprunner
 git-executable = ${git:location}/bin/git
 
-[slapos.core-repository]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/slapos.core.git
-branch = master
-git-executable = ${git:location}/bin/git
-
+#[slapos.core-repository]
+#recipe = slapos.recipe.build:gitclone
+#repository = http://git.erp5.org/repos/slapos.core.git
+#branch = master
+#git-executable = ${git:location}/bin/git
+#
 [check-recipe]
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+#  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&

software/slaprunner/instance-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("runner", "3") }}
+  publish-connection-informations
+
+{{ replicated.replicate("runner", "3", "runner-export", "runner-import", slapparameter_dict=slapparameter_dict) }}
+
+# Bubble up the parameters
+[request-runner]
+return = url ssh-public-key ssh-url notification-id ip backend_url url cloud9_url ssh_command password_recovery_code cloud9_backend_url
+config = instance-amount debug domain number authorized-key notify ip-list namebase runner1-computer-guid pbs-runner1-computer-guid runner2-computer-guid pbs-runner2-computer-guid runner3-computer-guid pbs-runner3-computer-guid
+# XXX Cedric LN Ugly hack, resilient stack and slaprunner stack sharing too much ssh sections
+config-authorized-key = ${request-pbs-runner-1:connection-ssh-key} ${request-pbs-runner-2:connection-ssh-key} ${slap-parameter:authorized-key}
+config-instance-amount = ${slap-parameter:instance-amount}
+config-debug = ${slap-parameter:debug}
+config-runner1-computer-guid = ${slap-parameter:runner1-computer-guid}
+config-pbs-runner1-computer-guid = ${slap-parameter:pbs-runner1-computer-guid}
+config-runner2-computer-guid = ${slap-parameter:runner2-computer-guid}
+config-pbs-runner2-computer-guid = ${slap-parameter:pbs-runner2-computer-guid}
+config-runner3-computer-guid = ${slap-parameter:runner3-computer-guid}
+config-pbs-runner3-computer-guid = ${slap-parameter:pbs-runner3-computer-guid}
+config-domain = ${slap-parameter:domain}
+
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+1_info = Set your passord in slaprunner in order to access cloud9
+backend_url = ${request-runner:connection-backend_url}
+url = ${request-runner:connection-url}
+cloud9_url = ${request-runner:connection-cloud9_url}
+cloud9_backend_url = ${request-runner:connection-cloud9_backend_url}
+ssh_command = ${request-runner:connection-ssh_command}
+password_recovery_code = ${request-runner:connection-password_recovery_code}
+
+[slap-parameter]
+# Default parameters for distributed deployment
+# I.e state "backup1 of maria should go there, ..."
+# XXX-Cedric: Hardcoded number of backups. Should be dynamically generated.
+runner1-computer-guid =
+pbs-runner1-computer-guid =
+runner2-computer-guid =
+pbs-runner2-computer-guid =
+runner3-computer-guid =
+pbs-runner3-computer-guid =
+# XXX-Cedric: Hardcoded parameters. Should be dynamically generated.
+domain =
+authorized-key =
+instance-amount = 10
+debug = false
\ No newline at end of file

software/slaprunner/instance-runner-export.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-export:output}
+
+parts +=
+  nginx_conf
+  nginx-launcher
+  cloud9
+  certificate-authority
+  ca-nginx
+  ca-node-frontend
+  slaprunner
+  test-runner
+  sshkeys-dropbear-runner
+  dropbear-server-add-authorized-key
+  sshkeys-authority
+  slaprunner-promise
+  slaprunner-frontend-promise
+  cloud9-promise
+  cloud9-frontend-promise
+  dropbear-promise
+  symlinks
+  node-frontend-promise
+  nginx-promise
+  urls
+  cron-entry-backup
+
+[exporter]
+recipe = slapos.cookbook:slaprunner.export
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-exporter
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+
+
+# Extends publish section with resilient parameters
+[urls]
+<= resilient-publish-connection-parameter
+backend_url = $${slaprunner:access-url}
+url =  https://$${request-frontend:connection-domain}
+cloud9_backend_url = $${node-frontend:access-url}
+cloud9_url = https://$${request-cloud9-frontend:connection-domain}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
+password_recovery_code = $${recovery-code:passwd}

software/slaprunner/instance-runner-import.cfg.in

+[buildout]
+extends = ${template-runner:output}
+          ${pbsready-import:output}
+
+parts +=
+  nginx_conf
+  nginx-launcher
+  cloud9
+  certificate-authority
+  ca-nginx
+  ca-node-frontend
+  slaprunner
+  test-runner
+  sshkeys-dropbear-runner
+  dropbear-server-add-authorized-key
+  sshkeys-authority
+  slaprunner-promise
+  cloud9-promise
+  dropbear-promise
+  symlinks
+  nginx-promise
+# have to repeat the next one, as it's not inherited from pbsready-import
+  import-on-notification
+
+[importer]
+recipe = slapos.cookbook:slaprunner.import
+wrapper = $${directory:bin}/$${slap-parameter:namebase}-importer
+srv-directory = $${directory:srv}
+etc-directory = $${directory:etc}
+backup-directory = $${directory:backup}
+shell-binary = ${dash:location}/bin/dash
+rsync-binary = ${rsync:location}/bin/rsync
+curl-binary = ${curl:location}/bin/curl
+backend-url = $${slaprunner:access-url}

software/slaprunner/instance-runner.cfg

 [buildout]
 parts =
+  nginx_conf
+  nginx-launcher
   cloud9
+  certificate-authority
+  ca-nginx
+  ca-node-frontend
   slaprunner
   test-runner
-  sshkeys-dropbear
+  sshkeys-dropbear-runner
   dropbear-server-add-authorized-key
   sshkeys-authority
   publish-connection-informations
   slaprunner-promise
   slaprunner-frontend-promise
   cloud9-promise
+  cloud9-frontend-promise
   dropbear-promise
   symlinks
-
+  request-cloud9-frontend
+  node-frontend-promise
+  nginx-promise
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -26,6 +34,7 @@ etc = $${buildout:directory}/etc/
 var = $${buildout:directory}/var/
 srv = $${buildout:directory}/srv/
 bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
 
 sshkeys = $${:srv}/sshkeys
 services = $${:etc}/service/
@@ -36,6 +45,9 @@ run = $${:var}/run/
 backup = $${:srv}/backup/
 promises = $${:etc}/promise/
 test = $${:etc}/test/
+nginx-data = $${directory:srv}/nginx
+ca-dir = $${:srv}/ssl
+
 
 [runnerdirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -58,8 +70,8 @@ bytes = 4
 # Deploy cloud9 and slaprunner
 [cloud9]
 recipe = slapos.cookbook:cloud9
-ip = $${slap-network-information:global-ipv6}
-port = 30000
+ip = $${slap-network-information:local-ipv4}
+port = 4443
 wrapper = $${directory:services}/cloud9
 working-directory = $${runnerdirectory:home}
 git-binary = ${git:location}/bin/git
@@ -82,13 +94,13 @@ instance-directory = $${runnerdirectory:instance-root}
 etc_dir = $${directory:etc}
 log_dir =  $${directory:log}
 run_dir = $${directory:run}
-ssh_client = $${sshkeys-dropbear:wrapper}
-public_key = $${sshkeys-dropbear:public-key}
-private_key = $${sshkeys-dropbear:private-key}
+ssh_client = $${sshkeys-dropbear-runner:wrapper}
+public_key = $${sshkeys-dropbear-runner:public-key}
+private_key = $${sshkeys-dropbear-runner:private-key}
 ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
-runner_port = 50000
+runner_port = 50005
 partition-amount = $${slap-parameter:instance-amount}
 cloud9-url = $${cloud9:access-url}
 wrapper = $${directory:services}/slaprunner
@@ -121,56 +133,205 @@ keys-directory = $${sshkeys-directory:keys}
 wrapper = $${directory:services}/sshkeys_authority
 keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[dropbear-server]
+[dropbear-runner-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
-port = 2222
+port = 22222
 home = $${directory:ssh}
-wrapper = $${directory:bin}/raw_sshd
-shell = /bin/bash
+wrapper = $${directory:bin}/runner_sshd
+shell = ${bash:location}/bin/bash
 rsa-keyfile = $${directory:ssh}/server_key.rsa
 dropbear-binary = ${dropbear:location}/sbin/dropbear
 
-[sshkeys-dropbear]
+[sshkeys-dropbear-runner]
 <= sshkeys-authority
 recipe = slapos.cookbook:sshkeys_authority.request
 name = dropbear
 type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${directory:services}/sshd
+executable = $${dropbear-runner-server:wrapper}
+public-key = $${dropbear-runner-server:rsa-keyfile}.pub
+private-key = $${dropbear-runner-server:rsa-keyfile}
+wrapper = $${directory:services}/runner_sshd
 
 [dropbear-server-add-authorized-key]
-<= dropbear-server
+<= dropbear-runner-server
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+#---------------------
+#--
+#-- Set node frontend
+
+[node-frontend]
+launcher = $${directory:bin}/node-frontend
+ip = $${slap-network-information:global-ipv6}
+port = $${cloud9:port}
+access-url = https://[$${:ip}]:$${:port}
+
+[node-frontend-launcher]
+recipe = slapos.recipe.template:jinja2
+template = ${node-frontend-template:location}/${node-frontend-template:filename}
+rendered = $${node-frontend:launcher}
+mode = 700
+context =
+	key ip node-frontend:ip
+	key port node-frontend:port
+	key key ca-node-frontend:key-file
+	key certificate ca-node-frontend:cert-file
+	key backend_ip nginx-frontend:local-ip
+	key backend_port nginx-frontend:port
+	raw shell_path ${bash:location}/bin/bash
+	raw node_env ${buildout:parts-directory}:${npm-modules:location}/node_modules
+	raw node_path ${nodejs:location}/bin/node
+	raw conf_path ${simple-proxy:location}/${simple-proxy:filename}
+
+#---------------------------
+#--
+#-- Set nginx frontend
+
+[tempdirectory]
+recipe = slapos.cookbook:mkdirectory
+client_body_temp_path = $${directory:tmp}/client_body_temp_path
+proxy_temp_path = $${directory:tmp}/proxy_temp_path
+fastcgi_temp_path = $${directory:tmp}/fastcgi_temp_path
+uwsgi_temp_path = $${directory:tmp}/uwsgi_temp_path
+scgi_temp_path = $${directory:tmp}/scgi_temp_path
+
+[nginx-frontend]
+# Options
+nb_workers = 2
+# Network
+local-ip = $${slap-network-information:local-ipv4}
+port = 30001
+global-ip = $${slap-network-information:global-ipv6}
+global-port = $${slaprunner:runner_port}
+# Backend
+cloud9-ip = $${cloud9:ip}
+cloud9-port = $${cloud9:port}
+runner-ip = $${slaprunner:ipv4}
+runner-port = $${slaprunner:runner_port}
+# SSL
+ssl-certificate = $${ca-nginx:cert-file}
+ssl-key = $${ca-nginx:key-file}
+# Log
+path_pid = $${directory:run}/nginx.pid
+path_log = $${directory:log}/nginx.log
+path_access_log = $${directory:log}/nginx.access.log
+path_error_log = $${directory:log}/nginx.error.log
+path_tmp = $${buildout:directory}/tmp
+# Config files
+path_nginx_conf = $${directory:etc}/nginx.conf
+# Executables
+bin_nginx = ${nginx:location}/sbin/nginx
+bin_launcher = $${directory:bin}/launcher
+# Utils
+path_shell = ${dash:location}/bin/dash
+# Misc.
+etc_dir = $${directory:etc}
+
+
+[nginx_conf]
+recipe = slapos.recipe.template:jinja2
+template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
+rendered = $${nginx-frontend:path_nginx_conf}
+context =
+    section param_nginx_frontend nginx-frontend
+    section param_tempdir tempdirectory
+
+[nginx-launcher]
+recipe = slapos.recipe.template:jinja2
+template = ${template_launcher:location}/${template_launcher:filename}
+rendered = $${nginx-frontend:bin_launcher}
+mode = 700
+context =
+    section param_nginx_frontend nginx-frontend
+
+#--------------------
+#--
+#-- ssl certificates
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${directory:services}/certificate_authority
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[ca-nginx]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${cadirectory:certs}/nginx_frontend.key
+cert-file = $${cadirectory:certs}/nginx_frontend.crt
+executable = $${nginx-launcher:rendered}
+wrapper = $${directory:services}/nginx-frontend
+# Put domain name
+name = example.com
+
+[ca-node-frontend]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = $${cadirectory:certs}/nodejs.key
+cert-file = $${cadirectory:certs}/nodejs.crt
+executable = $${node-frontend-launcher:rendered}
+wrapper = $${directory:services}/node-frontend
+# Put domain name
+name = example.com
+
+#--------------------
+#--
+#-- Request frontend
 
-# Request frontend
 [request-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
-name = Frontend
+name = SlapRunner Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
 config = url
 config-url = $${slaprunner:access-url}
-return = site_url
+return = site_url domain
+
+[request-cloud9-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Cloud9 Frontend
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url
+config-url = $${node-frontend:access-url}
+return = site_url domain
 
+#--------------------------------------
+#--
+#-- Send informations to SlapOS Master
 
-# Send informations to SlapOS Master
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
+1_info = Set your passord in slaprunner in order to access cloud9
 backend_url = $${slaprunner:access-url}
-url =  $${request-frontend:connection-site_url}
-cloud9_url = $${cloud9:access-url}
-ssh_command = ssh $${dropbear-server:host} -p $${dropbear-server:port}
+url =  https://$${request-frontend:connection-domain}
+cloud9_backend_url = $${node-frontend:access-url}
+cloud9_url = https://$${request-cloud9-frontend:connection-domain}
+ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
 password_recovery_code = $${recovery-code:passwd}
 
+#---------------------------
+#--
+#-- Deploy promises scripts
 
-# Deploy promises scripts
 [slaprunner-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/slaprunner
@@ -180,7 +341,7 @@ port = $${slaprunner:runner_port}
 [slaprunner-frontend-promise]
 recipe = slapos.cookbook:check_url_available
 path = $${directory:promises}/slaprunner_frontend
-url = $${request-frontend:connection-site_url}
+url = https://$${request-frontend:connection-domain}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
@@ -191,11 +352,31 @@ url = http://$${cloud9:ip}:$${cloud9:port}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
+[cloud9-frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${directory:promises}/cloud9-frontend-promise
+url = $${publish-connection-informations:cloud9_url}
+check-secure = 1
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[node-frontend-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promises}/node-frontend
+hostname = $${node-frontend:ip}
+port = $${node-frontend:port}
+
+[nginx-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promises}/nginx
+hostname = $${nginx-frontend:local-ip}
+port = $${nginx-frontend:port}
+
 [dropbear-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/dropbear
-hostname = $${dropbear-server:host}
-port = $${dropbear-server:port}
+hostname = $${dropbear-runner-server:host}
+port = $${dropbear-runner-server:port}
 
 [symlinks]
 recipe = cns.recipe.symlink

software/slaprunner/instance.cfg

@@ -4,8 +4,38 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = ${template-runner:output}
+resilient = $${instance-resilient:rendered}
+runner = ${template-runner:output}
+runner-import = ${instance-runner-import:output}
+runner-export = ${instance-runner-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+
+[instance-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${template-resilient:target}
+rendered = $${buildout:directory}/instance-resilient.cfg
+extensions = jinja2.ext.do
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-parameters:configuration
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+
+mode = 0644
+
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
\ No newline at end of file

software/slaprunner/launcher.in

+#! {{ param_nginx_frontend['path_shell'] }}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+# Run nginx
+exec {{ param_nginx_frontend['bin_nginx'] }} -c {{ param_nginx_frontend['path_nginx_conf'] }}

software/slaprunner/nginx_conf.in

+worker_processes {{ param_nginx_frontend['nb_workers'] }};
+
+pid {{ param_nginx_frontend['path_pid'] }};
+error_log {{ param_nginx_frontend['path_error_log'] }};
+
+daemon off;
+
+events {
+	worker_connections 1024;
+	accept_mutex off;
+}
+
+http {
+     default_type application/octet-stream;
+     access_log {{ param_nginx_frontend['path_access_log'] }} combined;
+     map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+     }
+     server {
+        listen {{ param_nginx_frontend['local-ip'] }}:{{ param_nginx_frontend['port'] }};
+        server_name _;
+
+        keepalive_timeout 90s;
+        client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
+        proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
+        fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
+        uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
+        scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
+        location / {
+            auth_basic  "Restricted";
+            auth_basic_user_file   {{ param_nginx_frontend['etc_dir'] }}/.htpasswd;
+            proxy_pass  http://{{ param_nginx_frontend['cloud9-ip'] }}:{{ param_nginx_frontend['cloud9-port'] }};
+            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+            proxy_redirect off;
+            proxy_buffering off;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header        Host            $host;
+            proxy_set_header        X-Real-IP       $remote_addr;
+            proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+     server {
+        listen [{{ param_nginx_frontend['global-ip'] }}]:{{ param_nginx_frontend['global-port'] }} ssl;
+        server_name _;
+        ssl_certificate     {{ param_nginx_frontend['ssl-certificate'] }};
+        ssl_certificate_key {{ param_nginx_frontend['ssl-key'] }};
+        ssl_protocols       SSLv3 TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+        keepalive_timeout 90s;
+        client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
+        proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
+        fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
+        uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
+        scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
+        location / {
+            proxy_pass  http://{{ param_nginx_frontend['runner-ip'] }}:{{ param_nginx_frontend['runner-port'] }};
+            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+            proxy_redirect off;
+            proxy_buffering off;
+            proxy_set_header        Host              $host;
+            proxy_set_header        X-Real-IP         $remote_addr;
+            proxy_set_header        X-Forwarded-Proto $scheme;
+            proxy_set_header        X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header        X-Forwarded-Host  $http_host;
+        }
+    }
+}

software/slaprunner/node-frontend.in

+#!{{ shell_path }}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+export NODE_PATH={{ node_env }}
+exec {{ node_path }} {{ conf_path }} {{ ip }} {{ port }} {{ key }} {{ certificate }} {{ backend_ip }} {{ backend_port }}
\ No newline at end of file

software/slaprunner/simple-proxy.js

+/*****************************************************************************
+*
+* Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+*
+* WARNING: This program as such is intended to be used by professional
+* programmers who take the whole responsibility of assessing all potential
+* consequences resulting from its eventual inadequacies and bugs
+* End users who are looking for a ready-to-use solution with commercial
+* guarantees and support are strongly adviced to contract a Free Software
+* Service Company
+*
+* This program is Free Software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 3
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+*
+*****************************************************************************/
+
+var fs = require('fs'),
+    util = require('util'),
+    colors = require('colors'),
+    http = require('http'),
+    httpProxy = require('http-proxy');
+
+
+var listenInterface = process.argv[2],
+    port = process.argv[3],
+    sslKeyFile = process.argv[4],
+    sslCertFile = process.argv[5],
+    backendIp = process.argv[6],
+    backendPort = process.argv[7];
+
+if (process.argv.length < 8) {
+  console.error("Too few arguments. Exiting.");
+  process.exit(1);
+}
+
+var middleware = function (req, res, proxy) {
+  return proxy.proxyRequest(req, res,{
+    host: backendIp,
+    port: backendPort
+  });
+};
+
+middleware.proxyWebSocketRequest = function (req, socket, head, proxy) {
+  return proxy.proxyWebSocketRequest(req, socket, head,{
+    host: backendIp,
+    port: backendPort
+  });
+};
+
+/**
+ * Create server
+ */
+var proxyServer = httpProxy.createServer(
+  middleware,
+  {
+    https: {
+      key: fs.readFileSync(
+        sslKeyFile,
+        'utf8'
+      ),
+      cert: fs.readFileSync(
+        sslCertFile,
+        'utf8'
+      )
+    },
+    source: {
+    host: listenInterface,
+    port: port
+  }}
+);
+
+console.log('HTTPS server starting and trying to listen on ' +
+            listenInterface + ':' + port);
+
+// Release the beast.
+proxyServer.listen(port, listenInterface);

software/slaprunner/software.cfg

@@ -55,17 +55,26 @@ meld3 = 0.6.10
 netaddr = 0.7.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
-pytz = 2012j
-slapos.cookbook = 0.71.1
-slapos.core = 0.34
-slapos.libnetworkcache = 0.13.3
+pytz = 2013b
+#slapos.cookbook = 0.71.1
+slapos.core = 0.35.1
+slapos.libnetworkcache = 0.13.4
 slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.34.0
+#slapos.toolbox = 0.34.0
 smmap = 0.8.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
-
+lock-file = 2.0
+rdiff-backup = 1.0.5
+slapos.recipe.cmmi = 0.2
+slapos.recipe.download = 1.0.dev-r4053
+slapos.toolbox = 0.35.1
+slapos.cookbook = 0.78.5
+cliff = 1.4
+cmd2 = 0.6.6
+prettytable = 0.7.2
+requests = 1.2.3
 # Required by:
 # slapos.core==0.34
 # slapos.toolbox==0.34.0

software/trac-svn/instance-trac.cfg

@@ -308,9 +308,8 @@ githttpbackend = ${git:location}/libexec/git-core/git-http-backend
 base-directory = $${trac-config:project_dir}/git
 
 [trac-admin]
-recipe = slapos.cookbook:pwgen
-file = $${buildout:directory}/.password
-pwgen-binary = ${pwgen:location}/bin/pwgen
+recipe = slapos.cookbook:generate.password
+storage-path = $${buildout:directory}/.password
 user = TracAdmin
 
 #---------------------
@@ -330,7 +329,7 @@ eggs-dirs =
 python-lib = ${python2.7:location}/lib
 trac-admin = ${buildout:bin-directory}/trac-admin
 admin-user = $${trac-admin:user}
-admin-password = $${trac-admin:password}
+admin-password = $${trac-admin:passwd}
 #MySQL informations
 mysql-username = $${mariadb-urlparse:username}
 mysql-password = $${mariadb-urlparse:password}
@@ -401,7 +400,7 @@ port = 9000
 shell = $${shell:wrapper}
 wrapper = $${rootdirectory:bin}/shellinaboxd_raw
 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
-password = $${trac-admin:password}
+password = $${trac-admin:passwd}
 directory = $${inittrac:site-dir}
 login-shell = $${rootdirectory:bin}/login
 certificate-directory = $${directory:shellinabox}
@@ -454,7 +453,7 @@ frontend_url = $${request-frontend:connection-site_url}
 git = $${request-frontend:connection-site_url}git/
 svn = $${request-frontend:connection-site_url}svn/
 admin_user = $${trac-admin:user}
-admin_password = $${trac-admin:password}
+admin_password = $${trac-admin:passwd}
 admin_shell = https://[$${shellinabox:ipv6}]:$${shellinabox:port}/
 
 #----------------

software/trac-svn/software.cfg

@@ -41,7 +41,6 @@ extends =
   ../../component/lxml-python/buildout.cfg
   ../../component/mysql-python/buildout.cfg
   ../../component/git/buildout.cfg
-  ../../component/pwgen/buildout.cfg
   ../../component/shellinabox/buildout.cfg
   ../../component/perl/buildout.cfg
 

software/varnish/instance-varnish-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "extends": "./schema-definitions.json#",
+  "properties": {
+    "tcpv4-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
+        "description": "Start allocating ports at this value, going upward",
+        "default": 6001
+      }]
+    },
+    "backend-url": {
+      "description": "The backend url that varnish will cache",
+      "type": "string"
+    },
+    "web-checker": {
+      "description": "Controls automated cache checker, disabled if null or empty"
+      "properties": {
+        "frontend-url": {
+          "description": "Override entry-point-url web checker will check the HTTP headers of all links in the web site, '%(ip)s' and '%(port)s' being substituted with varnish's listening ip and port, respectively",
+          "default": "http://%(ip)s:%(port)s/",
+          "type": "string"
+        },
+        "mail-address": {
+          "description": "Email address to which web checker result is sent",
+          "type": "string"
+        },
+        "smtp-host": {
+          "description": "The smtp server to be used to send the web checker result",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    }
+  }
+}

software/varnish/instance-varnish-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Varnish instanciation",
+  "properties": {
+    "url": {
+      "description": "Varnish HTTP service access information",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/varnish/instance-varnish.cfg.in

+{% set tcpv4_port = slapparameter_dict.get('tcpv4_port', 6001) -%}
+{% set ip = (ipv4_set | list)[0] %}
+[buildout]
+parts =
+  publish-varnish-connection-information
+  varnish-instance
+  cron
+  cron-entry-logrotate
+  {# When web_checker related parameter is given, web_checker will be enabled.-#}
+{% set web_checker_dict = slapparameter_dict.get('web-checker', {}) -%}
+{% if web_checker_dict -%}
+  web-checker
+  cron-entry-web-checker
+  logrotate-entry-web-checker
+
+[cron-entry-web-checker]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = web-checker
+frequency = 0 0 * * *
+command = ${varnish-instance:web-checker} ${web-checker:web-checker-config}
+
+[web-checker]
+recipe = slapos.cookbook:webchecker
+web-checker-config = ${rootdirectory:etc}/web_checker.cfg
+web-checker-working-directory = ${directory:web-checker}
+frontend-url = {{ web_checker_dict.get('frontend-url', 'http://%(ip)s:%(port)s/') % {
+  'ip': ip,
+  'port': tcpv4_port,
+} }}
+mail-address = {{ web_checker_dict['mail-address'] }}
+smtp-host = {{ web_checker_dict['smtp-host'] }}
+wget-binary-path = {{ parameter_dict['wget'] }}/bin/wget
+varnishlog-binary-path = ${varnish-instance:varnishlog-wrapper}
+web-checker-log = ${basedirectory:log}/web-checker.log
+
+[logrotate-entry-web-checker]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = web-checker
+log = ${web-checker:web-checker-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+{%- endif %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[publish-varnish-connection-information]
+recipe = slapos.cookbook:publish.serialised
+url = http://${varnish-instance:ip}:${varnish-instance:server-port}/
+
+[varnish-instance]
+recipe = slapos.cookbook:generic.varnish
+backend-url = {{ slapparameter_dict['backend-url'] }}
+
+# Network options
+ip = {{ ip }}
+server-port =  {{ tcpv4_port }}
+manager-port = {{ tcpv4_port + 1 }}
+stunnel-port = {{ tcpv4_port + 2}}
+
+# Paths: Running wrappers
+varnishd-wrapper = ${basedirectory:services}/varnishd
+varnishlog-wrapper = ${rootdirectory:bin}/varnishlog
+stunnel-wrapper = ${basedirectory:services}/stunnel
+
+# Binary information
+varnishd-binary = {{ parameter_dict['varnish'] }}/sbin/varnishd
+varnishlog-binary = {{ parameter_dict['varnish'] }}/bin/varnishlog
+shell-path = {{ parameter_dict['dash'] }}/bin/dash
+stunnel-binary = {{ parameter_dict['stunnel'] }}/bin/stunnel
+gcc-location = {{ parameter_dict['gcc'] }}/bin
+# Configuration by VCL
+vcl-file = ${rootdirectory:etc}/default.vcl
+pid-file = ${basedirectory:run}/varnishd.pid
+stunnel-conf-file = ${rootdirectory:etc}/stunnel.conf
+stunnel-pid-file = ${basedirectory:run}/stunnel.pid
+varnish-data = ${directory:varnish-data}
+# this will pass at -n option
+varnish-instance-name = ${directory:varnish-instance}
+web-checker = {{ parameter_dict['buildout-bin-directory'] }}/web_checker_utility
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = {{ parameter_dict['dcron'] }}/sbin/crond
+cron-entries = ${directory:cron-entries}
+crontabs = ${directory:crontabs}
+cronstamps = ${directory:cronstamps}
+binary = ${basedirectory:services}/crond
+catcher = ${cron-simplelogger:wrapper}
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${rootdirectory:bin}/cron_simplelogger
+log = ${basedirectory:log}/cron.log
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = {{ parameter_dict['logrotate'] }}/usr/sbin/logrotate
+gzip-binary = {{ parameter_dict['gzip'] }}/bin/gzip
+gunzip-binary = {{ parameter_dict['gzip'] }}/bin/gunzip
+# Directories
+wrapper = ${rootdirectory:bin}/logrotate
+conf = ${rootdirectory:etc}/logrotate.conf
+logrotate-entries = ${directory:logrotate-entries}
+backup = ${directory:logrotate-backup}
+state-file = ${rootdirectory:srv}/logrotate.status
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = ${rootdirectory:etc}/run
+run = ${rootdirectory:var}/run
+backup = ${rootdirectory:srv}/backup
+log = ${rootdirectory:var}/log
+backup = ${rootdirectory:srv}/backup
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+varnish-data = ${rootdirectory:srv}/varnish
+varnish-instance = ${directory:varnish-data}/instance
+cron-entries = ${rootdirectory:etc}/cron.d
+crontabs = ${rootdirectory:etc}/crontabs
+cronstamps = ${rootdirectory:etc}/cronstamps
+logrotate-backup = ${basedirectory:backup}/logrotate
+logrotate-entries = ${rootdirectory:etc}/logrotate.d
+web-checker = ${rootdirectory:srv}/web-checker
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+srv = ${buildout:directory}/srv
+bin = ${buildout:directory}/bin

software/varnish/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+extra-context =
+context =
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    key ipv4_set slap-configuration:ipv4
+    ${:extra-context}
+
+[dynamic-template-varnish-parameters]
+dash = {{ dash_location }}
+dcron = {{ dcron_location }}
+gcc =  {{ gcc_location }}
+gzip =  {{ gzip_location }}
+logrotate = {{ logrotate_location }}
+stunnel = {{ stunnel_location }}
+varnish = {{ varnish_location }}
+wget = {{ wget_location }}
+buildout-bin-directory = {{ buildout_bin_directory }}
+
+[dynamic-template-varnish]
+< = jinja2-template-base
+template = {{ template_varnish }}
+filename = instance-varnish.cfg
+extra-context =
+    section parameter_dict dynamic-template-varnish-parameters
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${dynamic-template-varnish:rendered}
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = ${slap_connection:computer_id}
+partition-id = ${slap_connection:partition_id}
+server-url = ${slap_connection:server_url}
+software-release-url = ${slap_connection:software_release_url}
+key-file = ${slap_connection:key_file}
+cert-file = ${slap_connection:cert_file}

software/varnish/software.cfg

+[buildout]
+# Local development
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+
+extensions =
+  slapos.zcbworkarounds
+  mr.developer
+
+find-links =
+    http://www.nexedi.org/static/packages/source/slapos.buildout/
+    http://www.nexedi.org/static/packages/source/hexagonit.recipe.download/
+    http://dist.repoze.org
+    http://www.nexedi.org/static/packages/source/
+
+extends =
+  ../../stack/slapos.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gcc/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/stunnel/buildout.cfg
+  ../../component/varnish/buildout.cfg
+  ../../component/wget/buildout.cfg
+
+parts =
+  dash
+  dcron
+  gcc-minimal
+  slapos-toolbox
+  stunnel
+  varnish-3.0
+  wget
+# Local development
+  slapos.cookbook-repository
+  check-recipe
+# Create instance template
+  template
+
+# Local development
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[template-jinja2-base]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:directory}/${:filename}
+# XXX: extra-context is needed because we cannot append to a key of an extended
+# section.
+extra-context =
+context =
+    key bin_directory buildout:bin-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    ${:extra-context}
+
+[template]
+< = template-jinja2-base
+# XXX: "template.cfg" is hardcoded in instanciation recipe
+filename = template.cfg
+template = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 8e906d749e19ee13fe5b7f4d9bfcf896
+extra-context =
+    key buildout_bin_directory buildout:bin-directory
+    key dash_location dash:location
+    key dcron_location dcron:location
+    key gcc_location gcc-minimal:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+    key stunnel_location stunnel:location
+    key template_varnish template-varnish:target
+    key varnish_location varnish-3.0:location
+    key wget_location wget:location
+
+[template-varnish]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-varnish.cfg.in
+md5sum = 4334d900f212d170fd0ca35865879bdf
+mode = 640
+
+[eggs]
+recipe = zc.recipe.egg
+python = python2.7
+eggs =
+  ${lxml-python:egg}
+  erp5.util
+  pytz
+  lock_file
+  inotifyx
+scripts =
+  web_checker_utility = erp5.util.webchecker:web_checker_utility
+
+[lxml-python]
+python = python2.7
+
+[slapos-toolbox]
+recipe = zc.recipe.egg
+python = ${eggs:python}
+eggs =
+  ${lxml-python:egg}
+  slapos.toolbox
+scripts =
+  killpidfromfile
+
+[versions]
+erp5.util = 0.4.34
+lxml = 2.3.6
+slapos.toolbox = 0.33.1
+
+[networkcache]
+# signature certificates of the following uploaders.
+#   Romain Courteaud
+#   Sebastien Robin
+#   Kazuhiko Shiozaki
+#   Cedric de Saint Martin
+#   Yingjie Xu
+#   Gabriel Monnerat
+#   Łukasz Nowak
+#   Test Agent (Automatic update from tests)
+#   Aurélien Calonne
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+  QUUGLQ==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB8jCCAVugAwIBAgIJAPu2zchZ2BxoMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV
+  BAMMB3RzeGRldjMwHhcNMTExMDE0MTIxNjIzWhcNMTIxMDEzMTIxNjIzWjASMRAw
+  DgYDVQQDDAd0c3hkZXYzMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCrPbh+
+  YGmo6mWmhVb1vTqX0BbeU0jCTB8TK3i6ep3tzSw2rkUGSx3niXn9LNTFNcIn3MZN
+  XHqbb4AS2Zxyk/2tr3939qqOrS4YRCtXBwTCuFY6r+a7pZsjiTNddPsEhuj4lEnR
+  L8Ax5mmzoi9nE+hiPSwqjRwWRU1+182rzXmN4QIDAQABo1AwTjAdBgNVHQ4EFgQU
+  /4XXREzqBbBNJvX5gU8tLWxZaeQwHwYDVR0jBBgwFoAU/4XXREzqBbBNJvX5gU8t
+  LWxZaeQwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQA07q/rKoE7fAda
+  FED57/SR00OvY9wLlFEF2QJ5OLu+O33YUXDDbGpfUSF9R8l0g9dix1JbWK9nQ6Yd
+  R/KCo6D0sw0ZgeQv1aUXbl/xJ9k4jlTxmWbPeiiPZEqU1W9wN5lkGuLxV4CEGTKU
+  hJA/yXa1wbwIPGvX3tVKdOEWPRXZLg==
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB7jCCAVegAwIBAgIJAJWA0jQ4o9DGMA0GCSqGSIb3DQEBBQUAMA8xDTALBgNV
+  BAMMBHg2MXMwIBcNMTExMTI0MTAyNDQzWhgPMjExMTEwMzExMDI0NDNaMA8xDTAL
+  BgNVBAMMBHg2MXMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANdJNiFsRlkH
+  vq2kHP2zdxEyzPAWZH3CQ3Myb3F8hERXTIFSUqntPXDKXDb7Y/laqjMXdj+vptKk
+  3Q36J+8VnJbSwjGwmEG6tym9qMSGIPPNw1JXY1R29eF3o4aj21o7DHAkhuNc5Tso
+  67fUSKgvyVnyH4G6ShQUAtghPaAwS0KvAgMBAAGjUDBOMB0GA1UdDgQWBBSjxFUE
+  RfnTvABRLAa34Ytkhz5vPzAfBgNVHSMEGDAWgBSjxFUERfnTvABRLAa34Ytkhz5v
+  PzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAFLDS7zNhlrQYSQO5KIj
+  z2RJe3fj4rLPklo3TmP5KLvendG+LErE2cbKPqnhQ2oVoj6u9tWVwo/g03PMrrnL
+  KrDm39slYD/1KoE5kB4l/p6KVOdeJ4I6xcgu9rnkqqHzDwI4v7e8/D3WZbpiFUsY
+  vaZhjNYKWQf79l6zXfOvphzJ
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAIlBksrZVkK8MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMzU3MCAXDTEyMDEyNjEwNTUyOFoYDzIxMTIwMTAyMTA1NTI4WjAT
+  MREwDwYDVQQDDAhDT01QLTM1NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ts+iGUwi44vtIfwXR8DCnLtHV4ydl0YTK2joJflj0/Ws7mz5BYkxIU4fea/6+VF3
+  i11nwBgYgxQyjNztgc9u9O71k1W5tU95yO7U7bFdYd5uxYA9/22fjObaTQoC4Nc9
+  mTu6r/VHyJ1yRsunBZXvnk/XaKp7gGE9vNEyJvPn2bkCAwEAAaNQME4wHQYDVR0O
+  BBYEFKuGIYu8+6aEkTVg62BRYaD11PILMB8GA1UdIwQYMBaAFKuGIYu8+6aEkTVg
+  62BRYaD11PILMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAMoTRpBxK
+  YLEZJbofF7gSrRIcrlUJYXfTfw1QUBOKkGFFDsiJpEg4y5pUk1s5Jq9K3SDzNq/W
+  it1oYjOhuGg3al8OOeKFrU6nvNTF1BAvJCl0tr3POai5yXyN5jlK/zPfypmQYxE+
+  TaqQSGBJPVXYt6lrq/PRD9ciZgKLOwEqK8w=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAPHoWu90gbsgMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCXZpZmlibm9kZTAeFw0xMjAzMTkyMzIwNTVaFw0xMzAzMTkyMzIwNTVaMBQx
+  EjAQBgNVBAMMCXZpZmlibm9kZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  ozBijpO8PS5RTeKTzA90vi9ezvv4vVjNaguqT4UwP9+O1+i6yq1Y2W5zZxw/Klbn
+  oudyNzie3/wqs9VfPmcyU9ajFzBv/Tobm3obmOqBN0GSYs5fyGw+O9G3//6ZEhf0
+  NinwdKmrRX+d0P5bHewadZWIvlmOupcnVJmkks852BECAwEAAaNQME4wHQYDVR0O
+  BBYEFF9EtgfZZs8L2ZxBJxSiY6eTsTEwMB8GA1UdIwQYMBaAFF9EtgfZZs8L2ZxB
+  JxSiY6eTsTEwMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc43YTfc6
+  baSemaMAc/jz8LNLhRE5dLfLOcRSoHda8y0lOrfe4lHT6yP5l8uyWAzLW+g6s3DA
+  Yme/bhX0g51BmI6gjKJo5DoPtiXk/Y9lxwD3p7PWi+RhN+AZQ5rpo8UfwnnN059n
+  yDuimQfvJjBFMVrdn9iP6SfMjxKaGk6gVmI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAMNZBmoIOXPBMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMTMyMCAXDTEyMDUwMjEyMDQyNloYDzIxMTIwNDA4MTIwNDI2WjAT
+  MREwDwYDVQQDDAhDT01QLTEzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  6peZQt1sAmMAmSG9BVxxcXm8x15kE9iAplmANYNQ7z2YO57c10jDtlYlwVfi/rct
+  xNUOKQtc8UQtV/fJWP0QT0GITdRz5X/TkWiojiFgkopza9/b1hXs5rltYByUGLhg
+  7JZ9dZGBihzPfn6U8ESAKiJzQP8Hyz/o81FPfuHCftsCAwEAAaNQME4wHQYDVR0O
+  BBYEFNuxsc77Z6/JSKPoyloHNm9zF9yqMB8GA1UdIwQYMBaAFNuxsc77Z6/JSKPo
+  yloHNm9zF9yqMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAl4hBaJy1
+  cgiNV2+Z5oNTrHgmzWvSY4duECOTBxeuIOnhql3vLlaQmo0p8Z4c13kTZq2s3nhd
+  Loe5mIHsjRVKvzB6SvIaFUYq/EzmHnqNdpIGkT/Mj7r/iUs61btTcGUCLsUiUeci
+  Vd0Ozh79JSRpkrdI8R/NRQ2XPHAo+29TT70=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
+  MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  o7aipd6MbnuGDeR1UJUjuMLQUariAyQ2l2ZDS6TfOwjHiPw/mhzkielgk73kqN7A
+  sUREx41eTcYCXzTq3WP3xCLE4LxLg1eIhd4nwNHj8H18xR9aP0AGjo4UFl5BOMa1
+  mwoyBt3VtfGtUmb8whpeJgHhqrPPxLoON+i6fIbXDaUCAwEAAaNQME4wHQYDVR0O
+  BBYEFEfjy3OopT2lOksKmKBNHTJE2hFlMB8GA1UdIwQYMBaAFEfjy3OopT2lOksK
+  mKBNHTJE2hFlMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAaNRx6YN2
+  M/p3R8/xS6zvH1EqJ3FFD7XeAQ52WuQnKSREzuw0dsw12ClxjcHiQEFioyTiTtjs
+  5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
+  x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB+DCCAWGgAwIBAgIJAKGd0vpks6T/MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+  BAMMCUNPTVAtMTU4NDAgFw0xMzA2MjAxMjE5MjBaGA8yMTEzMDUyNzEyMTkyMFow
+  FDESMBAGA1UEAwwJQ09NUC0xNTg0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+  gQDZTH9etPUC+wMZQ3UIiOwyyCfHsJ+7duCFYjuo1uZrhtDt/fp8qb8qK9ob+df3
+  EEYgA0IgI2j/9jNUEnKbc5+OrfKznzXjrlrH7zU8lKBVNCLzQuqBKRNajZ+UvO8R
+  nlqK2jZCXP/p3HXDYUTEwIR5W3tVCEn/Vda4upTLcPVE5wIDAQABo1AwTjAdBgNV
+  HQ4EFgQU7KXaNDheQWoy5uOU01tn1M5vNkEwHwYDVR0jBBgwFoAU7KXaNDheQWoy
+  5uOU01tn1M5vNkEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQASmqCU
+  Znbvu6izdicvjuE3aKnBa7G++Fdp2bdne5VCwVbVLYCQWatB+n4crKqGdnVply/u
+  +uZ16u1DbO9rYoKgWqjLk1GfiLw5v86pd5+wZd5I9QJ0/Sbz2vZk5S4ciMIGwArc
+  m711+GzlW5xe6GyH9SZaGOPAdUbI6JTDwLzEgA==
+  -----END CERTIFICATE-----

stack/cloudooo.cfg

@@ -49,6 +49,7 @@ extends =
   ../component/dcron/buildout.cfg
   ../component/coreutils/buildout.cfg
   ../component/cloudooo/buildout.cfg
+  ../component/haproxy/buildout.cfg
  
 versions = versions
 
@@ -77,6 +78,7 @@ parts =
   ffmpeg
 
   rdiff-backup
+  haproxy
   cloudooo
 
 # Local development

stack/erp5/buildout.cfg

@@ -19,7 +19,7 @@ allow-hosts += pybrary.net
 
 extends =
 # Exact version of Zope
-  https://raw.github.com/zopefoundation/Zope/2.13.20/versions.cfg
+  https://raw.github.com/zopefoundation/Zope/2.13.21/versions.cfg
   ../../stack/slapos.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -284,7 +284,7 @@ md5sum = 4d2207931975e580d04d0b0506034564
 [template-zope]
 < = download-base
 filename = instance-zope.cfg.in
-md5sum = f6d977cbd03b6f7346113a09e5578757
+md5sum = 5be0b7da2ce4957ab990e2bdd9b54928
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat

stack/erp5/instance-zope.cfg.in

@@ -180,7 +180,7 @@ runzope-binary = {{ bin_directory }}/runzope
 bt5-repository-list =
 
 [deadlock-debugger-password]
-recipe = slapos.cookbook:pwgen.stable
+recipe = slapos.cookbook:generate.password
 
 [zope-conf-parameter-base]
 ip = {{ ipv4 }}
@@ -199,7 +199,7 @@ context =
   key instance directory:instance
   key instance_products directory:instance-products
   raw deadlock_path /manage_debug_threads
-  key deadlock_debugger_password deadlock-debugger-password:password
+  key deadlock_debugger_password deadlock-debugger-password:passwd
   key tidstorage_ip tidstorage:ipv4
   key tidstorage_port tidstorage:ipv4-port
   key promise_path erp5-promise:promise-path

stack/resilient/README.txt

@@ -127,6 +127,27 @@ wrapper. I suggest you only add options and specify your export/import recipe.
 
 
 
+
+Checking that it works
+----------------------
+
+To check that your software instance is resilient you can proceed this way:
+Once all instances are successfully deployed, go to your export instance, connect as the instance user and run:
+$ ~/bin/exporter
+It is the script responsible for triggering the resiliency stack on your instance. After doing a backup of your data, it will notify the pull-backup instances of a new backup, triggering the transfer of this data to the import instances.
+
+Once this script is run successfully, go to your import instance, connect as its instance user and check ~/srv/backup/"your sofwtare"/, the location of the data you wanted to receive. The last part of the resiliency is up to your import script.
+
+DEBUGGING:
+Here is a partial list of things you can check to understand what is causing the problem:
+
+- Check that your import script does not fail and successfully places your data in ~/srv/backup/"your software" (as the import instance user) by runnig:
+$ ~/bin/"your software"-exporter
+- Check the export instance script is run successfully as this instance user by running:
+$ ~/bin/exporter
+- Check the pull-instance system did its job by going to one of your pull-backup instance, connect as its user and check the log : ~/var/log/equeue.log
+
+
 -----------------------------------------------------------------------------------------
 
 Finally, instance-mysoftware-import.cfg.in and
@@ -193,12 +214,12 @@ parts +=
 
 {{ replicated.replicate("Name", "3",
                         "mysoftware-export", "mysoftware-import",
-                        "ArgLeader","ArgBackup") }}
+                        "ArgLeader","ArgBackup", slapparameter_dict=slapparameter_dict) }}
 
 and it'll expend into the sections require to request Name0, Name1 and Name2,
 backuped and resilient. The leader will expend the section [ArgLeader], backups
-will expend [ArgBackup]. If you don't need to specify any options, you can
-omit the last two arguments in replicate().
+will expend [ArgBackup]. slapparameter_dict is the dict containing the parameters given to the instance. If you don't need to specify any options, you can
+omit the last three arguments in replicate().
 
 Since you will compile your template with jinja2, there should be no $${},
 because it is not yet possible to use jinja2 -> buildout template.
@@ -206,3 +227,36 @@ because it is not yet possible to use jinja2 -> buildout template.
 To compile with jinja2, see jinja2's recipe.
 
 
+Deploying your resilient software
+---------------------------------
+You can provide sla parameters to each request you make (a lot: for export, import and pbs).
+
+example:
+Here is a small example of parameters you can provide to control the deployment (case of a runner):
+<?xml version='1.0' encoding='utf-8'?>
+<instance>
+  <parameter id="-sla-1-computer_guid">COMP-GRP1</parameter>
+  <parameter id="-sla-pbs1-computer_guid">COMP-PBS1</parameter>
+  <parameter id="-sla-2-computer_guid">COMP-GROUP2</parameter>
+  <parameter id="-sla-runner2-computer_guid">COMP-RUN2</parameter>
+  <parameter id="-sla-2-network_guid">NET-2</parameter>
+  <parameter id="-sla-runner0-computer_guid">COMP-RUN0</parameter>
+</instance>
+Consequence on sla parameters by request:
+  * runner0: computer_guid = COMP-RUN0 (provided directly)
+  * runner1: computer_guid = COMP-GRP1 (provided by group 1)
+  * runner2: computer_guid = COMP-RUN2 (provided by group 2 but overided directly)
+             network_guid  = NET-2 (provided by group 2)
+  * PBS 1:   computer_guid = COMP-PBS1 (provided by group 1 but overided directly)
+  * PBS 2:   computer_guid = COMP-GRP2 (provided by group 2)
+             network_guid  = NET-2 (provided by group 2)
+
+
+Parameters are analysed this way:
+ * If it starts with "-sla-" it is not transmitted to requested instance and is used to do the request as sla.
+ * -sla-foo-bar=example (foo being a magic key) will be use for each request considering "foo" as a key to use and the sla parameter is "bar". So for each group using the "foo" key, sla parameter "bar" is used with value "example"
+About magic keys:
+We can find 2 kinds of magic keys:
+ * id : example, in "-sla-2-foo" 2 is the magic key and the parameter will be used for each request with id 2 (in case of kvm: kvm2 and PBS 2)
+ * nameid : example, in "-sla-kvm2-foo", foo will be used for kvm2 request. Name for pbs is "pbs" -> "-sla-pbs2-foo".
+IMPORTANT NOTE: in case the same foo parameter is asked for the group, the nameid key prevail

stack/resilient/buildout.cfg

 [buildout]
+extends =
+  ../../component/dropbear/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/rsync/buildout.cfg
 
 parts =
+  rdiff-backup
   pbsready
   pbsready-import
   pbsready-export
   template-replicated
   template-parts
   instance-frozen
-  template-resilient
+
+  # needed tools for resiliency
+  gzip
 
 
 #----------------
@@ -31,7 +39,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = 1b1308fd39476d48b5ca13db48ea6dc9
+md5sum = 3c2e73f49abdc52282fc045e6d91f3e9
 mode = 0644
 
 [pbsready-export]
@@ -40,20 +48,20 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 5d9e20c436fd307e8e4ab224a9a65792
+md5sum = a0e22a5de727544c5767d6bee059a77a
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 453d96f5a6c1230c01c878cc7640bae6
+md5sum = c67a9dad66490ae264f9e7003521bf59
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = 9e20f283bf709c63c9c6692d5e1f8972
+#md5sum = 9e20f283bf709c63c9c6692d5e1f8972
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
@@ -70,12 +78,10 @@ destination = ${buildout:directory}/template-parts.cfg.in
 # which will run without removing any content because it raises an error.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-frozen.cfg.in
+md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 output = ${buildout:directory}/instance-frozen.cfg
 
-[template-resilient]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/resilient.cfg.in
-output = ${buildout:directory}/resilient.cfg
-md5sum = 59e74d290d623de2c1e147e48f284fba
-mode = 0644
+[versions]
+# Pin Jinja2 to 2.6, as 2.7 breaks current code
+Jinja2 = 2.6
 

stack/resilient/instance-frozen.cfg.in

 [buildout]
-  parts = 
+eggs-directory = ${buildout:eggs-directory}
\ No newline at end of file
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+parts = 
+

stack/resilient/instance-pull-backup.cfg.in

@@ -105,7 +105,8 @@ promises-directory = $${basedirectory:promises}
 directory = $${directory:pbs-backup}
 cron-entries = $${cron:cron-entries}
 wrappers-directory = $${directory:pbs-wrappers}
-notifier-url = http://[$${notifier:host}]:$${notifier:port}/
+# XXX: this should be named "notifier-host"
+notifier-url = http://[$${notifier:host}]:$${notifier:port}
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
 

stack/resilient/pbsready-export.cfg.in

@@ -2,7 +2,21 @@
 
 extends = ${pbsready:output}
 
-parts +=
+# Explicitely define extended parts from pbsready
+# then add local parts
+parts =
+  resiliency
+  logrotate
+  logrotate-entry-cron
+  logrotate-entry-equeue
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  dropbear-server
+  sshkeys-dropbear
+  dropbear-server-pbs-authorized-key
+  notifier
+
   cron-entry-backup
 
 [resilient-publish-connection-parameter]

stack/resilient/pbsready-import.cfg.in

@@ -2,7 +2,21 @@
 
 extends = ${pbsready:output}
 
-parts +=
+# Explicitely define extended parts from pbsready
+# then add local parts
+parts =
+  resiliency
+  logrotate
+  logrotate-entry-cron
+  logrotate-entry-equeue
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  dropbear-server
+  sshkeys-dropbear
+  dropbear-server-pbs-authorized-key
+  notifier
+
   import-on-notification
   resilient-publish-connection-parameter
 

stack/resilient/resilient.cfg.in

-## not used at the moment
-
-[buildout]
-
-parts =
-  request-pull-backup-server
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
\ No newline at end of file

stack/resilient/template-replicated.cfg.in

-{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
+{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='', slapparameter_dict={}) %}
+
+{% set sla_parameter_dict = {} -%}
+# prepare sla-parameters
+{% if slapparameter_dict is defined -%}
+{%   for key in slapparameter_dict.keys() -%}
+{%     if key.startswith('-sla-') -%}
+{%       do sla_parameter_dict.__setitem__(key, slapparameter_dict.pop(key)) -%}
+{%     endif -%}
+{%   endfor -%}
+{% endif -%}
+
 
 ## Tells the Backupable recipe that we want a backup
 [resilient]
@@ -18,12 +29,42 @@ software-type = {{typeexport}}
 name = {{namebase}}0
 return = ssh-public-key ssh-url notification-id ip
 
-config = number authorized-key notify ip-list namebase
+config =
+# Resilient related parameters
+  number authorized-key notify ip-list namebase
+{% if slapparameter_dict is defined %}
+# Software Instance related parameters
+  {% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
+{% endif %}
 config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
 config-ip-list =
-
+# Bubble up all the instance parameters to the requested export instance.
+{% if slapparameter_dict is defined %}
+{% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
+{% endfor %}
+{% endif %}
+{% if sla_parameter_dict -%}
+{%   set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%}
+{%   set sla_key_secondary = "-sla-%s-" % (0) -%}
+{%   set sla_key_main_length = sla_key_main | length -%}
+{%   set sla_key_secondary_length = sla_key_secondary | length -%}
+{%   set sla_dict = {} -%}
+{%   for key in sla_parameter_dict.keys() -%}
+{%     if key.startswith(sla_key_main) -%}
+{%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
+{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
+{%     endif -%}
+{%   endfor -%}
+{%   if sla_dict %}
+sla = {{ ' '.join(sla_dict.keys()) }}
+{%     for key, value in sla_dict.iteritems() -%}
+sla-{{ key }} = {{ value }}
+{%     endfor -%}
+{%   endif -%}
+{% endif -%}
 
 {% for id in range(1,nbbackup|int) %}
 
@@ -45,12 +86,29 @@ config-number = {{id}}
 config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
 config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
 config-ip-list =
-
-sla = computer_guid
-sla-computer_guid = ${slap-parameter:{{namebase}}{{id}}-computer-guid}
-
-
-{% endfor %}
+{% if sla_parameter_dict -%}
+{%   set sla_key_main = "-sla-%s%s-" % (namebase, id) -%}
+{%   set sla_key_secondary = "-sla-%s-" % (id) -%}
+{%   set sla_key_main_length = sla_key_main | length -%}
+{%   set sla_key_secondary_length = sla_key_secondary | length -%}
+{%   set sla_dict = {} -%}
+{%   for key in sla_parameter_dict.keys() -%}
+{%     if key.startswith(sla_key_main) -%}
+{%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
+{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
+{%     endif -%}
+{%   endfor -%}
+{%   if sla_dict %}
+sla = {{ ' '.join(sla_dict.keys()) }}
+{%     for key, value in sla_dict.iteritems() -%}
+sla-{{ key }} = {{ value }}
+{%     endfor -%}
+{%   endif %}
+{% endif %}
+
+
+{% endfor -%}
 
 [iplist]
 config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
@@ -90,8 +148,27 @@ software-type = pull-backup
 name = PBS ({{namebase}} / {{id}})
 return = ssh-key notification-url feeds-url
 slave = false
-sla = computer_guid
-sla-computer_guid = ${slap-parameter:pbs-{{namebase}}{{id}}-computer-guid}
+{% if sla_parameter_dict -%}
+{%   set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%}
+{%   set sla_key_secondary = "-sla-%s-" % (id) -%}
+{%   set sla_key_main_length = sla_key_main | length -%}
+{%   set sla_key_secondary_length = sla_key_secondary | length -%}
+{%   set sla_dict = {} -%}
+{%   for key in sla_parameter_dict.keys() -%}
+{%     if key.startswith(sla_key_main) -%}
+{%       do sla_dict.__setitem__(key[sla_key_main_length:], sla_parameter_dict.get(key)) -%}
+{%     elif key.startswith(sla_key_secondary) and not sla_dict.has_key(key[sla_key_secondary_length:]) -%}
+{%         do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
+{%     endif -%}
+{%   endfor -%}
+{%   if sla_dict %}
+sla = {{ ' '.join(sla_dict.keys()) }}
+{%     for key, value in sla_dict.iteritems() -%}
+sla-{{ key }} = {{ value }}
+{%     endfor %}
+{%   endif %}
+{% endif %}
+
 
 [request-pull-backup-server-{{namebase}}-{{id}}]
 <= request-pbs-common
@@ -135,3 +212,4 @@ pbs-{{namebase}}{{id}}-computer-guid =
 {% endfor %}
 
 {% endmacro %}
+

stack/slapos.cfg

@@ -78,6 +78,9 @@ zc.buildout = 1.6.0-dev-SlapOS-012
 zc.recipe.egg = 1.3.2
 # Use own version of h.r.download to be able to open xz-like archives
 hexagonit.recipe.download = 1.6nxd002
+# Use pinned version of setuptools. Other versions work, but changing
+# version makes buildout recompile everything. Developers' nightmare.
+setuptools = 0.9.8
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache