Merge branch 'master' into lamp

CHANGES.txt

 Changes
 =======
 
+0.71.1 (2013-01-04)
+-------------------
+
+ * Frontend: Sort instances by reference to avoid attacks. [Cedric de Saint
+   Martin]
+ * Frontend: Add public_ipv4 parameter support to ease deployment of slave
+   frontend. [Cedric de Saint Martin]
+ * Frontend: Move apache_frontend wrappers to watched directory (etc/service).
+   [Cedric de Saint Martin]
+ * Frontend: Add native path to varnish environment. [Cedric de Saint Martin]
+
+0.71 (2012-12-20)
+-----------------
+
+ * frontend: Add "path" parameter for Zope instances. [Cedric de Saint Martin]
+
+0.70 (2012-11-05)
+-----------------
+
+ * KVM: Add support for disk-type, second nbd and cpu-count. [Cedric de Saint
+   Martin]
+
+0.69 (2012-10-30)
+-----------------
+
+ * handle multiple notification_url values in notifier recipe [Marco Mariani]
+ * createWrapper() sh alternative to execute.execute() for simple cases
+   [Marco Mariani]
+ * fixed secret key generation in apachephp config [Marco Mariani]
+
+0.68.1 (2012-10-03)
+-------------------
+
+  * slaprunner: fix "logfile" parameter to "log_file"
+
+0.68 (2012-10-02)
+-----------------
+
+  * request.py: Remove useless calls to master, fix "update" method. [Cedric
+    de Saint Martin]
+  * Add webrunner test recipe. [Alain Takoudjou]
+  * Add logfile for slaprunner. [Cedric de Saint Martin]
+  * Fix check_url_available promise (syntax + checks + IPv6 support). [Cedric
+    de Saint Martin]
+
+0.67 (2012-09-26)
+-----------------
+
+  * Add check_page_content promise generator. [Cedric Le Ninivin]
+  * Fix check_url_available recipe. [Cedric de Saint Martin]
+  * Set up timezone database in mariab's mysql table so that we can use
+    timezone conversion function. [Kazuhiko Shiozaki]
+  * Add many resiliency-based recipes [Timothée Lacroix]
+  * Fix and unify request and requestoptional recipes [Cedric de Saint Martin]
+  * Fix Dropbear. [Antoine Catton]
+
 0.66 (2012-09-10)
 -----------------
 

component/apache-perl/buildout.cfg

+[buildout]
+parts = apache-perl
+
+extends =
+  ../apache/buildout.cfg
+  ../perl/buildout.cfg
+  ../libuuid/buildout.cfg
+
+[apache-perl]
+# Note: Shall react on each build of apache and reinstall itself
+recipe = hexagonit.recipe.cmmi
+url = http://perl.apache.org/dist/mod_perl-2.0.5.tar.gz
+md5sum = 03d01d135a122bd8cebd0cd5b185d674
+configure-command =
+  ${perl:location}/bin/perl Makefile.PL
+configure-options =
+  MP_AP_PREFIX=${apache-2.2:location}
+
+environment =
+  # CPPFLAGS=-I${libuuid:location}/include
+  MP_CCOPTS=-L${libuuid:location}/lib

component/apache-php/buildout.cfg

@@ -55,10 +55,50 @@ configure-options =
   --enable-bz2
   --enable-ftp
 
+
+# Changing TMPDIR is required for PEAR installation.
+# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
+# XXX we could mkdir tmp there
+
 environment =
   PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
   LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
+  TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
+
+
+
+
+[apache-php-postgres]
+<=apache-php
+configure-options =
+  --with-apxs2=${apache:location}/bin/apxs
+  --with-libxml-dir=${libxml2:location}
+  --with-zlib-dir=${zlib:location}
+  --with-bz2-dir=${bzip2:location}
+  --with-mcrypt=${libmcrypt:location}
+  --with-gd
+  --with-jpeg-dir=${libjpeg:location}
+  --with-png-dir=${libpng:location}
+  --enable-gd-native-ttf
+  --with-ttf
+  --with-freetype-dir=${freetype:location}
+  --with-curl=${curl:location}
+  --with-zip-dir=${zip:location}
+  --with-imap=${cclient:location}
+  --with-iconv-dir=${libiconv:location}
+  --with-gettext=${gettext:location}
+  --with-ldap=${openldap:location}
+  --with-imap-ssl
+  --with-openssl=${openssl:location}
+  --enable-libxml
+  --enable-mbstring
+  --enable-session
+  --enable-exif
+  --enable-zip
+  --enable-bz2
+  --enable-ftp
+  --with-pgsql=${postgresql:location}
 
 
 [libmcrypt]

component/apache/buildout.cfg

@@ -87,7 +87,7 @@ configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_na
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
-  CPPFLAGS =-I${libuuid:location}/include
+  CPPFLAGS =-I${libuuid:location}/include -I${openssl:location}/include
   LDFLAGS =-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gdbm:location}/lib
 
 [mod_antiloris-apache-2.4.patch]

component/busybox/buildout.cfg

@@ -5,7 +5,7 @@ parts = busybox
 [busybox]
 recipe = slapos.recipe.build
 url = http://git.busybox.net/busybox/snapshot/busybox-1_20_1.tar.gz
-md5sum = 15758fc37ae8051d6def1b8afb691821
+md5sum = 2dcfee8add6b9c52d6a91e97ba705b66
 script =
     extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
     workdir = guessworkdir(extract_dir)

component/cloud9/buildout.cfg

@@ -16,7 +16,7 @@ parts =
 
 [node-sm]
 recipe = slapos.recipe.build:npm
-packages = sm@0.2.7
+packages = sm@0.2.11
 node = nodejs
 environment =
   PATH=${nodejs:location}/bin:%(PATH)s
@@ -25,10 +25,10 @@ environment =
 # Online IDE written in javascript/node.js
 # URL : c9.io
 # You can use it using the following command :
-# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} 
+# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
 recipe = plone.recipe.command
 stop-on-error = true
-commit = 97db1467c517d265438684bd2a70b0b76ee282f6
+commit = 5d18344936baf1d86b0fa5fc2c690051b4c77cb2
 repository = https://github.com/ajaxorg/cloud9.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 git-binary = ${git:location}/bin/git
@@ -37,26 +37,33 @@ command = export GIT_SSL_NO_VERIFY=true; export HOME=${:location}; (${:git-binar
 update-command =
 executable = ${:location}/bin/cloud9.js
 
+[cloud9-session-directory.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = cloud9-session-directory.patch
+download-only = true
+md5sum = 5dc8cc28447ed3747b8a53c768d872aa
+
 [cloud9-git]
 # Online IDE written in javascript/node.js
 # URL : c9.io
 # You can use it using the following command :
-# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} 
+# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
 recipe = plone.recipe.command
 stop-on-error = true
-commit = c66284221143c175fc889418d499da6f37492a7c
+commit = 6b0112363dd0f9523430808394a162d3cc699550
 repository = https://github.com/ajaxorg/cloud9.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
-command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install) || (rm -fr ${:location}; exit 1)
+command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1)
 update-command =
 executable = ${:location}/server.js
 
 [cloud9-npm]
 # Online IDE written in javascript/node.js
-# URL : c9.io 
+# URL : c9.io
 # You can use it using the following command :
-# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} 
+# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
 recipe = slapos.recipe.npm
 # Node part has to be specified, otherwise system node is used.
 node = nodejs

component/cloud9/cloud9-session-directory.patch

+diff --git a/configs/default.js b/configs/default.js
+index 6d1c85f..be35b37 100644
+--- a/configs/default.js
++++ b/configs/default.js
+@@ -22,6 +22,8 @@ var vfsUrl = "/vfs";
+ var port = argv.p || process.env.PORT || 3131;
+ var host = argv.l || process.env.IP || "localhost";
+ 
++var home = process.env['HOME']
++
+ var config = [
+     {
+         packagePath: "connect-architect/connect",
+@@ -167,7 +169,7 @@ var config = [
+     },
+     {
+         packagePath: "connect-architect/connect.session.file",
+-        sessionsPath: __dirname + "/../.sessions"
++        sessionsPath: home + "/.sessions"
+     },
+     "./cloud9.permissions",
+     {
\ No newline at end of file

component/fonts/buildout.cfg

@@ -3,6 +3,7 @@ parts =
   liberation-fonts
   ipaex-fonts
   ipa-fonts
+  ocrb-fonts
 
 [fonts]
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -32,3 +33,10 @@ strip-top-level-dir = true
 url = http://info.openlab.ipa.go.jp/ipafont/fontdata/IPAfont00303.zip
 md5sum = 39a828acf27790adbe4944dfb4d94bb1
 destination = ${fonts:location}/${:_buildout_section_name_}
+
+[ocrb-fonts]
+recipe = hexagonit.recipe.download
+strip-top-level-dir = true
+url = http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Ftsukurimashou%2F56948%2Focr-0.2.zip
+md5sum = 9f2acd83291a31dbe053912f4115db75
+destination = ${fonts:location}/${:_buildout_section_name_}

component/gcc/buildout.cfg

@@ -170,4 +170,4 @@ make-targets = install -j1
 # until gcc will be simplified by using more robust build recipe (like
 # slapos.recipe.build) each time any of parts which reuses this one gets updated
 # the hack-revision have to be increased
-hack-revision = 1
+hack-revision = 2

component/ghostscript/buildout.cfg

@@ -29,5 +29,5 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.05.tar.bz2
-md5sum = 8bcef1f33ddf8a4d12b2cf8da385c191
+url = http://downloads.ghostscript.com/public/ghostscript-9.06.tar.bz2
+md5sum = 46f9ebe40dc52755287b30704270db11

component/git/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [git]
 recipe = hexagonit.recipe.cmmi
-url = http://git-core.googlecode.com/files/git-1.7.10.4.tar.gz
-md5sum = 68319d593d051ef76c26e945bbd2d7ac
+url = http://git-core.googlecode.com/files/git-1.7.12.tar.gz
+md5sum = ceb1a6b17a3e33bbc70eadf8fce5876c
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/groonga/buildout.cfg

@@ -7,12 +7,12 @@ parts =
 
 [groonga]
 recipe = hexagonit.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-2.0.5.tar.gz
-md5sum = 6ea6634ceee97f701add89ee39f4c1da
+url = http://packages.groonga.org/source/groonga/groonga-2.0.6.tar.gz
+md5sum = 64467d1ab8d5e0df8348a636972a95a7
 configure-options =
   --disable-static
   --disable-glibtest
   --disable-benchmark
   --disable-document
-  --disable-nginx
+  --disable-groonga-httpd
   --without-mecab

component/haproxy/buildout.cfg

@@ -9,8 +9,8 @@ parts = haproxy
 
 [haproxy]
 recipe = hexagonit.recipe.cmmi
-url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.21.tar.gz
-md5sum = f36d5d13fd4a44ab454eee5d06cb38a2
+url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.22.tar.gz
+md5sum = a0b007c76f6a78524f3b3dd5e704979c
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/imagemagick/buildout.cfg

@@ -28,7 +28,7 @@ filename = imagemagick-6.6.6-1-no-gsx-gsc-probe.patch
 
 [imagemagick]
 recipe = hexagonit.recipe.cmmi
-url = ftp://ftp.imagemagick.org/pub/ImageMagick/ImageMagick-6.7.8-8.tar.bz2
+url = http://ftp.vim.org/ImageMagick/ImageMagick-6.7.8-8.tar.bz2
 md5sum = 4e5c8f102f3e7401587c924f5b4bca15
 depends =
   ${libtiff:version}

component/java/buildout.cfg

@@ -29,9 +29,9 @@ slapos_promisee =
   directory:javaws
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual.jsp?locale=en
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59621 7466b0b86bef21e3e31ae578b2b9f472
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59623 c5cfb8393d257b51e7a70e56b7784ac9
+# http://java.com/en/download/manual_v6.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
 script =
   import glob
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
@@ -47,48 +47,20 @@ script =
   workdir = guessworkdir(extract_dir)
   self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
 
-[java-sdk-1.6.0]
+[java-re-1.7.0]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
   directory:lib
   directory:man
   directory:plugin
-  directory:javaws
-  file:jre/lib/rt.jar
-  file:bin/java
-x86 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-i586.bin bdb5f05bd20c6aa9a4729726191bf6fd
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-x64.bin 94f93a3ff03f824a238ecd79ad90433e
-script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0777)
-  subprocess.call(["patch", auto_extract_bin, "-i", "${jdk-6u27-no-user-interaction-patch:location}/${jdk-6u27-no-user-interaction-patch:filename}"])
-  subprocess.call([auto_extract_bin])
-  self.cleanup_dir_list.append(extract_dir)
-  workdir = guessworkdir(extract_dir)
-  self.copyTree(os.path.join(workdir, "jdk1.6.0_27"), "%(location)s")
-
-[java-sdk-1.7.0]
-recipe = slapos.recipe.build
-slapos_promisee =
-  directory:bin
-  directory:lib
-  directory:man
-  directory:jre
-  file:jre/lib/rt.jar
+  file:lib/rt.jar
   file:bin/java
-  file:bin/javac
-x86 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-i586.tar.gz f97244a104f03731e5ff69f0dd5a9927
-x86-64 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-x64.tar.gz b3c1ef5faea7b180469c129a49762b64
+# http://java.com/en/download/manual.jsp?locale=en
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
-

component/libdmtx/buildout.cfg

@@ -3,8 +3,10 @@
 
 [buildout]
 extends =
+  ../bzip2/buildout.cfg
   ../imagemagick/buildout.cfg
   ../jbigkit/buildout.cfg
+  ../zlib/buildout.cfg
 parts =
   libdmtx
   dmtx-utils
@@ -26,4 +28,4 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${imagemagick:location}/lib/pkgconfig:${libdmtx:location}/lib/pkgconfig
   CPPFLAGS=-I${libdmtx:location}/include
-  LDFLAGS=-Wl,-rpath=${jbigkit:location}/lib
+  LDFLAGS=-Wl,-rpath=${jbigkit:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${zlib:location}/lib

component/librsvg/buildout.cfg

@@ -14,7 +14,8 @@ url = http://ftp.gnome.org/pub/gnome/sources/libcroco/0.6/libcroco-0.6.3.tar.bz2
 md5sum = e1e93eeff4367c896f3959af34ba20eb
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${pkgconfig:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${pkgconfig:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+  LDFLAGS=-Wl,-rpath=${zlib:location}/lib
 
 [librsvg]
 recipe = hexagonit.recipe.cmmi
@@ -36,4 +37,4 @@ configure-options =
 environment = 
   PATH=${gdk-pixbuf:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${cairo:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
-  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${zlib:location}/lib

component/libtiff/buildout.cfg

@@ -9,11 +9,9 @@ parts =
 
 [libtiff]
 recipe = hexagonit.recipe.cmmi
-version = 4.0.2
-#url = http://download.osgeo.org/libtiff/tiff-${:version}.tar.gz
-# server is down - circumvent
-url = http://www.imagemagick.org/download/delegates/tiff-${:version}.tar.gz
-md5sum = 04a08fa1e07e696e820a0c3f32465a13
+version = 4.0.3
+url = http://download.osgeo.org/libtiff/tiff-${:version}.tar.gz
+md5sum = 051c1068e6a0627f461948c365290410
 configure-options =
   --disable-static
   --without-x

component/mariadb/buildout.cfg

@@ -59,9 +59,11 @@ environment =
   LDFLAGS=-L${libaio:location}/lib
 
 [mroonga-mariadb]
+# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
+# http://mroonga.github.com/
 recipe = hexagonit.recipe.cmmi
-url = https://github.com/downloads/mroonga/mroonga/mroonga-2.05.tar.gz
-md5sum = 6aee0e6ff948bba7c02854326afc11df
+url = https://github.com/downloads/mroonga/mroonga/mroonga-2.06.tar.gz
+md5sum = 89ce640a57ed8f2161b8358ff4c29c66
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/mydumper/buildout.cfg

@@ -3,6 +3,7 @@
 extends =
     ../cmake/buildout.cfg
     ../glib/buildout.cfg
+    ../pkgconfig/buildout.cfg
     ../openssl/buildout.cfg
     ../pcre/buildout.cfg
     ../mariadb/buildout.cfg
@@ -22,6 +23,7 @@ buildout-bin-dir = ${buildout:bin-directory}
 cmake-command = ${cmake:location}/bin/cmake
 mysql-config = ${mariadb:location}/bin/mysql_config
 mysqllib = ${mariadb:location}/lib
+path = ${pkgconfig:location}/bin
 pkg-config-path = ${glib:location}/lib/pkgconfig/:${pcre:location}/lib/pkgconfig/:${openssl:location}/lib/pkgconfig/
 libraries = ${zlib:location}/lib/:${glib:location}/lib/:${pcre:location}/lib/:${mariadb:location}/lib/:${openssl:location}/lib/
 includes = ${zlib:location}/include/:${glib:location}/include/:${pcre:location}/include/:${mariadb:location}/include:${openssl:location}/include/
@@ -39,7 +41,7 @@ script =
     extract_dir = self.extract(url)
     workdir = guessworkdir(extract_dir)
     self.applyPatchList(self.options['mydumper-patches'], cwd=workdir)
-    env['PATH'] = self.options['buildout-bin-dir'] + ':' + env.get('PATH', '')
+    env['PATH'] = self.options['path'] + ':' + self.options['buildout-bin-dir'] + ':' + env.get('PATH', '')
     env['PKG_CONFIG_PATH'] = self.options['pkg-config-path'] + ':' + \
                              env.get('PKG_CONFIG_PATH', '')
     env['CMAKE_INCLUDE_PATH'] = self.options['includes']

component/noVNC/buildout.cfg

@@ -4,6 +4,6 @@ parts =
 
 [noVNC]
 recipe = slapos.recipe.build:download-unpacked
-url = http://cloud.github.com/downloads/kanaka/noVNC/novnc-0.3.tar.gz
-md5sum = 95d3c58921fa188c179491e8ef2acc12
+url = http://cloud.github.com/downloads/kanaka/noVNC/novnc-0.4.tar.gz
+md5sum = 5703d5d46022d8723796dcbbf821ee7f
 strip-top-level-dir = true

component/nodejs/buildout.cfg

@@ -14,8 +14,8 @@ parts =
 [nodejs-0.8]
 # Server-side Javascript.
 recipe = hexagonit.recipe.cmmi
-url = http://nodejs.org/dist/v0.8.8/node-v0.8.8.tar.gz
-md5sum = f4dae84e96a94b768404c14633bccd49
+url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
+md5sum = 284fd2c7578064c339d9cf6a3a475ac7
 configure-options =
   --openssl-includes=${openssl:location}/include
   --openssl-libpath=${openssl:location}/lib

component/percona-toolkit/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 recipe = hexagonit.recipe.cmmi
 depends =
   ${perl:version}
-url = http://www.percona.com/redir/downloads/percona-toolkit/2.0.4/percona-toolkit-2.0.4.tar.gz
-md5sum = df7dffcccb48d50f143849629228d4b4
+url = http://www.percona.com/redir/downloads/percona-toolkit/2.1.3/percona-toolkit-2.1.3.tar.gz
+md5sum = 6f8c4a0a0e43e467766cbdabaec9ef93
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/pkgconfig/buildout.cfg

@@ -9,6 +9,9 @@ extends =
   ../glib/buildout.cfg
   ../popt/buildout.cfg
 
+[pkg-config]
+<= pkgconfig
+
 [pkgconfig]
 recipe = hexagonit.recipe.cmmi
 url = ftp://mirror.ovh.net/gentoo-distfiles/distfiles/pkg-config-0.26.tar.gz

component/poppler/buildout.cfg

@@ -13,8 +13,8 @@ extends =
 
 [poppler]
 recipe = hexagonit.recipe.cmmi
-md5sum = 45dd2c16c8c4d1a39e830e45745c4e25
-url = http://poppler.freedesktop.org/poppler-0.20.2.tar.gz
+md5sum = 28c40266f374e1960a7bcead17d39f96
+url = http://poppler.freedesktop.org/poppler-0.20.3.tar.gz
 depends =
   ${libtiff:version}
 configure-options =

component/postgresql/buildout.cfg

+[buildout]
+extends = 
+  ../openssl/buildout.cfg
+  ../readline/buildout.cfg
+  ../zlib/buildout.cfg
+  ../ncurses/buildout.cfg
+parts = postgresql
+ 
+[postgresql]
+recipe = hexagonit.recipe.cmmi
+url = http://ftp.postgresql.org/pub/source/v9.1.6/postgresql-9.1.6.tar.bz2
+md5sum = 000755f66c0de58bbd4cd2b89b45b8e2
+configure-options = --with-openssl
+environment =
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib 

component/python-setuptools/buildout.cfg

@@ -2,7 +2,7 @@
 parts = python-setuptools
 
 [setuptools-download]
-recipe = slapos.recipe.download
+recipe = slapos.recipe.build:download
 filename = setuptools-0.6c11-py2.7.egg
 url = http://pypi.python.org/packages/2.7/s/setuptools/${:filename}
 md5sum = fe1f997bc722265116870bc7919059ea

component/rpm2cpio/buildout.cfg

@@ -4,5 +4,5 @@ parts =
 
 [rpm2cpio]
 recipe = slapos.recipe.build:download
-url = http://ruda.googlecode.com/hg-history/2989f0531de63872cc7327590798d1898ac91246/rpm/rpm2cpio.py
+url = https://raw.github.com/ruda/rpm2cpio/e196173f1f6b746463b7398e381b94a42edfa345/rpm2cpio.py
 md5sum = c5bb6227d99e1ff5df880f997cbed2e3

component/slapos/buildout.cfg

+# This file is used to install SlapOS Node itself.
+
 [buildout]
 extends =
   ../../stack/shacache-client.cfg
@@ -25,6 +27,7 @@ parts =
   slapos
   cfg-environment
   sh-environment
+  py
 
 find-links =
   http://www.nexedi.org/static/packages/source/slapos.buildout/
@@ -46,13 +49,13 @@ exec-sitecustomize = false
 allowed-eggs-from-site-packages =
 
 [environment]
-# Note: For now original PATH is appeneded to the end, as not all tools are
+# Note: For now original PATH is appended to the end, as not all tools are
 #       provided by SlapOS
 PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${m4:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin:${swig:location}/bin:${buildout:bin-directory}:${patch:location}/bin:$PATH
 CFLAGS=-I${bzip2:location}/include -I${gdbm:location}/include -I${gettext:location}/include -I${glib:location}/include -I${libxml2:location}/include -I${libxslt:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${popt:location}/include -I${readline:location}/include -I${sqlite3:location}/include -I${zlib:location}/include
 CPPFLAGS=${:CFLAGS}
 LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${libxslt:location}/lib -Wl,-rpath=${libxslt:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkconfig:${sqlite3:location}/lib/pkconfig
+PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig
 LD_LIBRARY_PATH=${bzip2:location}/lib:${gdbm:location}/lib:${gettext:location}/lib:${glib:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${popt:location}/lib:${readline:location}/lib:${sqlite3:location}/lib:${zlib:location}/lib
 
 [cfg-environment]
@@ -112,26 +115,37 @@ scripts =
   slapgrid-supervisord = slapos.grid.svcbackend:supervisord
   slapproxy = slapos.proxy:main
   slapos = slapos.entry:main
+  slapos-watchdog = slapos.grid.watchdog:main
+
+[py]
+recipe = zc.recipe.egg
+eggs = 
+    ${slapos:eggs}
+python = python2.7
+interpreter = py
+scripts = py
 
 [versions]
-zc.buildout = 1.6.0-dev-SlapOS-006
+# Use our own buildout version
+zc.buildout = 1.6.0-dev-SlapOS-010
 
+# Generated by buildout-versions
 Jinja2 = 2.6
 Werkzeug = 0.8.3
 buildout-versions = 1.7
 collective.recipe.template = 1.9
 hexagonit.recipe.cmmi = 1.6
-lxml = 3.0alpha2
-meld3 = 0.6.8
+lxml = 3.0.1
+meld3 = 0.6.9
 netaddr = 0.7.10
-slapos.core = 0.28.6
+slapos.core = 0.33.1
 slapos.libnetworkcache = 0.13.2
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.33.1
 Flask = 0.9
 
 # Required by:
@@ -139,21 +153,21 @@ Flask = 0.9
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.33.1
 netifaces = 0.8
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.33.1
 # slapos.libnetworkcache==0.13.2
-# supervisor==3.0a12
-# zc.buildout==1.6.0-dev-SlapOS-006
+# supervisor==3.0b1
+# zc.buildout==1.6.0-dev-SlapOS-010
 # zope.interface==4.0.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.core==0.28.6
-supervisor = 3.0a12
+# slapos.core==0.33.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.core==0.28.6
+# slapos.core==0.33.1
 zope.interface = 4.0.1

component/sphinx/README

+This component is not used anymore - but might be revived within 4 months.
+If not used within 6 months, feel free to delete this.

component/sphinx/buildout.cfg

@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch
 recipe = hexagonit.recipe.cmmi
 depends =
   ${mariadb:version}
-url = http://sphinxsearch.com/files/sphinx-2.0.2-beta.tar.gz
-md5sum = fafe0f1a71d0ded32404c067eba7d0b3
+url = http://sphinxsearch.com/files/sphinx-2.0.5-release.tar.gz
+md5sum = e71fdb5b0c2911247d48fb30550b9584
 configure-options =
   --with-mysql
   --with-mysql-includes=${mariadb:location}/include/mysql

component/stunnel/buildout.cfg

@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py
 
 [stunnel-4]
 recipe = hexagonit.recipe.cmmi
-url = http://mirror.bit.nl/stunnel/stunnel-4.53.tar.gz
-md5sum = ab3bfc915357d67da18c73f73610d593
+url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.54.tar.gz
+md5sum = c2b1db99e3ed547214568959a8ed18ac
 pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook
 configure-options =
   --enable-ipv6

component/w3m/buildout.cfg

@@ -45,6 +45,6 @@ patches =
 
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkconfig
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig
   CPPFLAGS=-I${ncurses:location}/include/ -I${zlib:location}/include/ -I${garbage-collector:location}/include
   LDFLAGS=-Wl,--as-needed -L${garbage-collector:location}/lib -Wl,-rpath=${garbage-collector:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

setup.py

+##############################################################################
+#
+# Copyright (c) 2010-2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
 from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.67-dev'
+version = '0.71.2-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -26,7 +52,6 @@ setup(name=name,
       packages=find_packages(),
       include_package_data=True,
       install_requires=[
-        'PyXML', # for full blown python interpreter
         'lxml', # for full blown python interpreter
         'netaddr', # to manipulate on IP addresses
         'setuptools', # namespaces
@@ -36,38 +61,41 @@ setup(name=name,
         'xml_marshaller', # need to communication with slapgrid
         'zc.buildout', # plays with buildout
         'zc.recipe.egg', # for scripts generation
+        'pytz', # for timezone database
         ],
       zip_safe=True,
       entry_points={
         'zc.buildout': [
+          'addresiliency = slapos.recipe.addresiliency:Recipe',
           'agent = slapos.recipe.agent:Recipe',
           'apache.frontend = slapos.recipe.apache_frontend:Recipe',
+          'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
           'apacheproxy = slapos.recipe.apacheproxy:Recipe',
-          'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
-          'certificate_authority.request = slapos.recipe.certificate_authority:Request',
           'certificate_authority = slapos.recipe.certificate_authority:Recipe',
+          'certificate_authority.request = slapos.recipe.certificate_authority:Request',
+          'check_page_content = slapos.recipe.check_page_content:Recipe',
           'check_port_listening = slapos.recipe.check_port_listening:Recipe',
           'check_url_available = slapos.recipe.check_url_available:Recipe',
-          'check_page_content = slapos.recipe.check_page_content:Recipe',
           'cloud9 = slapos.recipe.cloud9:Recipe',
           'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe',
-          'cron.d = slapos.recipe.dcron:Part',
           'cron = slapos.recipe.dcron:Recipe',
+          'cron.d = slapos.recipe.dcron:Part',
           'davstorage = slapos.recipe.davstorage:Recipe',
           'downloader = slapos.recipe.downloader:Recipe',
+          'dropbear = slapos.recipe.dropbear:Recipe',
           'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
           'dropbear.client = slapos.recipe.dropbear:Client',
-          'dropbear = slapos.recipe.dropbear:Recipe',
+          'dumpmdb = slapos.recipe.dumpmdb:Recipe',
           'duplicity = slapos.recipe.duplicity:Recipe',
           'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
           'equeue = slapos.recipe.equeue:Recipe',
           'erp5.bootstrap = slapos.recipe.erp5_bootstrap:Recipe',
           'erp5.promise = slapos.recipe.erp5_promise:Recipe',
-          'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
-          'erp5testnode = slapos.recipe.erp5testnode:Recipe',
           'erp5.test = slapos.recipe.erp5_test:Recipe',
           'erp5.update = slapos.recipe.erp5_update:Recipe',
+          'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
+          'erp5testnode = slapos.recipe.erp5testnode:Recipe',
           'firefox = slapos.recipe.firefox:Recipe',
           'fontconfig = slapos.recipe.fontconfig:Recipe',
           'generate.mac = slapos.recipe.generatemac:Recipe',
@@ -82,57 +110,62 @@ setup(name=name,
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'helloworld = slapos.recipe.helloworld:Recipe',
+          'importmdb = slapos.recipe.importmdb:Recipe',
           'java = slapos.recipe.java:Recipe',
           'kumofs = slapos.recipe.kumofs:Recipe',
-          'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
           'kvm = slapos.recipe.kvm:Recipe',
+          'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
+          'lamp = slapos.recipe.lamp:Request',
           'lamp.request = slapos.recipe.lamp:Request',
           'lamp.simple = slapos.recipe.lamp:Simple',
-          'lamp = slapos.recipe.lamp:Request',
           'lamp.static = slapos.recipe.lamp:Static',
-          'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'libcloud = slapos.recipe.libcloud:Recipe',
+          'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'lockfile = slapos.recipe.lockfile:Recipe',
-          'logrotate.d = slapos.recipe.logrotate:Part',
           'logrotate = slapos.recipe.logrotate:Recipe',
+          'logrotate.d = slapos.recipe.logrotate:Part',
           'memcached = slapos.recipe.memcached:Recipe',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
           'mydumper = slapos.recipe.mydumper:Recipe',
           'mysql = slapos.recipe.mysql:Recipe',
           'nbdserver = slapos.recipe.nbdserver:Recipe',
           'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
+          'notifier = slapos.recipe.notifier:Recipe',
           'notifier.callback = slapos.recipe.notifier:Callback',
           'notifier.notify = slapos.recipe.notifier:Notify',
-          'notifier = slapos.recipe.notifier:Recipe',
           'novnc = slapos.recipe.novnc:Recipe',
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
+          'postgres = slapos.recipe.postgres:Recipe',
+          'postgres.export = slapos.recipe.postgres:ExportRecipe',
+          'postgres.import = slapos.recipe.postgres:ImportRecipe',
           'proactive = slapos.recipe.proactive:Recipe',
           'publish = slapos.recipe.publish:Recipe',
           'publishurl = slapos.recipe.publishurl:Recipe',
           'pwgen = slapos.recipe.pwgen:Recipe',
           'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
-          'requestoptional = slapos.recipe.requestoptional:Recipe',
           'request = slapos.recipe.request:Recipe',
+          'requestoptional = slapos.recipe.request:RequestOptional',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
-          'shellinabox = slapos.recipe.shellinabox:Recipe',
           'shell = slapos.recipe.shell:Recipe',
+          'shellinabox = slapos.recipe.shellinabox:Recipe',
           'signalwrapper= slapos.recipe.signal_wrapper:Recipe',
           'simplelogger = slapos.recipe.simplelogger:Recipe',
           'siptester = slapos.recipe.siptester:SipTesterRecipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapcontainer = slapos.recipe.container:Recipe',
-          'slapmonitor = slapos.recipe.slapmonitor:Recipe',
+          'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
+          'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',
           'slapreport = slapos.recipe.slapreport:Recipe',
           'slaprunner = slapos.recipe.slaprunner:Recipe',
+          'slaprunner.test = slapos.recipe.slaprunner:Test',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
-          'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
+          'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
           'stunnel = slapos.recipe.stunnel:Recipe',
           'symbolic.link = slapos.recipe.symbolic_link:Recipe',
-          'testnode = slapos.recipe.testnode:Recipe',
           'tidstorage = slapos.recipe.tidstorage:Recipe',
           'urlparse = slapos.recipe._urlparse:Recipe',
           'uuid = slapos.recipe._uuid:Recipe',

slapos/recipe/README.testnode.txt

-testnode
-========
-
-Generic testnode.

slapos/recipe/template.py → slapos/recipe/addresiliency/__init__.py

@@ -24,23 +24,75 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from slapos.recipe.librecipe import GenericSlapRecipe
+
 import os
 
-from slapos.recipe.librecipe import GenericBaseRecipe
 
-class Recipe(GenericBaseRecipe):
+class Recipe(GenericSlapRecipe):
+    """ This class provides the installation of the resilience
+        scripts on the partition.
+
+        bin/takeover will perform a rename (must be run manually).
+        bin/bully will monitor, run elections and perform renames when needed.
+    """
+
+    def _install(self):
+        path_list = []
+
+        confpath = os.path.join(self.options['etc'], 'bully.conf')
+
+        ip_list = self.parameter_dict['ip-list']
+        print 'Creating bully configuration with ips : %s\n' % ip_list
+
+        conf = self.createFile(confpath,
+                               self.substituteTemplate(
+                               self.getTemplateFilename('bully.conf.in'),
+                               {
+                                   'self_id': int(self.parameter_dict['number']),
+                                   'ip_list': ip_list
+                                   }
+                               ))
+        path_list.append(conf)
+
+        slap_connection = self.buildout['slap-connection']
+
+        if self.optionIsTrue('enable-bully-service', default=False):
+            bully_dir = self.options['services']
+        else:
+            bully_dir = self.options['bin']
+
+        bully_wrapper = self.createPythonScript(
+            name=os.path.join(bully_dir, self.options['wrapper-bully']),
+            absolute_function='slapos.recipe.addresiliency.bully.run',
+            arguments={
+                'confpath': confpath,
+                'server_url': slap_connection['server-url'],
+                'key_file': slap_connection.get('key-file'),
+                'cert_file': slap_connection.get('cert-file'),
+                'computer_id': slap_connection['computer-id'],
+                'partition_id': slap_connection['partition-id'],
+                'software': slap_connection['software-release-url'],
+                'namebase': self.parameter_dict['namebase'],
+            })
+
+        path_list.append(bully_wrapper)
+
+        takeover_wrapper = self.createPythonScript(
+            name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
+            absolute_function='slapos.recipe.addresiliency.takeover.run',
+            arguments={
+                'server_url': slap_connection['server-url'],
+                'key_file': slap_connection.get('key-file'),
+                'cert_file': slap_connection.get('cert-file'),
+                'computer_id': slap_connection['computer-id'],
+                'partition_id': slap_connection['partition-id'],
+                'software': slap_connection['software-release-url'],
+                'namebase': self.parameter_dict['namebase'],
+            })
 
-    def install(self):
-        mapping = self.options.copy()
-        for key in ('output', 'template', 'recipe', 'mode'):
-            if key in mapping:
-                del mapping[key]
+        path_list.append(takeover_wrapper)
 
-        with open(self.options['output'], 'w') as output, \
-             open(self.options['template'], 'r') as template:
-            output.write(template.read() % mapping)
+        return path_list
 
-        if 'mode' in self.options:
-            os.chmod(self.options['output'], int(self.options['mode'], 8))
 
-        return [self.options['output'], ]

slapos/recipe/addresiliency/bully.py

+# -*- coding: utf-8 -*-
+
+import logging
+import Queue
+import socket
+import thread
+import time
+
+import slapos.recipe.addresiliency.renamer
+import slapos
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+BASE_PORT = 50000
+SLEEPING_MINS = 2           # XXX was 30, increase after testing
+
+MSG_PING = 'ping'
+MSG_HALT = 'halt'
+MSG_VICTORY = 'victory'
+
+MSG_OK = 'ok'
+
+STATE_NORMAL = 'normal'
+STATE_WAITINGCONFIRM = 'waitingConfirm'
+STATE_ELECTION = 'election'
+STATE_REORGANIZATION = 'reorganization'
+
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm, renamer, confpath):
+        self.comm = comm
+        self.participant_id = 0
+        self.state = STATE_NORMAL
+        self.halter_id = 0
+        self.inElection = False
+        self.alive = True
+
+        self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
+
+        self.renamer = renamer
+        self.okCanal = self.comm.create_canal([MSG_OK])
+        self.confpath = confpath
+        self.loadConnectionInfo()
+
+    def loadConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.total_participants = len(params[0].split())
+        new_id = int(params[1])
+        if self.participant_id != new_id:
+            self.halter_id = new_id
+            self.participant_id = new_id 
+        log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
+            if self.participant_id == 0:
+                continue
+            self.comm.send(MSG_PING, 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == MSG_PING:
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_HALT:
+                self.state = STATE_WAITINGCONFIRM
+                self.halter_id = int(sender)
+                self.comm.send(MSG_OK, sender)
+
+            elif message == MSG_VICTORY:
+                if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
+                    log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
+                    self.comm.send(MSG_OK, sender)
+                self.state = STATE_NORMAL
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfo()
+        # Check if I'm the highest instance alive
+        for higher in range(self.participant_id + 1, self.total_participants):
+            self.comm.send(MSG_PING, higher)
+            message, sender = self.okCanal.get()
+            if message:
+                log.info('{} is alive ({})'.format(higher, self.participant_id))
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        # I should be the new coordinator, halt those below me
+        log.info('Should be ME : {}'.format(self.participant_id))
+        self.state = STATE_ELECTION
+        self.halter_id = self.participant_id
+        ups = []
+        for lower in range(self.participant_id):
+            self.comm.send(MSG_HALT, lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = STATE_REORGANIZATION
+        for up in ups:
+            self.comm.send(MSG_VICTORY, up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            log.info('Something is wrong... let\'s start over')
+            return self.election()
+        self.state = STATE_NORMAL
+        self.active = True
+        log.info('{} Is THE LEADER'.format(self.participant_id))
+
+        self.renamer.failover()
+
+        self.inElection = False
+
+        return True
+
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.queue = Queue.Queue()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.queue.put([message, sender])
+
+    def get(self):
+        try:
+            return self.queue.get(timeout=self.timeout)
+        except Queue.Empty:
+            return [None, None]
+
+
+
+class Wrapper(object):
+
+    def __init__(self, confpath, timeout=20):
+        self.canals = []
+        self.ips = []
+        self.participant_id = 0
+        self.timeout = timeout
+        self.confpath = confpath
+        self.getConnectionInfo()
+        self.socket = None
+
+    def getConnectionInfo(self):
+        params = open(self.confpath, 'r').readlines()
+        self.ips = params[0].split()
+        self.participant_id = int(params[1])
+        log.debug('I am {} of {}'.format(self.participant_id, self.ips))
+
+    def start(self):
+        self.getConnectionInfo()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
+        self.socket.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfo()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.participant_id)))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+            pass
+        finally:
+            s.close()
+
+    def create_canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = self.socket.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, int(sender))
+
+
+
+
+def run(args):
+    confpath = args.pop('confpath')
+
+    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
+                                                          key_file = args.pop('key_file'),
+                                                          cert_file = args.pop('cert_file'),
+                                                          computer_guid = args.pop('computer_id'),
+                                                          partition_id = args.pop('partition_id'),
+                                                          software_release = args.pop('software'),
+                                                          namebase = args.pop('namebase'))
+
+    if args:
+        raise ValueError('Unknown arguments: %s' % ', '.join(args))
+
+    wrapper = Wrapper(confpath=confpath, timeout=20)
+
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
+
+    # idle waiting for connection infos
+    while computer.total_participants < 2:
+        computer.loadConnectionInfo()
+        time.sleep(30)
+
+    log.info('Starting')
+
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.aliveManagement, ())
+
+    computer.main()
+

slapos/recipe/addresiliency/renamer.py

+# -*- coding: utf-8 -*-
+
+import logging
+import time
+
+import slapos
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+
+    def _failover(self):
+        """\
+        This method does
+
+        - retrieve the broken computer partition
+        - change its reference to 'broken-...' and its software type to 'frozen'
+        - retrieve the winner computer partition (attached to this process)
+        - change its reference to replace the broken one.
+          later, slapgrid will change its software_type as well.
+
+        Then, after running slapgrid-cp a few times, the winner takes over and
+        a new cp is created to replace it as an importer.
+        """
+
+        slap = slapos.slap.slap()
+        slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
+
+        # partition that will take over.
+        cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                   partition_id=self.partition_id)
+        # XXX although we can already rename cp_winner, to change its software type we need to
+        # get hold of the root cp as well
+
+        cp_exporter_ref = self.namebase + '0'       # this is ok. the boss is always number zero.
+
+        # partition to be deactivated
+        cp_broken = cp_winner.request(software_release=self.software_release,
+                                      software_type='frozen',
+                                      state='stopped',
+                                      partition_reference=cp_exporter_ref)
+
+        broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
+
+        log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
+
+        cp_broken.rename(new_name=broken_new_ref)
+
+        cp_broken.stopped()
+
+        log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
+
+        # update name (and later, software type) for the partition that will take over
+
+        cp_winner.rename(new_name=cp_exporter_ref)
+        cp_winner.bang(message='partitions have been renamed!')
+
+
+
+    def failover(self):
+        try:
+            self._failover()
+            log.info('Renaming done')
+        except slapos.slap.ServerError:
+            log.info('Internal server error')
+

slapos/recipe/addresiliency/takeover.py

+# -*- coding: utf-8 -*-
+
+import slapos.recipe.addresiliency.renamer
+
+def run(args):
+    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
+                                                          key_file = args.pop('key_file'),
+                                                          cert_file = args.pop('cert_file'),
+                                                          computer_guid = args.pop('computer_id'),
+                                                          partition_id = args.pop('partition_id'),
+                                                          software_release = args.pop('software'),
+                                                          namebase = args.pop('namebase'))
+
+    renamer.failover()
+
+

slapos/recipe/addresiliency/template/bully.conf.in

+%(ip_list)s
+%(self_id)s

slapos/recipe/addresiliency/template/bully.py.in

+#!%(executable)s
+
+import select
+import socket
+import threading
+import time
+import sys
+
+sys.path[:] = %(syspath)s
+
+import slapos
+from slapos import slap as slapmodule
+
+port = 50000
+size = 1024
+wait = True
+
+
+def loadConnectionInfos():
+  connectionInfos = {}
+  file = open('%(confpath)s', 'r')
+  params = file.read().split('\n')
+  file.close()
+  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+  connectionInfos['self_id'] = int(params[1])
+
+  connectionInfos['server_list'] = \
+      [(i, ip_list[i]) for i in range(len(ip_list))]
+
+  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
+
+  return connectionInfos
+
+
+def rename_broken_and_stop():
+  try:
+    slap = slapmodule.slap()
+    slap.initializeConnection('%(server_url)s',
+                              '%(key_file)s',
+                              '%(cert_file)s')
+    computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
+                                                        partition_id='%(partition_id)s')
+    broken = computer_partition.request(software_release='%(software)s',
+                                        software_type='frozen',
+                                        partition_reference='%(namebase)s0')
+
+    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+    broken.stopped()
+    computer_partition.rename('%(namebase)s0')
+    print 'renaming done\n'
+  except slapos.slap.slap.ServerError:
+    print 'Internal server error\n'
+
+def election():
+  global wait
+  connection = loadConnectionInfos()
+  message = "%%s, %%s" %% (connection['self_id'], "Election")
+  victory = True
+
+  for (remote_id, addr) in connection['server_list']:
+    if remote_id > connection['self_id']:
+      try:
+        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s.connect((addr, port + remote_id))
+        s.send(message)
+        reply = s.recv(size)
+        if reply == "%%s, %%s" %% (remote_id, "Alive"):
+          victory = False
+      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+        pass
+      finally:
+        s.close()
+
+  if victory:
+    wait = True
+    for (remote_id, addr) in connection['server_list']:
+      if remote_id < connection['self_id']:
+        try:
+          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+          s.connect((addr, port + remote_id))
+          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+          pass
+        finally:
+          s.close()
+    rename_broken_and_stop()
+
+
+def failure_detect():
+  global wait
+  connection = loadConnectionInfos()
+  while True:
+    time.sleep(30)
+    if wait:
+      print 'waiting 30 minutes\n'
+      time.sleep(30 * 60)
+      wait = False
+    if not connection['server_list'][0]:
+      continue
+    (remote_id, addr) = connection['server_list'][0]
+    try:
+      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+      s.connect((addr, port + remote_id))
+      s.close()
+    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+      s.close()
+      election()
+
+failure_detect_thread = threading.Thread(target=failure_detect)
+failure_detect_thread.start()
+
+connection = loadConnectionInfos()
+s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+s.bind((connection['self_ip'], port + connection['self_id']))
+s.listen(5)
+#election()
+while True:
+  force_election = False
+  client, _ = s.accept()
+  client_message = client.recv(1024)
+  if client_message:
+    client_id, message = client_message.split(', ')
+    client_id = eval(client_id)
+    if message == "Victory":
+      wait = True
+      print "%%s wins" %% client_id
+    elif message == "Election":
+      print "%%s starts an election" %% client_id
+      if client_id < connection['self_id']:
+        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
+        force_election = True
+  client.close()
+  if force_election:
+    election()

slapos/recipe/addresiliency/template/bully_new.py.in

+#!%(executable)s
+
+import logging
+import os
+import socket
+import sys
+import thread
+import time
+
+sys.path[:] = %(syspath)s
+
+from slapos import slap as slapmodule
+import slapos
+
+BASE_PORT = 50000
+SLEEPING_MINS = 2
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+
+class Renamer(object):
+    def __init__(self, server_url, key_file, cert_file, computer_guid,
+                 partition_id, software_release, namebase):
+        self.server_url = server_url
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.computer_guid = computer_guid
+        self.partition_id = partition_id
+        self.software_release = software_release
+        self.namebase = namebase
+
+    def _failover(self):
+        slap = slapmodule.slap()
+        slap.initializeConnection(self.server_url,
+                                  self.key_file,
+                                  self.cert_file)
+        computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
+                                                            partition_id=self.partition_id)
+        broken = computer_partition.request(software_release=self.software_release,
+                                            software_type='frozen',
+                                            partition_reference=self.namebase+'0')
+
+        broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
+        broken.stopped()
+        computer_partition.rename(self.namebase+'0')
+
+    def failover(self):
+        try:
+            log.info('renaming done')
+        except slapos.slap.slap.ServerError:
+            log.info('Internal server error')
+
+
+
+## Leader is always number 0
+
+class ResilientInstance(object):
+
+    def __init__(self, comm, renamer, confpath):
+        self.comm = comm
+        self.id = 0
+        self.state = 'normal'
+        self.halter = 0
+        self.inElection = False
+        self.alive = True
+        self.lastPing = time.clock()
+
+        self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
+
+        self.renamer = renamer
+        self.okCanal = self.comm.canal(['ok'])
+        self.confpath = confpath
+        self.loadConnectionInfos()
+
+    def loadConnectionInfos(self):
+        file = open(self.confpath, 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
+        new_id = int(params[1])
+        if self.id != new_id:
+            self.halter = new_id
+            self.id = new_id 
+
+    ## Needs to be changed to use the master
+    def aliveManagement(self):
+        while self.alive:
+            log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
+            time.sleep(SLEEPING_MINS*60)
+            if self.id == 0:
+                continue
+            self.comm.send('ping', 0)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            self.election()
+
+    def listen(self):
+        while self.alive:
+            self.comm.recv()
+
+    def main(self):
+        while self.alive:
+            message, sender = self.mainCanal.get()
+            if message == 'ping':
+                self.comm.send('ok', sender)
+
+            elif message == 'halt':
+                self.state = 'waitingConfirm'
+                self.halter = sender
+                self.comm.send('ok', sender)
+
+            elif message == 'victory':
+                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
+                    log.info('{} thinks {} is the leader'.format(self.id, sender))
+                    self.comm.send('ok', sender)
+                self.state = 'normal'
+
+    def election(self):
+        self.inElection = True
+        self.loadConnectionInfos()
+        #Check if I'm the highest instance alive
+        for higher in range(self.id + 1, self.nbComp):
+            self.comm.send('ping', higher)
+            message, sender = self.okCanal.get()
+            if message:
+                log.info('{} is alive ({})'.format(higher, self.id))
+                self.inElection = False
+                return False
+            continue
+
+        if not self.alive:
+            return False
+
+        #I should be the new coordinator, halt those below me
+        log.info('Should be ME : {}'.format(self.id))
+        self.state = 'election'
+        self.halter = self.id
+        ups = []
+        for lower in range(self.id):
+            self.comm.send('halt', lower)
+            message, sender = self.okCanal.get()
+            if message:
+                ups.append(lower)
+
+        #Broadcast Victory
+        self.state = 'reorganization'
+        for up in ups:
+            self.comm.send('victory', up)
+            message, sender = self.okCanal.get()
+            if message:
+                continue
+            log.info('Something is wrong... let\'s start over')
+            return self.election()
+        self.state = 'normal'
+        self.active = True
+        log.info('{} Is THE LEADER'.format(self.id))
+
+        self.renamer.failover()
+
+        self.inElection = False
+
+        return True
+
+
+class FilteredCanal(object):
+
+    def __init__(self, accept, timeout):
+        self.accept = accept
+        self.list = []
+        self.lock = thread.allocate_lock()
+        self.timeout = timeout
+
+    def append(self, message, sender):
+        if message in self.accept:
+            self.lock.acquire()
+            self.list.append([message, sender])
+            self.lock.release()
+
+    def get(self):
+        start = time.clock()
+        while (time.clock() - start < self.timeout):
+            self.lock.acquire()
+            if self.list:
+                self.lock.release()
+                return self.list.pop(0)
+            self.lock.release()
+        return [None, None]
+
+
+class Wrapper(object):
+
+    def __init__(self, confpath, timeout=20):
+        self.canals = []
+        self.ips = []
+        self.id = 0
+        self.timeout = timeout
+        self.confpath = confpath
+        self.getConnectionInfos()
+        self.socket = None
+
+    def getConnectionInfos(self):
+        file = open(self.confpath, 'r')
+        params = file.read().split('\n')
+        file.close()
+        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
+        self.id = int(params[1])
+
+    def start(self):
+        self.getConnectionInfos()
+        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
+        self.socket.listen(5)
+
+    def send(self, message, number):
+        self.getConnectionInfos()
+        try:
+            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            s.connect((self.ips[number], BASE_PORT + number))
+            s.send(message + (' {}\n'.format(self.id)))
+        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
+            pass
+        finally:
+            s.close()
+
+    def canal(self, accept):
+        created = FilteredCanal(accept, self.timeout)
+        self.canals.append(created)
+        return created
+
+    def recv(self):
+        client, _ = self.socket.accept()
+        client_message = client.recv(1024)
+        if client_message:
+            message, sender = client_message.split()
+            for canal in self.canals:
+                canal.append(message, int(sender))
+
+
+
+def main():
+
+    renamer = Renamer(server_url = '%(server_url)s',
+                      key_file = '%(key_file)s',
+                      cert_file = '%(cert_file)s',
+                      computer_guid = '%(computer_id)s',
+                      partition_id = '%(partition_id)s',
+                      software_release = '%(software)s',
+                      namebase = '%(namebase)s')
+
+    confpath = '%(confpath)s'
+
+    wrapper = Wrapper(confpath=confpath, timeout=20)
+
+    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
+
+    #idle waiting for connection infos
+    while computer.nbComp < 2 :
+        computer.loadConnectionInfos()
+        time.sleep(30)
+
+    log.info('Starting')
+
+    computer.comm.start()
+    thread.start_new_thread(computer.listen, ())
+    thread.start_new_thread(computer.main, ())
+    thread.start_new_thread(computer.aliveManagement, ())
+
+    while True:
+        # XXX tight loop
+        continue
+
+
+if __name__ == '__main__':
+    main()
+

slapos/recipe/agent/__init__.py

@@ -25,65 +25,42 @@
 #
 #############################################################################
 
-import os
-import sys
 import zc.buildout
-import slapos.slap
-from slapos.recipe.librecipe import BaseSlapRecipe
-from slapos.recipe.librecipe import GenericSlapRecipe
-import json
-import ConfigParser
+from slapos.recipe.librecipe import GenericBaseRecipe
+import sys
 
-# XXX: BaseSlapRecipe and GenericSlapRecipe are deprecated, use
-# GenericBaseRecipe and move partition parameter fetching to software release.
-class Recipe(BaseSlapRecipe, GenericSlapRecipe):
+class Recipe(GenericBaseRecipe):
   def install(self):
-    self.path_list = []
-    crond = self.installCrond()
+    path_list = []
+
+    configuration_path = self.options["config"]
+    header = """[DEFAULT]
+master_url = %s
+key = %s
 
-    slap = slapos.slap.slap()
-    slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
-    parameter_dict = slap.registerComputerPartition(
-      self.computer_id,
-      self.computer_partition_id,
-    ).getInstanceParameterDict()
+cert = %s
+
+max_install_duration = %s
+max_uninstall_duration = %s
+max_request_duration = %s
+max_destroy_duration = %s
+""" % (self.options["master-url"],
+      "\n  ".join(self.options["key"].split("\n")),
+       "\n  ".join(self.options["cert"].split("\n")),
+       self.options["default_max_install_duration"],
+       self.options["default_max_uninstall_duration"],
+       self.options["default_max_request_duration"],
+       self.options["default_max_destroy_duration"])
 
-    # XXX: should probably expect one more (SR-originating) parameter instead
-    # of using self.work_directory .
-    configuration_path = os.path.join(self.work_directory, "agent.cfg")
     with open(configuration_path, "w") as configuration:
-      configuration.write(parameter_dict["configuration"])
-    agent_crond_path = os.path.join(crond, "agent")
-    with open(agent_crond_path, "w") as agent_crond:
-      agent_crond.write("*/5 * * * * %s -S %s --pidfile=%s --log=%s "
-        "%s 2>&1 > /dev/null\n" % (
-          self.options["python_binary"],
-          self.options["agent_binary"],
-          self.options["pidfile"],
-          self.options["log"],
-          configuration_path,
-      ))
+      configuration.write(header + self.options["configuration"])
+
+    path_list.append(self.createPythonScript(
+                       self.options['wrapper'],
+                       'slapos.recipe.librecipe.execute.execute',
+                         [self.options["agent_binary"], '--pidfile=%s' % self.options["pidfile"],
+                          "--log=%s" % self.options["log"], configuration_path]))
 
-    return self.path_list + [configuration_path, agent_crond_path]
+    path_list.append(configuration_path)
 
-  def installCrond(self):
-    _, ws = self.egg.working_set()
-    timestamps = self.createDataDirectory('cronstamps')
-    cron_output = os.path.join(self.log_directory, 'cron-output')
-    self._createDirectory(cron_output)
-    catcher = zc.buildout.easy_install.scripts([('catchcron',
-      __name__ + '.catdatefile', 'catdatefile')], ws, sys.executable,
-      self.bin_directory, arguments=[cron_output])[0]
-    self.path_list.append(catcher)
-    cron_d = os.path.join(self.etc_directory, 'cron.d')
-    crontabs = os.path.join(self.etc_directory, 'crontabs')
-    self._createDirectory(cron_d)
-    self._createDirectory(crontabs)
-    wrapper = zc.buildout.easy_install.scripts([('crond',
-      'slapos.recipe.librecipe.execute', 'execute')], ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
-        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
-      )[0]
-    self.path_list.append(wrapper)
-    return cron_d
+    return path_list

slapos/recipe/apache_frontend/__init__.py

@@ -28,6 +28,7 @@ from slapos.recipe.librecipe import BaseSlapRecipe
 import os
 import pkg_resources
 import hashlib
+import operator
 import sys
 import zc.buildout
 import zc.recipe.egg
@@ -43,6 +44,9 @@ class Recipe(BaseSlapRecipe):
         'template/%s' % template_name)
 
   def _install(self):
+    # Define directory not defined in deprecated lib
+    self.service_directory = os.path.join(self.etc_directory, 'service')
+
     # Check for mandatory arguments
     frontend_domain_name = self.parameter_dict.get("domain")
     if frontend_domain_name is None:
@@ -69,9 +73,17 @@ class Recipe(BaseSlapRecipe):
 
     rewrite_rule_list = []
     rewrite_rule_zope_list = []
+    rewrite_rule_zope_path_list = []
     slave_dict = {}
     service_dict = {}
 
+    # Sort slave instance by reference to avoid most security issues
+    slave_instance_list = sorted(slave_instance_list,
+                                 key=operator.itemgetter('slave_reference'))
+
+    # dict of used domains, only used to track duplicates
+    domain_dict = {}
+
     for slave_instance in slave_instance_list:
       backend_url = slave_instance.get("url", None)
       reference = slave_instance.get("slave_reference")
@@ -97,6 +109,12 @@ class Recipe(BaseSlapRecipe):
         domain = "%s.%s" % (reference.replace("-", "").lower(),
             frontend_domain_name)
 
+      if domain_dict.get(domain):
+        # This domain already has been processed, skip this new one
+        continue
+      else:
+        domain_dict[domain] = True
+
       # Define the URL where the instance will be available
       # WARNING: we use default ports (443, 80) here.
       slave_dict[reference] = "%s%s/" % (scheme, domain)
@@ -118,6 +136,9 @@ class Recipe(BaseSlapRecipe):
         # RewriteMap for Zope Virtual Host Monster websites.
         if slave_instance.get("type", "").lower() in ['zope']:
           rewrite_rule_zope_list.append(rewrite_rule)
+          # For Zope, we have another dict containing the path e.g '/erp5/...
+          rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
+          rewrite_rule_zope_path_list.append(rewrite_rule_path)
         else:
           rewrite_rule_list.append(rewrite_rule)
 
@@ -152,6 +173,7 @@ class Recipe(BaseSlapRecipe):
         name=frontend_domain_name,
         rewrite_rule_list=rewrite_rule_list,
         rewrite_rule_zope_list=rewrite_rule_zope_list,
+        rewrite_rule_zope_path_list=rewrite_rule_zope_path_list,
         key=key, certificate=certificate)
 
     # Send connection informations about each slave
@@ -160,9 +182,12 @@ class Recipe(BaseSlapRecipe):
           "instance: %s" % reference)
       try:
         connection_dict = {
-           'frontend_ipv6_address': self.getGlobalIPv6Address(),
-           'frontend_ipv4_address': self.getLocalIPv4Address(),
-           'site_url': url
+            # Send the public IPs (if possible) so that user knows what IP
+            # to bind to its domain name
+            'frontend_ipv6_address': self.getGlobalIPv6Address(),
+            'frontend_ipv4_address': self.parameter_dict.get("public-ipv4",
+                self.getLocalIPv4Address()),
+            'site_url': url,
         }
         self.setConnectionDict(connection_dict, reference)
       except:
@@ -289,7 +314,7 @@ class Recipe(BaseSlapRecipe):
    self._createDirectory(crontabs)
    wrapper = zc.buildout.easy_install.scripts([('crond',
      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-     self.wrapper_directory, arguments=[
+     self.service_directory, arguments=[
        self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
      )[0]
@@ -346,10 +371,13 @@ class Recipe(BaseSlapRecipe):
     )
     self._writeFile(openssl_configuration, pkg_resources.resource_string(
       __name__, 'template/openssl.cnf.ca.in') % config)
+
+    # XXX-Cedric: Don't use this, but use slapos.recipe.certificate_authority
+    #             from the instance profile.
     self.path_list.extend(zc.buildout.easy_install.scripts([
       ('certificate_authority', __name__ + '.certificate_authority',
          'runCertificateAuthority')],
-        self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
+        self.ws, sys.executable, self.service_directory, arguments=[dict(
           openssl_configuration=openssl_configuration,
           openssl_binary=self.options['openssl_binary'],
           certificate=os.path.join(self.ca_dir, 'cacert.pem'),
@@ -382,6 +410,8 @@ class Recipe(BaseSlapRecipe):
         name + '.lock')
     apache_conf['document_root'] = os.path.join(self.data_root_directory,
         'htdocs')
+    apache_conf['instance_home'] = os.path.join(self.work_directory)
+    apache_conf['httpd_home'] = self.options['httpd_home']
     apache_conf['ip_list'] = ip_list
     apache_conf['port'] = port
     apache_conf['server_admin'] = 'admin@'
@@ -419,10 +449,11 @@ class Recipe(BaseSlapRecipe):
         "-f", config_file,
         "-a", varnish_config["port"], "-T", varnish_config["control_port"],
         "-s", varnish_config["storage"]]
-    environment = dict(PATH=self.options["binutils_directory"])
+    environment = dict(PATH="%s:%s" % (self.options["binutils_directory"],
+                                       os.environ.get('PATH')))
     wrapper = zc.buildout.easy_install.scripts([(name,
       'slapos.recipe.librecipe.execute', 'executee')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[varnish_argument_list,
+      sys.executable, self.service_directory, arguments=[varnish_argument_list,
       environment])[0]
     self.path_list.append(wrapper)
 
@@ -461,7 +492,7 @@ class Recipe(BaseSlapRecipe):
           stunnel_conf))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
       'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[
+      sys.executable, self.service_directory, arguments=[
         [self.options['stunnel_binary'].strip(), stunnel_conf_path],
         [certificate, key]]
       )[0]
@@ -470,8 +501,17 @@ class Recipe(BaseSlapRecipe):
 
   def installFrontendApache(self, ip_list, key, certificate, name,
                             port=4443, plain_http_port=8080,
-                            rewrite_rule_list=[], rewrite_rule_zope_list=[],
+                            rewrite_rule_list=None,
+                            rewrite_rule_zope_list=None,
+                            rewrite_rule_zope_path_list=None,
                             access_control_string=None):
+    if rewrite_rule_list is None:
+      rewrite_rule_list = []
+    if rewrite_rule_zope_list is None:
+      rewrite_rule_zope_list = []
+    if rewrite_rule_zope_path_list is None:
+      rewrite_rule_zope_path_list = []
+
     # Create htdocs, populate it with default 404 document
     htdocs_location = os.path.join(self.data_root_directory, 'htdocs')
     self._createDirectory(htdocs_location)
@@ -512,9 +552,14 @@ class Recipe(BaseSlapRecipe):
     # Create configuration file and rewritemaps
     apachemap_name = "apachemap.txt"
     apachemapzope_name = "apachemapzope.txt"
+    apachemapzopepath_name = "apachemapzopepath.txt"
+
     self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
     self.createConfigurationFile(apachemapzope_name,
         "\n".join(rewrite_rule_zope_list))
+    self.createConfigurationFile(apachemapzopepath_name,
+        "\n".join(rewrite_rule_zope_path_list))
+
     apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
     apache_conf['ssl_snippet'] = self.substituteTemplate(
         self.getTemplateFilename('apache.ssl-snippet.conf.in'),
@@ -532,12 +577,17 @@ class Recipe(BaseSlapRecipe):
 
     path = self.substituteTemplate(
         self.getTemplateFilename('apache.conf.path-protected.in'),
-        dict(path='/', access_control_string='none'))
+        dict(path='/',
+             access_control_string='none',
+             document_root=apache_conf['document_root'],
+        )
+    )
 
     apache_conf.update(**dict(
       path_enable=path,
       apachemap_path=os.path.join(self.etc_directory, apachemap_name),
       apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
+      apachemapzopepath_path=os.path.join(self.etc_directory, apachemapzopepath_name),
       apache_domain=name,
       https_port=port,
       plain_http_port=plain_http_port,
@@ -553,7 +603,7 @@ class Recipe(BaseSlapRecipe):
 
     self.path_list.extend(zc.buildout.easy_install.scripts([(
       'frontend_apache', 'slapos.recipe.erp5.apache', 'runApache')], self.ws,
-          sys.executable, self.wrapper_directory, arguments=[
+          sys.executable, self.service_directory, arguments=[
             dict(
               required_path_list=[key, certificate],
               binary=self.options['httpd_binary'],

slapos/recipe/apache_frontend/template/apache.conf.in

@@ -5,12 +5,13 @@
 PidFile "%(pid_file)s"
 ServerName %(server_name)s
 DocumentRoot %(document_root)s
+ServerRoot %(instance_home)s
 
 %(listen)s
 
 ServerAdmin %(server_admin)s
 DefaultType text/plain
-TypesConfig conf/mime.types
+TypesConfig %(httpd_home)s/conf/mime.types
 AddType application/x-compress .Z
 AddType application/x-gzip .gz .tgz
 
@@ -32,23 +33,23 @@ CustomLog "%(access_log)s" common
 #LoadModule unixd_module modules/mod_unixd.so
 #LoadModule access_compat_module modules/mod_access_compat.so
 #LoadModule authz_core_module modules/mod_authz_core.so
-LoadModule authz_host_module modules/mod_authz_host.so
-LoadModule log_config_module modules/mod_log_config.so
-LoadModule deflate_module modules/mod_deflate.so
-LoadModule setenvif_module modules/mod_setenvif.so
-LoadModule version_module modules/mod_version.so
-LoadModule proxy_module modules/mod_proxy.so
-LoadModule proxy_http_module modules/mod_proxy_http.so
-LoadModule ssl_module modules/mod_ssl.so
-LoadModule mime_module modules/mod_mime.so
-LoadModule dav_module modules/mod_dav.so
-LoadModule dav_fs_module modules/mod_dav_fs.so
-LoadModule negotiation_module modules/mod_negotiation.so
-LoadModule rewrite_module modules/mod_rewrite.so
-LoadModule headers_module modules/mod_headers.so
-LoadModule cache_module modules/mod_cache.so
-LoadModule mem_cache_module modules/mod_mem_cache.so
-LoadModule antiloris_module modules/mod_antiloris.so
+LoadModule authz_host_module  %(httpd_home)s/modules/mod_authz_host.so
+LoadModule log_config_module  %(httpd_home)s/modules/mod_log_config.so
+LoadModule deflate_module     %(httpd_home)s/modules/mod_deflate.so
+LoadModule setenvif_module    %(httpd_home)s/modules/mod_setenvif.so
+LoadModule version_module     %(httpd_home)s/modules/mod_version.so
+LoadModule proxy_module       %(httpd_home)s/modules/mod_proxy.so
+LoadModule proxy_http_module  %(httpd_home)s/modules/mod_proxy_http.so
+LoadModule ssl_module         %(httpd_home)s/modules/mod_ssl.so
+LoadModule mime_module        %(httpd_home)s/modules/mod_mime.so
+LoadModule dav_module         %(httpd_home)s/modules/mod_dav.so
+LoadModule dav_fs_module      %(httpd_home)s/modules/mod_dav_fs.so
+LoadModule negotiation_module %(httpd_home)s/modules/mod_negotiation.so
+LoadModule rewrite_module     %(httpd_home)s/modules/mod_rewrite.so
+LoadModule headers_module     %(httpd_home)s/modules/mod_headers.so
+LoadModule cache_module       %(httpd_home)s/modules/mod_cache.so
+LoadModule mem_cache_module   %(httpd_home)s/modules/mod_mem_cache.so
+LoadModule antiloris_module   %(httpd_home)s/modules/mod_antiloris.so
 
 # The following directives modify normal HTTP response behavior to
 # handle known problems with browser implementations.
@@ -99,17 +100,24 @@ Header append Vary User-Agent
   ProxyTimeout 600
   RewriteEngine On
 
-  # Define the two rewritemaps : one for zope, one generic
+  # Define the two RewriteMaps (key -> value store): one for Zope, one generic
+  # containing: rewritten URL -> original URL (a.k.a VirtualHostBase in Zope)
   RewriteMap apachemapzope txt:%(apachemapzope_path)s
   RewriteMap apachemapgeneric txt:%(apachemap_path)s
 
+  # Define another RewriteMap for Zope, containing:
+  # rewritten URL -> VirtualHostRoot
+  RewriteMap apachemapzopepath txt:%(apachemapzopepath_path)s
+
   # First, we check if we have a zope backend server
   # If so, let's use Virtual Host Daemon rewrite
   RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
-  RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:%%{SERVER_PORT}/VirtualHostRoot/$1 [L,P]
+  # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
+  RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:443/${apachemapzopepath:%%{SERVER_NAME}}/VirtualHostRoot/$1 [L,P]
 
   # If we have generic backend server, let's rewrite without virtual host daemon
   RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
+  # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
   RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
 
   # If nothing exist : put a nice error

slapos/recipe/apache_frontend/template/apache.conf.path-protected.in

-# Path protected
-<Location %(path)s>
+<Directory %(path)s>
   Order Deny,Allow
   Allow from %(access_control_string)s
-</Location>
+</Directory>
+
+<Directory %(document_root)s>
+  Order Allow,Deny
+  Allow from All
+</Directory>

slapos/recipe/apachephp/__init__.py

@@ -78,7 +78,7 @@ class Recipe(GenericBaseRecipe):
     if not os.path.exists(secret_key_filename):
       secret_key = uuencode(os.urandom(45)).strip()
       # Remove unsafe characters
-      secret_key = secret_key.translate(None, '"\'')
+      secret_key = secret_key.translate(None, '"\'\\')
       with open(secret_key_filename, 'w') as secret_key_file:
         secret_key_file.write(secret_key)
     else:

slapos/recipe/apachephp/template/php.ini.in

@@ -15,4 +15,4 @@ date.timezone = Europe/Paris
 file_uploads = On
 upload_max_filesize = 8M
 post_max_size = 8M
-magic_quotes_gpc=0ff
+magic_quotes_gpc=Off

slapos/recipe/apacheproxy/template/apache.in

@@ -3,7 +3,6 @@
 
 # Basic server configuration
 PidFile "%(pid_file)s"
-LockFile "%(lock_file)s"
 Listen %(ip)s:%(port)s
 ServerAdmin someone@email
 DefaultType text/plain
@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
 <Directory />
     Options FollowSymLinks
     AllowOverride None
-    Order deny,allow
-    Deny from all
+    Require all denied
 </Directory>
 
 ProxyPass / %(backend_url)s
 
 # List of modules
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule log_config_module modules/mod_log_config.so
 LoadModule setenvif_module modules/mod_setenvif.so

slapos/recipe/check_url_available/template/check_url.in

@@ -9,11 +9,22 @@ if [ -z $URL ]; then
   exit 3
 fi
 
-CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null)
+CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
+
+if [ $? -eq 3 ]; then
+  echo "URL malformed: $URL." >&2
+  exit 1
+fi
+
+if [ $? -eq 7 ]; then
+  echo "Failed to connect to host: $URL." >&2
+  exit 1
+fi
 
 if [ ! $CODE ]; then
   echo "$URL is not available (server not reachable)." >&2
   exit 1
+fi
 
 if [ $CODE -eq 000 ]; then
   echo "$URL is not available (server not reachable)." >&2

slapos/recipe/dcron.py

@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe):
   def install(self):
     self.logger.info("Installing dcron...")
 
-    path_list = []
+    options = self.options
+    script = self.createWrapper(name=options['binary'],
+                                command=options['dcrond-binary'].strip(),
+                                parameters=[
+                                    '-s', options['cron-entries'],
+                                    '-c', options['crontabs'],
+                                    '-t', options['cronstamps'],
+                                    '-f', '-l', '5',
+                                    '-M', options['catcher']
+                                    ])
 
-    cronstamps = self.options['cronstamps']
-    cron_d = self.options['cron-entries']
-    crontabs = self.options['crontabs']
-    catcher = self.options['catcher']
-
-    binary = self.options['binary']
-
-    script = self.createPythonScript(binary,
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs,
-       '-t', cronstamps, '-f', '-l', '5', '-M', catcher]
-      )
-    path_list.append(script)
     self.logger.debug('Main cron executable created at : %r', script)
 
     self.logger.info("dcron successfully installed.")
 
-    return path_list
+    return [script]
 
 
 

slapos/recipe/dropbear.py

@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe):
 
     return [wrapper]
 
+
+def keysplit(s):
+  """
+  Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
+  and return the individual key_type + key strings.
+  """
+  si = iter(s.split(' '))
+  while True:
+    key_type = next(si)
+    try:
+      key_value = next(si)
+    except StopIteration:
+      # odd number of elements, should not happen, yield the last one by itself
+      yield key_type
+      break
+    yield '%s %s' % (key_type, key_value)
+
+
 class AddAuthorizedKey(GenericBaseRecipe):
 
   def install(self):
@@ -167,6 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe):
     path_list.append(ssh)
 
     authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
-    authorized_keys.append(self.options['key'])
+    for key in keysplit(self.options['key']):
+      # XXX key might actually be the string 'None' or 'null'
+      authorized_keys.append(key)
 
     return path_list
+

slapos/recipe/equeue.py

@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
-    commandline = [self.options['equeue-binary']]
-    commandline.extend(['--database', self.options['database']])
-    commandline.extend(['-l', self.options['log']])
+    parameters = [
+      '--database', self.options['database'],
+      '-l', self.options['log'],
+    ]
 
     if 'loglevel' in self.options:
-      commandline.extend(['--loglevel', self.options['loglevel']])
+      parameters.extend(['--loglevel', self.options['loglevel']])
 
-    commandline.append(self.options['socket'])
+    parameters.append(self.options['socket'])
+
+    wrapper = self.createWrapper(name=self.options['wrapper'],
+                                 command=self.options['equeue-binary'],
+                                 parameters=parameters)
+
+    return [wrapper]
 
-    return [self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      commandline,
-    )]

slapos/recipe/generic_mysql/__init__.py

@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe):
     # TODO: move to a separate recipe (ack'ed by Cedric)
     # percona toolkit (formerly known as maatkit) installation
     for pt_script_name in (
+        'pt-align',
         'pt-archiver',
+        'pt-collect',
         'pt-config-diff',
         'pt-deadlock-logger',
+        'pt-diskstats',
         'pt-duplicate-key-checker',
         'pt-fifo-split',
         'pt-find',
+        'pt-fingerprint',
         'pt-fk-error-logger',
         'pt-heartbeat',
         'pt-index-usage',
+        'pt-ioprofile',
         'pt-kill',
         'pt-log-player',
+        'pt-mext',
+        'pt-mysql-summary',
         'pt-online-schema-change',
+        'pt-pmp',
         'pt-query-advisor',
         'pt-query-digest',
         'pt-show-grants',
+        'pt-sift',
         'pt-slave-delay',
         'pt-slave-find',
         'pt-slave-restart',
+        'pt-stalk',
+        'pt-summary',
         'pt-table-checksum',
         'pt-table-sync',
+        'pt-table-usage',
         'pt-tcp-model',
         'pt-trend',
         'pt-upgrade',

slapos/recipe/generic_mysql/mysql.py

@@ -2,6 +2,7 @@ import os
 import subprocess
 import time
 import sys
+import pytz
 
 
 def runMysql(args):
@@ -41,32 +42,50 @@ def updateMysql(args):
   sleep = 30
   is_succeed = False
   while True:
-    if not is_succeed:
-      mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
-      mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-      result = mysql_upgrade.communicate()[0]
-      if mysql_upgrade.returncode is None:
-        mysql_upgrade.kill()
-      if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
-        print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
-        print 'Sleeping for %ss and retrying' % sleep
+    mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
+    mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    result = mysql_upgrade.communicate()[0]
+    if mysql_upgrade.returncode is None:
+      mysql_upgrade.kill()
+    if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
+      print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
+    else:
+      if mysql_upgrade.returncode == 0:
+        print "MySQL database upgraded with result:\n%s" % result
       else:
-        if mysql_upgrade.returncode == 0:
-          print "MySQL database upgraded with result:\n%s" % result
-        else:
-          print "No need to upgrade MySQL database"
-        mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
-        mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        result = mysql.communicate(conf['mysql_script'])[0]
-        if mysql.returncode is None:
-          mysql.kill()
+        print "No need to upgrade MySQL database"
+      mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
+      mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
+          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+      result = mysql.communicate(conf['mysql_script'])[0]
+      if mysql.returncode is None:
+        mysql.kill()
+      if mysql.returncode != 0:
+        print 'Command %r failed with:\n%s' % (mysql_list, result)
+      else:
+        # import timezone database
+        mysql_tzinfo_to_sql_binary = os.path.join(
+          os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
+        zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
+        mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
+        mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
         if mysql.returncode != 0:
-          print 'Command %r failed with:\n%s' % (mysql_list, result)
-          print 'Sleeping for %ss and retrying' % sleep
+          print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
         else:
+          mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          result = mysql.communicate(timezone_sql)[0]
+          if mysql.returncode is None:
+            mysql.kill()
+          if mysql.returncode != 0:
+            print 'Command %r failed with:\n%s' % (mysql_list, result)
           is_succeed = True
-          print 'SlapOS initialisation script succesfully applied on database.'
+    if is_succeed:
+      print 'SlapOS initialisation script succesfully applied on database.'
+      break
+    print 'Sleeping for %ss and retrying' % sleep
     sys.stdout.flush()
     sys.stderr.flush()
     time.sleep(sleep)

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -24,7 +24,7 @@ long_query_time = 1
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so;handlersocket.so
+plugin-load = ha_mroonga.so;handlersocket.so
 
 # By default only 100 connections are allowed, when using zeo
 # we may have much more connections

slapos/recipe/haproxy/__init__.py

@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe):
     server_snippet = ""
     i = 0
     name = self.options['name']
-    for address in self.options['backend-list'].split():
+    backend_list = self.options['backend-list']
+    if isinstance(backend_list, str):
+      # BBB
+      backend_list = backend_list.split()
+    for address in backend_list:
       i += 1
       server_snippet += self.substituteTemplate(
           snippet_filename, dict(

slapos/recipe/kvm/__init__.py

@@ -33,14 +33,25 @@ class Recipe(GenericBaseRecipe):
   kvm instance configuration.
   """
   def install(self):
+    # Sanitize drive type parameter
+    self.options.setdefault('disk-type', 'virtio')
+    if not self.options.get('disk-type') in ['ide', 'scsi', 'sd',
+        'mtd', 'floppy', 'pflash', 'virtio']:
+      print 'Warning: "disk-type" parameter is not in allowed values. Using ' \
+          '"virtio" value.'
+      self.options['disk-type'] = 'virtio'
+
     config = dict(
       tap_interface=self.options['tap'],
       vnc_ip=self.options['vnc-ip'],
       vnc_port=self.options['vnc-port'],
-      nbd_ip=self.options['nbd-ip'],
+      nbd_ip=self.options['nbd-host'],
       nbd_port=self.options['nbd-port'],
+      nbd2_ip=self.options.get('nbd2-host', ''),
+      nbd2_port=self.options.get('nbd2-port', 1024),
       disk_path=self.options['disk-path'],
       disk_size=self.options['disk-size'],
+      disk_type=self.options['disk-type'],
       mac_address=self.options['mac-address'],
       smp_count=self.options['smp-count'],
       ram_size=self.options['ram-size'],
@@ -50,7 +61,6 @@ class Recipe(GenericBaseRecipe):
       shell_path=self.options['shell-path'],
       qemu_path=self.options['qemu-path'],
       qemu_img_path=self.options['qemu-img-path'],
-      # XXX Weak password
       vnc_passwd=self.options['passwd']
     )
 
@@ -67,4 +77,3 @@ class Recipe(GenericBaseRecipe):
 
 
     return [runner_path, controller_path]
-

slapos/recipe/kvm/template/kvm_run.in

@@ -30,26 +30,32 @@ def getSocketStatus(host, port):
 disk_path = '%(disk_path)s'
 if not os.path.exists(disk_path):
   subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
-      '%(disk_path)s', '%(disk_size)sG'])
+      disk_path, '%(disk_size)sG'])
 
-kvm_argument_list = ['%(qemu_path)s', '-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s',
+kvm_argument_list = ['%(qemu_path)s',
+  '-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s',
   '-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no',
   '-smp', '%(smp_count)s',
   '-m', '%(ram_size)s',
-  '-drive', 'file=%(disk_path)s,if=virtio,boot=on',
+  '-drive', 'file=%(disk_path)s,if=%(disk_type)s',
   '-vnc', '%(vnc_ip)s:1,ipv4,password',
   '-boot', 'menu=on',
   '-qmp', 'unix:%(socket_path)s,server',
   '-pidfile', '%(pid_file_path)s',
 ]
 
-# Try to connect to NBD server
-s = getSocketStatus('%(nbd_ip)s', %(nbd_port)s)
-if s is None:
-  # NBD is not available : launch kvm without it
-  print 'Warning : Nbd is not available.'
-  os.execv('%(qemu_path)s', kvm_argument_list)
-else:
-  # NBD is available
-  kvm_argument_list.extend(['-cdrom', 'nbd:[%(nbd_ip)s]:%(nbd_port)s'])
-  os.execv('%(qemu_path)s', kvm_argument_list)
+# Try to connect to NBD server (and second nbd if defined)
+for nbd_ip, nbd_port in (
+    ('%(nbd_ip)s', %(nbd_port)s), ('%(nbd2_ip)s', %(nbd2_port)s)):
+  if nbd_ip and nbd_port:
+    s = getSocketStatus(nbd_ip, nbd_port)
+    if s is None:
+      # NBD is not available : launch kvm without it
+      print 'Warning : Nbd is not available.'
+    else:
+      # NBD is available
+      kvm_argument_list.extend([
+          '-drive',
+          'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
+
+os.execv('%(qemu_path)s', kvm_argument_list)

slapos/recipe/lamp/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/librecipe/__init__.py

@@ -90,13 +90,22 @@ class BaseSlapRecipe:
     ]
 
     # SLAP related information
-    slap_connection = buildout['slap_connection']
-    self.computer_id = slap_connection['computer_id']
-    self.computer_partition_id = slap_connection['partition_id']
-    self.server_url = slap_connection['server_url']
-    self.software_release_url = slap_connection['software_release_url']
-    self.key_file = slap_connection.get('key_file')
-    self.cert_file = slap_connection.get('cert_file')
+    try:
+      slap_connection = buildout['slap_connection']
+      self.computer_id = slap_connection['computer_id']
+      self.computer_partition_id = slap_connection['partition_id']
+      self.server_url = slap_connection['server_url']
+      self.software_release_url = slap_connection['software_release_url']
+      self.key_file = slap_connection.get('key_file')
+      self.cert_file = slap_connection.get('cert_file')
+    except zc.buildout.buildout.MissingSection:
+      slap_connection = buildout['slap-connection']
+      self.computer_id = slap_connection['computer-id']
+      self.computer_partition_id = slap_connection['partition-id']
+      self.server_url = slap_connection['server-url']
+      self.software_release_url = slap_connection['software-release-url']
+      self.key_file = slap_connection.get('key-file')
+      self.cert_file = slap_connection.get('cert-file')
 
     # setup egg to give possibility to generate scripts
     self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
@@ -145,13 +154,6 @@ class BaseSlapRecipe:
     self._writeExecutable(wrapper_path, file_content)
     return wrapper_path
 
-  def createReportRunningWrapper(self, file_content):
-    """Creates report runnig wrapper and returns its path"""
-    report_wrapper_path = os.path.join(self.wrapper_report_directory,
-        'slapreport')
-    self._writeExecutable(report_wrapper_path, file_content)
-    return report_wrapper_path
-
   def substituteTemplate(self, template_location, mapping_dict):
     """Returns template content after substitution"""
     return open(template_location, 'r').read() % mapping_dict

slapos/recipe/librecipe/generic.py

+# -*- coding: utf-8 -*-
+# vim: set et sts=2:
 ##############################################################################
 #
 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
@@ -24,6 +26,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import io
 import logging
 import os
 import sys
@@ -35,6 +38,8 @@ import urlparse
 import pkg_resources
 import zc.buildout
 
+from slapos.recipe.librecipe import shlex
+
 class GenericBaseRecipe(object):
   """Boilerplate class for all Buildout recipes providing helpful methods like
      creating configuration file, creating wrappers, generating passwords, etc.
@@ -88,6 +93,21 @@ class GenericBaseRecipe(object):
   def createExecutable(self, name, content, mode=0700):
     return self.createFile(name, content, mode)
 
+  def addLineToFile(self, filepath, line, encoding='utf8'):
+    """Append a single line to a text file, if the line does not exist yet.
+
+    line must be unicode."""
+
+    try:
+      lines = io.open(filepath, 'r', encoding=encoding).readlines()
+    except IOError:
+      lines = []
+
+    if not line in lines:
+      lines.append(line)
+      with io.open(filepath, 'w+', encoding=encoding) as f:
+        f.write(u'\n'.join(lines))
+
   def createPythonScript(self, name, absolute_function, arguments=''):
     """Create a python script using zc.buildout.easy_install.scripts
 
@@ -107,6 +127,34 @@ class GenericBaseRecipe(object):
       path, arguments=arguments)[0]
     return script
 
+  def createWrapper(self, name, command, parameters, comments=[], parameters_extra=False):
+    """
+    Creates a very simple (one command) shell script for process replacement.
+    Takes care of quoting.
+    """
+
+    lines = [ '#!/bin/sh' ]
+
+    for comment in comments:
+      lines.append('# %s' % comment)
+
+    lines.append('exec %s' % shlex.quote(command))
+
+    for param in parameters:
+      if len(lines[-1]) < 40:
+        lines[-1] += ' ' + shlex.quote(param)
+      else:
+        lines[-1] += ' \\'
+        lines.append('\t' + shlex.quote(param))
+
+    if parameters_extra:
+        # pass-through further parameters
+        lines[-1] += ' \\'
+        lines.append('\t$@')
+
+    content = '\n'.join(lines) + '\n'
+    return self.createFile(name, content, 0700)
+
   def createDirectory(self, parent, name, mode=0700):
     path = os.path.join(parent, name)
     if not os.path.exists(path):

slapos/recipe/librecipe/shlex.py

+# -*- coding: utf-8 -*-
+
+"""
+backported part of shlex.py from Python 3.3
+"""
+
+import re
+
+
+_find_unsafe = re.compile(r'[^\w@%+=:,./-]', 256).search
+
+def quote(s):
+    """Return a shell-escaped version of the string *s*."""
+    if not s:
+        return "''"
+    if _find_unsafe(s) is None:
+        return s
+
+    # use single quotes, and put single quotes into double quotes
+    # the string $'b is then quoted as '$'"'"'b'
+    return "'" + s.replace("'", "'\"'\"'") + "'"

slapos/recipe/mydumper.py

@@ -28,85 +28,76 @@ import subprocess
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def dump(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
-
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
-
-  if args['compression']:
-    mydumper_cmd.append('--compress')
+def _mydumper_base_cmd(mydumper, database, user, password,
+                       socket=None, host=None, port=None, **kw):
+  cmd = [mydumper]
+  cmd.extend(['-B', database])
 
-  if args['rows'] is not None:
-    mydumper_cmd.extend(['-r', args['rows']])
-
-  mydumper_cmd.extend(['-o', args['directory']])
+  if socket:
+    cmd.extend(['-S', socket])
+  else:
+    cmd.extend(['-h', host])
+    cmd.extend(['-P', port])
 
-  subprocess.check_call(mydumper_cmd)
+  cmd.extend(['-u', user])
+  if password:
+    cmd.extend(['-p', password])
 
+  return cmd
 
 
-def do_import(args):
-  mydumper_cmd = [args['mydumper']]
-  mydumper_cmd.extend(['-B', args['database']])
+def do_export(args):
+  cmd = _mydumper_base_cmd(**args)
 
-  if args['socket'] is not None:
-    mydumper_cmd.extend(['-S', args['socket']])
-  else:
-    mydumper_cmd.extend(['-h', args['host']])
-    mydumper_cmd.etxned(['-P', args['port']])
+  if args['compression']:
+    cmd.append('--compress')
 
-  mydumper_cmd.extend(['-u', args['user']])
-  if args['password'] is not None:
-    mydumper_cmd.extend(['-p', args['password']])
+  if args['rows'] is not None:
+    cmd.extend(['-r', args['rows']])
 
-  mydumper_cmd.append('--overwrite-tables')
+  cmd.extend(['-o', args['directory']])
 
-  mydumper_cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
-  subprocess.check_call(mydumper_cmd)
 
+def do_import(args):
+  cmd = _mydumper_base_cmd(**args)
+  cmd.append('--overwrite-tables')
+  cmd.extend(['-d', args['directory']])
+  subprocess.check_call(cmd)
 
 
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    # Host or socket should be defined
-    try:
-      self.options['host']
-    except:
-      self.options['socket']
-
-    config = dict(database=self.options['database'],
-                  socket=self.options.get('socket'),
-                  host=self.options.get('host'),
-                  port=self.options.get('port', 3306),
-                  directory=self.options['backup-directory'],
-                  user=self.options['user'],
-                  password=self.options.get('password'),
-                 )
-
-    name = __name__
+    config = {
+      'database': self.options['database'],
+      'directory': self.options['backup-directory'],
+      'user': self.options['user'],
+      'password': self.options.get('password'),
+    }
+
+    if self.options.get('host'):
+      config['host'] = self.options['host']
+      config['port'] = self.options.get('port', 3306)
+    elif self.options.get('socket'):
+      config['socket'] = self.options['socket']
+    else:
+      raise ValueError("host or socket must be defined")
+
     if self.optionIsTrue('import', False):
-      config.update(mydumper=self.options['myloader-binary'])
-      name += '.do_import'
+      function = do_import
+      config['mydumper'] = self.options['myloader-binary']
     else:
-      config.update(mydumper=self.options['mydumper-binary'],
-                    compression=self.optionIsTrue('compression', default=False),
-                    rows=self.options.get('rows'),
-                   )
-      name += '.dump'
+      function = do_export
+      config['mydumper'] = self.options['mydumper-binary']
+      config['compression'] = self.optionIsTrue('compression', default=False)
+      config['rows'] = self.options.get('rows')
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-                                      name,
-                                      config)
+    wrapper = self.createPythonScript(name=self.options['wrapper'],
+                                      absolute_function = '%s.%s' % (__name__, function.func_name),
+                                      arguments=config)
 
     return [wrapper]
+

slapos/recipe/mysql/__init__.py

@@ -31,6 +31,7 @@ import subprocess
 from slapos.recipe.librecipe import GenericBaseRecipe
 from slapos.recipe.librecipe import filehash
 
+
 class Recipe(GenericBaseRecipe):
 
   def _options(self, options):

slapos/recipe/mysql/template/my.cnf.in

@@ -19,7 +19,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

slapos/recipe/notifier.py

@@ -31,59 +31,76 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    commandline = [self.options['server-binary']]
-    commandline.extend(['--callbacks', self.options['callbacks']])
-    commandline.extend(['--feeds', self.options['feeds']])
-    commandline.extend(['--equeue-socket', self.options['equeue-socket']])
-    commandline.append(self.options['host'])
-    commandline.append(self.options['port'])
+    options = self.options
+    script = self.createWrapper(name=options['wrapper'],
+                                command=options['server-binary'],
+                                parameters=[
+                                   '--callbacks', options['callbacks'],
+                                   '--feeds', options['feeds'],
+                                   '--equeue-socket', options['equeue-socket'],
+                                   options['host'], options['port']
+                                   ],
+                                comments=[
+                                    '',
+                                    'Upon receiving a notification, execute the callback(s).',
+                                    ''])
+    return [script]
 
-    return [self.createPythonScript(self.options['wrapper'],
-                                    'slapos.recipe.librecipe.execute.execute',
-                                    commandline)]
 
 class Callback(GenericBaseRecipe):
 
   def createCallback(self, notification_id, callback):
     callback_id = sha512(notification_id).hexdigest()
-    callback = self.createFile(os.path.join(self.options['callbacks'],
-                                            callback_id),
-                               callback)
-    return callback
+
+    filepath = os.path.join(self.options['callbacks'], callback_id)
+    self.addLineToFile(filepath, callback)
+    return filepath
 
   def install(self):
+    # XXX this path is returned multiple times, one for each callback that has been added.
     return [self.createCallback(self.options['on-notification-id'],
                                 self.options['callback'])]
 
 class Notify(GenericBaseRecipe):
 
-  def createNotifier(self, notifier_binary, executable, wrapper, **kwargs):
-    if not os.path.exists(kwargs['log']):
+  def createNotifier(self, notifier_binary, wrapper, executable,
+                     log, title, notification_url, feed_url):
+
+    if not os.path.exists(log):
       # Just a touch
-      open(kwargs['log'], 'w').close()
-
-    commandline = [notifier_binary,
-                   '-l', kwargs['log'],
-                   '--title', kwargs['title'],
-                   '--feed', kwargs['feed_url'],
-                   '--notification-url', kwargs['notification_url'],
-                   executable]
-    return self.createPythonScript(wrapper,
-                                   'slapos.recipe.librecipe.execute.execute',
-                                   [str(i) for i in commandline])
+      open(log, 'w').close()
+
+    parameters = [
+            '-l', log,
+            '--title', title,
+            '--feed', feed_url,
+            '--notification-url',
+            ]
+    parameters.extend(notification_url.split(' '))
+    parameters.extend(['--executable', executable])
+
+    return self.createWrapper(name=wrapper,
+                              command=notifier_binary,
+                              parameters=parameters,
+                              comments=[
+                                  '',
+                                  'Call an executable and send notification(s).',
+                                  ''])
+
 
   def install(self):
-    feedurl = self.unparseUrl(scheme='http', host=self.options['host'],
-                              port=self.options['port'],
-                              path='/get/%s' % self.options['name'])
-
-    script = self.createNotifier(
-      self.options['notifier-binary'],
-      wrapper=self.options['wrapper'],
-      executable=self.options['executable'],
-      log=os.path.join(self.options['feeds'], self.options['name']),
-      title=self.options['title'],
-      notification_url=self.options['notify'],
-      feed_url=feedurl,
-    )
+    feed_url = self.unparseUrl(scheme='http', host=self.options['host'],
+                               port=self.options['port'],
+                               path='/get/%s' % self.options['name'])
+
+    log = os.path.join(self.options['feeds'], self.options['name'])
+
+    options = self.options
+    script = self.createNotifier(notifier_binary=options['notifier-binary'],
+                                 wrapper=options['wrapper'],
+                                 executable=options['executable'],
+                                 log=log,
+                                 title=options['title'],
+                                 notification_url=options['notify'],
+                                 feed_url=feed_url)
     return [script]

slapos/recipe/pbs.py

@@ -24,14 +24,15 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from json import loads as unjson
+
 from hashlib import sha512
-from urlparse import urlparse
+import inspect
+import json
 import os
+import signal
 import subprocess
 import sys
-import signal
-import inspect
+import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -43,8 +44,7 @@ from slapos import slap as slapmodule
 def promise(args):
 
   def failed_ssh(partition, ssh):
-    # Bad python 2 syntax, looking forward python 3 to have print(file=)
-    print >> sys.stderr, "SSH Connection failed"
+    sys.stderr.write("SSH Connection failed\n")
     try:
       ssh.terminate()
     except:
@@ -75,16 +75,20 @@ def promise(args):
 
   slap = slapmodule.slap()
   slap.initializeConnection(args['server_url'],
-    key_file=args.get('key_file'), cert_file=args.get('cert_file'))
+                            key_file=args.get('key_file'),
+                            cert_file=args.get('cert_file'))
+
   partition = slap.registerComputerPartition(args['computer_id'],
                                              args['partition_id'])
 
+  ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
+                         stdin=subprocess.PIPE,
+                         stdout=open(os.devnull, 'w'),
+                         stderr=open(os.devnull, 'w'))
+
   # Rdiff Backup protocol quit command
   quitcommand = 'q' + chr(255) + chr(0) * 7
-  ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
 
-  ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
-                         stdout=open(os.devnull), stderr=open(os.devnull))
   ssh.stdin.write(quitcommand)
   ssh.stdin.flush()
   ssh.stdin.close()
@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     promise_path = os.path.join(self.options['promises-directory'],
                                 url_hash)
-    parsed_url = urlparse(url)
+    parsed_url = urlparse.urlparse(url)
     promise_dict = self.promise_base_dict.copy()
     promise_dict.update(user=parsed_url.username,
                         host=parsed_url.hostname,
@@ -127,15 +131,17 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     host = parsed_url.hostname
     known_hosts_file[host] = entry['server-key']
 
-    remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
+    # XXX use -y because the host might not yet be in the
+    #     trusted hosts file until the next time slapgrid is run.
+
+    remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % \
       {
         'ssh': self.options['sshclient-binary'],
         'user': parsed_url.username,
         'host': parsed_url.hostname,
       }
 
-    command = [self.options['rdiffbackup-binary']]
-    command.extend(['--remote-schema', remote_schema])
+    parameters = ['--remote-schema', remote_schema]
 
     remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
                                                'path': parsed_url.path}
@@ -144,38 +150,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
                                            name_hash)
 
     if entry['type'] == 'push':
-      command.extend(['--restore-as-of', 'now'])
-      command.append('--force')
-      command.extend([local_directory, remote_directory])
+      parameters.extend(['--restore-as-of', 'now'])
+      parameters.append('--force')
+      parameters.extend([local_directory, remote_directory])
+      comments = ['','Push data to a PBS *-import instance.','']
     else:
-      command.extend([remote_directory, local_directory])
+      parameters.extend([remote_directory, local_directory])
+      comments = ['','Pull data from a PBS *-export instance.','']
 
     wrapper_basepath = os.path.join(self.options['wrappers-directory'],
                                     url_hash)
 
-    wrapper_path = wrapper_basepath
     if 'notify' in entry:
-      wrapper_path = '%s_raw' % wrapper_basepath
+      wrapper_path = wrapper_basepath + '_raw'
+    else:
+      wrapper_path = wrapper_basepath
 
-    wrapper = self.createPythonScript(
-       wrapper_path,
-      'slapos.recipe.librecipe.execute.execute',
-      [str(i) for i in command]
-    )
+    wrapper = self.createWrapper(name=wrapper_path,
+                                 command=self.options['rdiffbackup-binary'],
+                                 parameters=parameters,
+                                 comments = comments)
     path_list.append(wrapper)
 
     if 'notify' in entry:
       feed_url = '%s/get/%s' % (self.options['notifier-url'],
                                 entry['notification-id'])
-      wrapper = self.createNotifier(
-        self.options['notifier-binary'],
-        wrapper=wrapper_basepath,
-        executable=wrapper_path,
-        log=os.path.join(self.options['feeds'], entry['notification-id']),
-        title=entry.get('title', 'Untitled'),
-        notification_url=entry['notify'],
-        feed_url=feed_url,
-      )
+      wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
+                                    wrapper=wrapper_basepath,
+                                    executable=wrapper_path,
+                                    log=os.path.join(self.options['feeds'], entry['notification-id']),
+                                    title=entry.get('title', 'Untitled'),
+                                    notification_url=entry['notify'],
+                                    feed_url=feed_url,
+                                  )
       path_list.append(wrapper)
       #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
 
@@ -190,39 +197,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     return path_list
 
+
   def _install(self):
     path_list = []
 
-
     if self.optionIsTrue('client', True):
       self.logger.info("Client mode")
 
       slap_connection = self.buildout['slap-connection']
-      self.promise_base_dict = dict(
-        server_url=slap_connection['server-url'],
-        computer_id=slap_connection['computer-id'],
-        cert_file=slap_connection.get('cert-file'),
-        key_file=slap_connection.get('key-file'),
-        partition_id=slap_connection['partition-id'],
-        ssh_client=self.options['sshclient-binary'],
-      )
-
-      slaves = unjson(self.options['slave-instance-list'])
+      self.promise_base_dict = {
+              'server_url': slap_connection['server-url'],
+              'computer_id': slap_connection['computer-id'],
+              'cert_file': slap_connection.get('cert-file'),
+              'key_file': slap_connection.get('key-file'),
+              'partition_id': slap_connection['partition-id'],
+              'ssh_client': self.options['sshclient-binary'],
+        }
+
+      slaves = json.loads(self.options['slave-instance-list'])
       known_hosts = KnownHostsFile(self.options['known-hosts'])
       with known_hosts:
+        # XXX this API could be cleaner
         for slave in slaves:
           path_list.extend(self.add_slave(slave, known_hosts))
-
     else:
-      command = [self.options['rdiffbackup-binary']]
       self.logger.info("Server mode")
-      command.extend(['--restrict', self.options['path']])
-      command.append('--server')
 
-      wrapper = self.createPythonScript(
-        self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        command)
+      wrapper = self.createWrapper(name=self.options['wrapper'],
+                                   command=self.options['rdiffbackup-binary'],
+                                   parameters=[
+                                       '--restrict', self.options['path'],
+                                       '--server'
+                                       ])
       path_list.append(wrapper)
 
     return path_list
+

slapos/recipe/postgres/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import md5
+import os
+import subprocess
+import textwrap
+from zc.buildout import UserError
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+
+
+class Recipe(GenericBaseRecipe):
+    """\
+    This recipe creates:
+
+        - a Postgres cluster
+        - configuration to allow connections from IPV6 only (or unix socket)
+        - a superuser with provided name and generated password
+        - a database with provided name
+        - a foreground start script in the services directory
+
+    then adds the connection URL to the options.
+    The URL can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
+    """
+
+    def fetch_ipv6_host(self, options):
+        """
+        Returns a string represtation of ipv6_host.
+        May receive a regular string, a set or a string serialized by buildout.
+        """
+        ipv6_host = options['ipv6_host']
+
+        if isinstance(ipv6_host, set):
+            return ipv6_host.pop()
+        else:
+            return ipv6_host
+
+
+    def _options(self, options):
+        options['password'] = self.generatePassword()
+        options['url'] = 'postgresql://%(user)s:%(password)s@[%(ipv4_host)s]:%(port)s/%(dbname)s' % options
+
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+
+        if not os.path.exists(pgdata):
+            self.createCluster()
+            self.createConfig()
+            self.createDatabase()
+            self.createSuperuser()
+            self.createRunScript()
+
+        return [
+                # XXX should we really return something here?
+                # os.path.join(pgdata, 'postgresql.conf')
+                ]
+
+
+    def check_exists(self, path):
+        if not os.path.isfile(path):
+            raise IOError('File not found: %s' % path)
+
+
+    def createCluster(self):
+        initdb_binary = os.path.join(self.options['bin'], 'initdb')
+        self.check_exists(initdb_binary)
+
+        pgdata = self.options['pgdata-directory']
+
+        try:
+            subprocess.check_call([initdb_binary,
+                                   '-D', pgdata,
+                                   '-A', 'ident',
+                                   '-E', 'UTF8',
+                                   ])
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create cluster directory in %s' % pgdata)
+
+
+    def createConfig(self):
+        pgdata = self.options['pgdata-directory']
+
+        with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
+            cfg.write(textwrap.dedent("""\
+                    listen_addresses = '%s,%s'
+                    logging_collector = on
+                    log_rotation_size = 50MB
+                    max_connections = 100
+                    datestyle = 'iso, mdy'
+
+                    lc_messages = 'en_US.UTF-8'
+                    lc_monetary = 'en_US.UTF-8'
+                    lc_numeric = 'en_US.UTF-8'
+                    lc_time = 'en_US.UTF-8'
+                    default_text_search_config = 'pg_catalog.english'
+
+                    unix_socket_directory = '%s'
+                    unix_socket_permissions = 0700
+                    """ % (
+                        self.options['ipv4_host'],
+                        self.fetch_ipv6_host(self.options),
+                        pgdata,
+                        )))
+
+
+        with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
+            # see http://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html
+
+            cfg.write(textwrap.dedent("""\
+                    # TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+                    # "local" is for Unix domain socket connections only (check unix_socket_permissions!)
+                    local   all             all                                     ident
+                    host    all             all             127.0.0.1/32            md5
+                    host    all             all             %s/32                   md5
+                    host    all             all             ::1/128                 md5
+                    host    all             all             %s/128                  md5
+                    """ % (self.options['ipv4_host'], self.fetch_ipv6_host(self.options))))
+
+
+    def createDatabase(self):
+        self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
+
+
+    def createSuperuser(self):
+        """
+        Creates a Postgres superuser - other than "slapuser#" for use by the application.
+        """
+
+        # http://postgresql.1045698.n5.nabble.com/Algorithm-for-generating-md5-encrypted-password-not-found-in-documentation-td4919082.html
+
+        user = self.options['user']
+        password = self.options['password']
+
+        # encrypt the password to avoid storing in the logs
+        enc_password = 'md5' + md5.md5(password+user).hexdigest()
+
+        self.runPostgresCommand(cmd="""CREATE USER "%s" ENCRYPTED PASSWORD '%s' SUPERUSER""" % (user, enc_password))
+
+
+    def runPostgresCommand(self, cmd):
+        """
+        Executes a command in single-user mode, with no daemon running.
+
+        Multiple commands can be executed by providing newlines,
+        preceeded by backslash, between them.
+        See http://www.postgresql.org/docs/9.1/static/app-postgres.html
+        """
+
+        pgdata = self.options['pgdata-directory']
+        postgres_binary = os.path.join(self.options['bin'], 'postgres')
+
+        try:
+            p = subprocess.Popen([postgres_binary,
+                                  '--single',
+                                  '-D', pgdata,
+                                  'postgres',
+                                  ], stdin=subprocess.PIPE)
+
+            p.communicate(cmd+'\n')
+        except subprocess.CalledProcessError:
+            raise UserError('Could not create database %s' % pgdata)
+
+
+    def createRunScript(self):
+        """
+        Creates a script that runs postgres in the foreground.
+        'exec' is used to allow easy control by supervisor.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                exec %(bin)s/postgres \\
+                    -D %(pgdata-directory)s
+                """ % self.options)
+        name = os.path.join(self.options['services'], 'postgres-start')
+        self.createExecutable(name, content=content)
+
+
+
+class ExportRecipe(GenericBaseRecipe):
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+        wrapper = self.options['wrapper']
+        self.createBackupScript(wrapper)
+        return [wrapper]
+
+
+    def createBackupScript(self, wrapper):
+        """
+        Create a script to backup the database in 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                umask 077
+                %(bin)s/pg_dump \\
+                        --host=%(pgdata-directory)s \\
+                        --format=custom \\
+                        --file=%(backup-directory)s/database.dump \\
+                        %(dbname)s
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+class ImportRecipe(GenericBaseRecipe):
+
+    def install(self):
+        pgdata = self.options['pgdata-directory']
+        wrapper = self.options['wrapper']
+        self.createRestoreScript(wrapper)
+        return [wrapper]
+
+
+    def createRestoreScript(self, wrapper):
+        """
+        Create a script to restore the database from 'custom' format.
+        """
+        content = textwrap.dedent("""\
+                #!/bin/sh
+                %(bin)s/pg_restore \\
+                        --host=%(pgdata-directory)s \\
+                        --dbname=%(dbname)s \\
+                        --clean \\
+                        --no-owner \\
+                        --no-acl \\
+                        %(backup-directory)s/database.dump
+                """ % self.options)
+        self.createExecutable(wrapper, content=content)
+
+
+
+

slapos/recipe/request.py

@@ -25,8 +25,11 @@
 #
 ##############################################################################
 import logging
-
 from slapos import slap as slapmodule
+import slapos.recipe.librecipe.generic as librecipe
+import traceback
+
+DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
 class Recipe(object):
   """
@@ -97,17 +100,15 @@ class Recipe(object):
     request = slap.registerComputerPartition(
       options['computer-id'], options['partition-id']).request
 
-    isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
-
     return_parameters = []
     if 'return' in options:
       return_parameters = [str(parameter).strip()
-        for parameter in options['return'].split()]
+                          for parameter in options['return'].split()]
     else:
       self.logger.debug("No parameter to return to main instance."
         "Be careful about that...")
 
-    software_type = options.get('software-type', 'RootInstanceSoftware')
+    software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
 
     filter_kw = {}
     if 'sla' in options:
@@ -120,26 +121,46 @@ class Recipe(object):
         partition_parameter_kw[config_parameter] = \
             options['config-%s' % config_parameter]
 
-    self.instance = instance = request(software_url, software_type,
-      name, partition_parameter_kw=partition_parameter_kw,
-      filter_kw=filter_kw, shared=isSlave)
+    isSlave = options.get('slave', '').lower() in \
+        librecipe.GenericBaseRecipe.TRUE_VALUES
+
+    self._raise_request_exception = None
+    self._raise_request_exception_formatted = None
+    self.instance = None
+    try:
+      self.instance = request(software_url, software_type,
+          name, partition_parameter_kw=partition_parameter_kw,
+          filter_kw=filter_kw, shared=isSlave)
+      # XXX what is the right way to get a global id?
+      options['instance_guid'] = self.instance.getId()
+    except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
+      self._raise_request_exception = exc
+      self._raise_request_exception_formatted = traceback.format_exc()
 
     for param in return_parameters:
+      options['connection-%s' % param] = ''
+      if not self.instance:
+        continue
       try:
         options['connection-%s' % param] = str(
-          instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
-        options['connection-%s' % param] = ''
+          self.instance.getConnectionParameter(param))
+      except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
         if self.failed is None:
           self.failed = param
 
   def install(self):
+    if self._raise_request_exception:
+      raise self._raise_request_exception
+
     if self.failed is not None:
       # Check instance status to know if instance has been deployed
       try:
-        status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet, please try again'
+        if self.instance._computer_id is not None:
+          status = self.instance.getState()
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
+        status = 'not ready yet'
       except AttributeError:
         status = 'unknown'
       error_message = 'Connection parameter %s not found. '\
@@ -150,3 +171,31 @@ class Recipe(object):
     return []
 
   update = install
+
+class RequestOptional(Recipe):
+  """
+  Request a SlapOS instance. Won't fail if request failed or is not ready.
+  Same as slapos.cookbook:request, but won't raise in case of problem.
+  """
+  def install(self):
+    if self._raise_request_exception_formatted:
+      self.logger.warning('Optional request failed:')
+      self.logger.warning(self._raise_request_exception_formatted)
+    elif self.failed is not None:
+      # Check instance status to know if instance has been deployed
+      try:
+        if self.instance._computer_id is not None:
+          status = self.instance.getState()
+        else:
+          status = 'not ready yet'
+      except (slapmodule.NotFoundError, slapmodule.ServerError):
+        status = 'not ready yet'
+      except AttributeError:
+        status = 'unknown'
+      error_message = 'Connection parameter %s not found. '\
+          'Requested instance is currently %s. If this error persists, '\
+          'check status of this instance.' % (self.failed, status)
+      self.logger.warning(error_message)
+    return []
+
+  update = install

slapos/recipe/requestoptional.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import logging
-
-from slapos import slap as slapmodule
-
-class Recipe(object):
-  """ Request a SlapOS instance. Won't fail if instance is not ready.
-  """
-
-  def __init__(self, buildout, name, options):
-    self.logger = logging.getLogger(name)
-
-    slap = slapmodule.slap()
-
-    self.software_release_url = options['software-url']
-    self.name = options['name']
-
-    slap.initializeConnection(options['server-url'],
-                              options.get('key-file'),
-                              options.get('cert-file'),
-                             )
-    computer_partition = slap.registerComputerPartition(
-      options['computer-id'], options['partition-id'])
-    self.request = computer_partition.request
-
-    self.isSlave = False
-    if 'slave' in options:
-      self.isSlave = options['slave'].lower() in ['y', 'yes', 'true', '1']
-
-    self.return_parameters = []
-    if 'return' in options:
-      self.return_parameters = [str(parameter).strip()
-                               for parameter in options['return'].split()]
-    else:
-      self.logger.debug("No parameter to return to main instance."
-                          "Be careful about that...")
-
-    software_type = 'RootSoftwareInstance'
-    if 'software-type' in options:
-      software_type = options['software-type']
-
-    filter_kw = {}
-    if 'sla' in options:
-      for sla_parameter in options['sla'].split():
-        filter_kw[sla_parameter] = options['sla-%s' % sla_parameter]
-
-    partition_parameter_kw = {}
-    if 'config' in options:
-      for config_parameter in options['config'].split():
-        partition_parameter_kw[config_parameter] = \
-            options['config-%s' % config_parameter]
-
-    self.instance = self.request(options['software-url'], software_type,
-      self.name, partition_parameter_kw=partition_parameter_kw,
-      filter_kw=filter_kw, shared=self.isSlave)
-
-    self.failed = None
-    for param in self.return_parameters:
-      try:
-        options['connection-%s' % param] = str(
-            self.instance.getConnectionParameter(param))
-      except slapmodule.NotFoundError:
-        options['connection-%s' % param] = ''
-        if self.failed is None:
-          self.failed = param
-
-  def install(self):
-    if self.failed is not None:
-      # Check instance status to know if instance has been deployed
-      try:
-        status = self.instance.getState()
-      except slapmodule.NotFoundError:
-        status = 'not ready yet'
-      except AttributeError:
-        status = 'unknown'
-      error_message = 'Connection parameter %s not found. '\
-          'Requested instance is currently %s. If this error persists, '\
-          'check status of this instance.' % (self.failed, status)
-      self.logger.error(error_message)
-    return []
-
-  update = install

slapos/recipe/slapmonitor/__init__.py

@@ -24,31 +24,31 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
 
-class Recipe(GenericBaseRecipe):
-  """
-  Slapmonitor instance configuration.
-  """
+from slapos.recipe.librecipe import GenericBaseRecipe
 
-  def __init__(self, buildout, name, options):
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+class MonitorRecipe(GenericBaseRecipe):
 
   def install(self):
-    config = dict(
-      pid_file_path=self.options['pid-file'],
-      database_path=self.options['database-path'],
-      slapmonitor_path = self.options['slapmonitor-path'],
-      shell_path=self.options['shell-path'],
-    )
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapmonitor-path'],
+                                parameters=[
+                                    options['pid-file'],
+                                    options['database-path'],
+                                    ])
+    return [script]
 
-    # Runners
-    runner_path = self.createExecutable(
-      self.options['path'],
-      self.substituteTemplate(self.getTemplateFilename('slapmonitor_run.in'),
-                              config))
 
-    return [runner_path]
+class MonitorXMLRecipe(GenericBaseRecipe):
+
+  def install(self):
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapmonitor-xml-path'],
+                                parameters=[
+                                    options['database-path'],
+                                    ],
+                                parameters_extra=True)
+    return [script]
 
-  def update(self):
-    pass

slapos/recipe/slapmonitor/template/slapmonitor_run.in

-#!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-#
-exec %(slapmonitor_path)s %(pid_file_path)s %(database_path)s
-

slapos/recipe/slapreport/__init__.py

@@ -24,36 +24,23 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
-  """
-  Slapmonitor instance configuration.
-  """
-
-  def __init__(self, buildout, name, options):
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
 
   def install(self):
-    config = dict(
-      pid_file_path=self.options['pid-file'],
-      consumption_log_path=self.options['consumption-log-path'],
-      database_path=self.options['database-path'],
-      slapreport_path = self.options['slapreport-path'],
-      logbox_ip = self.options['logbox-ip'],
-      logbox_port = self.options['logbox-port'],
-      logbox_user = self.options['logbox-user'],
-      logbox_passwd = self.options['logbox-passwd'],
-      shell_path=self.options['shell-path'],
-    )
-
-    # Runners
-    runner_path = self.createExecutable(
-      self.options['path'],
-      self.substituteTemplate(self.getTemplateFilename('slapreport_run.in'),
-                              config))
-
-    return [runner_path]
+    options = self.options
+    script = self.createWrapper(name=options['path'],
+                                command=options['slapreport-path'],
+                                parameters=[
+                                    options['pid-file'],
+                                    options['consumption-log-path'],
+                                    options['database-path'],
+                                    options['logbox-ip'],
+                                    options['logbox-port'],
+                                    options['logbox-user'],
+                                    options['logbox-passwd'],
+                                    ])
+    return [script]
 
-  def update(self):
-    pass

slapos/recipe/slapreport/template/slapreport_run.in

-#!%(shell_path)s
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-#
-exec %(slapreport_path)s %(pid_file_path)s %(consumption_log_path)s %(database_path)s %(logbox_ip)s %(logbox_port)s %(logbox_user)s %(logbox_passwd)s

slapos/recipe/slaprunner/__init__.py

@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
     self.instance_directory = options['instance-directory'].strip()
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options['cloud9-url'].strip()
+    self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
     # Set slaprunner access URL
     options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
 
@@ -59,6 +60,8 @@ class Recipe(GenericBaseRecipe):
           'supervisord.conf'),
         runner_workdir=self.workdir,
         etc_dir=self.options['etc_dir'],
+        run_dir=self.options['run_dir'],
+        log_dir=self.options['log_dir'],
         runner_host=self.ipv6,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
@@ -79,10 +82,12 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
 
     environment = dict(
-        PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'],
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
         GIT_SSH=self.options['ssh_client']
     )
-    launch_args = [self.options['slaprunner'].strip(), config_file]
+    launch_args = [self.options['slaprunner'].strip(), config_file,
+                   '--log_file', self.log_file]
     if self.optionIsTrue('debug', default=False):
       launch_args.append('--debug')
 
@@ -93,3 +98,71 @@ class Recipe(GenericBaseRecipe):
     path_list.append(wrapper)
 
     return path_list
+
+class Test(GenericBaseRecipe):
+  def _options(self, options):
+    self.ipv4 = options['ipv4'].strip()
+    self.ipv6 = options['ipv6'].strip()
+    self.proxy_port = options['proxy_port'].strip()
+    self.runner_port = options['runner_port'].strip()
+    self.workdir = options['working-directory'].strip()
+    self.software_directory = options['software-directory'].strip()
+    self.instance_directory = options['instance-directory'].strip()
+    self.partition_amount = options['partition-amount'].strip()
+    self.cloud9_url = options['cloud9-url'].strip()
+    # Set slaprunner access URL
+    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+
+  def install(self):
+    path_list = []
+
+    configuration = dict(
+        software_root=self.software_directory,
+        instance_root=self.instance_directory,
+        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
+        computer_id='slaprunner',
+        partition_amount=self.partition_amount,
+        slapgrid_sr=self.options['slapgrid_sr'],
+        slapgrid_cp=self.options['slapgrid_cp'],
+        slapproxy=self.options['slapproxy'],
+        supervisor=self.options['supervisor'],
+        supervisord_config=os.path.join(self.instance_directory, 'etc',
+          'supervisord.conf'),
+        runner_workdir=self.workdir,
+        etc_dir=self.options['etc_dir'],
+        run_dir=self.options['etc_dir'],
+        log_dir=self.workdir,
+        runner_host=self.ipv6,
+        runner_port=self.runner_port,
+        ipv4_address=self.ipv4,
+        ipv6_address=self.ipv6,
+        proxy_host=self.ipv4,
+        proxy_port=self.proxy_port,
+        proxy_database=os.path.join(self.workdir, 'proxy.db'),
+        git=self.options['git-binary'],
+        ssh_client=self.options['ssh_client'],
+        public_key=self.options['public_key'],
+        private_key=self.options['private_key'],
+        cloud9_url=self.cloud9_url
+    )
+
+    config_file = self.createFile(self.options['slapos.cfg'],
+        self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
+        configuration))
+    path_list.append(config_file)
+
+    environment = dict(
+        PATH=os.path.dirname(
+            self.options['git-binary']) + ':' + os.environ['PATH'],
+        GIT_SSH=self.options['ssh_client'],
+        CONFIG_FILE_PATH=config_file
+    )
+    launch_args = [self.options['slaprunnertest'].strip()]
+
+    wrapper = self.createPythonScript(self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        (launch_args, environment)
+    )
+    path_list.append(wrapper)
+
+    return path_list

slapos/recipe/slaprunner/execute.py

-import os
-
-
-def execute(args):
-  """Portable execution with process replacement"""
-  if args.get("path", None):
-    os.environ['PATH'] = args["path"]
-  os.execv(args["launch_args"][0], args["launch_args"])

slapos/recipe/slaprunner/template/slapos.cfg.in

@@ -20,6 +20,8 @@ runner_port = %(runner_port)s
 ipv4_address = %(ipv4_address)s
 ipv6_address = %(ipv6_address)s
 etc_dir = %(etc_dir)s
+run_dir = %(run_dir)s
+log_dir = %(log_dir)s
 
 [slapproxy]
 host = %(proxy_host)s

slapos/recipe/softwaretype.py

@@ -87,13 +87,17 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
+    self.logger.info('Deploying instance with software type %s' % \
+        software_type)
 
+    # Raise if request software_type does not exist ...
     if software_type not in self.options:
-      if 'default' in self.options:
+      # ... Except for backward compatibility. Then use "default".
+      if software_type in ['RootSoftwareInstance']:
         software_type = 'default'
       else:
-        raise zc.buildout.UserError("This software type isn't mapped. And "
-                                    "there's no default software type.")
+        raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
+            software_type)
 
     instance_file_path = self.options[software_type]
 

slapos/recipe/symbolic_link.py

@@ -27,6 +27,9 @@
 import os
 import logging
 
+import zc.buildout
+
+
 class Recipe:
   def __init__(self, buildout, name, options):
     self.buildout = buildout

slapos/recipe/testnode/__init__.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import BaseSlapRecipe
-import os
-import pkg_resources
-import zc.buildout
-import zc.recipe.egg
-import sys
-
-CONFIG = dict(
-  proxy_port='5000',
-  computer_id='COMPUTER',
-  partition_reference='test0',
-)
-
-class Recipe(BaseSlapRecipe):
-  def __init__(self, buildout, name, options):
-    self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
-    BaseSlapRecipe.__init__(self, buildout, name, options)
-
-  def installSlapOs(self):
-    CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
-    CONFIG['working_directory'] = self.createDataDirectory('testnode')
-    CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'],
-        'software')
-    CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'],
-        'instance')
-    CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'],
-        'proxy.db')
-    CONFIG['proxy_host'] = self.getLocalIPv4Address()
-    CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'],
-        CONFIG['proxy_port'])
-    self._createDirectory(CONFIG['software_root'])
-    self._createDirectory(CONFIG['instance_root'])
-    CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg',
-        self.substituteTemplate(pkg_resources.resource_filename(__name__,
-          'template/slapos.cfg.in'), CONFIG))
-    self.path_list.append(CONFIG['slapos_config'])
-
-  def setupRunningWrapper(self):
-    self.path_list.extend(zc.buildout.easy_install.scripts([(
-      'testnode',
-        __name__+'.testnode', 'run')], self.ws,
-          sys.executable, self.wrapper_directory, arguments=[
-            dict(
-              environment=self.getRuntimeEnvironment(),
-              computer_id=CONFIG['computer_id'],
-              instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')),
-              instance_root=CONFIG['instance_root'],
-              ipv4_address=self.getLocalIPv4Address(),
-              ipv6_address=self.getGlobalIPv6Address(),
-              master_url=CONFIG['master_url'],
-              profile_url=self.parameter_dict['profile_url'],
-              proxy_database=CONFIG['proxy_database'],
-              slapgrid_partition_binary=self.options['slapgrid_partition_binary'],
-              slapgrid_software_binary=self.options['slapgrid_software_binary'],
-              slapos_config=CONFIG['slapos_config'],
-              slapproxy_binary=self.options['slapproxy_binary'],
-              software_root=CONFIG['software_root'],
-              buildbot_binary=self.options['buildbot_binary'],
-              working_directory=CONFIG['working_directory'],
-              buildbot_host=self.parameter_dict['buildbot_host'],
-              slave_name=self.parameter_dict['slave_name'],
-              slave_password=self.parameter_dict['slave_password'],
-              bin_directory=self.bin_directory,
-              # botenvironemnt is splittable string of key=value to substitute
-              # environment of running bot
-              bot_environment=self.parameter_dict.get('bot_environment', ''),
-              partition_reference=CONFIG['partition_reference'],
-            )
-          ]))
-
-  def installLocalSvn(self):
-    svn_dict = dict(svn_binary = self.options['svn_binary'])
-    svn_dict.update(self.parameter_dict)
-    svn_path = os.path.join(self.bin_directory, 'svn')
-    self._writeExecutable(svn_path, """\
-#!/bin/sh
-%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
---non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
-    self.path_list.append(svn_path)
-    svnversion = os.path.join(self.bin_directory, 'svnversion')
-    if os.path.lexists(svnversion):
-      os.unlink(svnversion)
-    os.symlink(self.options['svnversion_binary'], svnversion)
-    self.path_list.append(svnversion)
-
-  def installLocalGit(self):
-    git = os.path.join(self.bin_directory, 'git')
-    if os.path.lexists(git):
-      os.unlink(git)
-    os.symlink(self.options['git_binary'], git)
-    self.path_list.append(git)
-
-  def installLocalZip(self):
-    zip = os.path.join(self.bin_directory, 'zip')
-    if os.path.lexists(zip):
-      os.unlink(zip)
-    os.symlink(self.options['zip_binary'], zip)
-    self.path_list.append(zip)
-
-  def installLocalPython(self):
-    """Installs local python fully featured with eggs"""
-    self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws,
-          sys.executable, self.bin_directory, scripts=None,
-          interpreter='python'))
-
-  def installLocalRunUnitTest(self):
-    link = os.path.join(self.bin_directory, 'runUnitTest')
-    destination = os.path.join(CONFIG['instance_root'],
-        CONFIG['partition_reference'], 'bin', 'runUnitTest')
-    if os.path.lexists(link):
-      if not os.readlink(link) != destination:
-        os.unlink(link)
-    if not os.path.lexists(link):
-      os.symlink(destination, link)
-    self.path_list.append(link)
-
-  def _installBuildbot(self):
-    self.setupRunningWrapper()
-    self.installLocalPython()
-    self.installLocalGit()
-    self.installLocalSvn()
-    self.installLocalRunUnitTest()
-    return self.path_list
-
-  def getRuntimeEnvironment(self):
-    env = {}
-    env['PATH'] = ':'.join([self.bin_directory] +
-        os.environ['PATH'].split(':'))
-    return env
-
-  def _installProfileTesting(self):
-    self.path_list.extend(zc.buildout.easy_install.scripts([(
-      'testnode',
-        __name__+'.profile_testnode', 'run')], self.ws,
-          sys.executable, self.wrapper_directory, arguments=[
-            dict(
-              environment=self.getRuntimeEnvironment(),
-              slapgrid_environment=eval(self.parameter_dict.get(
-                'slapgrid_environment', '{}')),
-              profile_path=self.parameter_dict.get('profile_path',
-                'slapos/software.cfg'),
-              repository=self.parameter_dict['repository'],
-              # Optional URL of test aggreagation system
-              test_suite_master_url=self.parameter_dict['test_suite_master_url'],
-              suite_name=self.parameter_dict['suite_name'],
-              branch=self.parameter_dict.get('branch', 'master'),
-              # internal parameters
-              software_root=CONFIG['software_root'],
-              computer_id=CONFIG['computer_id'],
-              git_binary=self.options['git_binary'],
-              master_url=CONFIG['master_url'],
-              proxy_database=CONFIG['proxy_database'],
-              slapgrid_software_binary=self.options['slapgrid_software_binary'],
-              slapos_config=CONFIG['slapos_config'],
-              slapproxy_binary=self.options['slapproxy_binary'],
-              working_directory=CONFIG['working_directory'],
-              bin_directory=self.bin_directory,
-              partition_reference=CONFIG['partition_reference'],
-            )
-          ]))
-    return self.path_list
-
-  def _install(self):
-    self.requirements, self.ws = self.egg.working_set()
-    self.path_list = []
-    self.installSlapOs()
-    self.installLocalZip()
-    flavour = self.parameter_dict.get('flavour', 'buildbot')
-    if flavour == 'buildbot':
-      return self._installBuildbot()
-    elif flavour == 'profile-testing':
-      return self._installProfileTesting()
-    raise NotImplementedError('Falvour %r is unknown'% flavour)

slapos/recipe/testnode/erp5testreporthandler.py

-import urlparse
-import urllib
-import httplib
-import mimetools
-from random import randint
-import tempfile
-import os
-import stat
-import zipfile
-import mimetypes
-import datetime
-
-TB_SEP = "============================================================="\
-    "========="
-
-def get_content_type(f):
-  return mimetypes.guess_type(f.name)[0] or 'application/octet-stream'
-
-class ConnectionHelper:
-  def __init__(self, url):
-    self.conn = urlparse.urlparse(url)
-    if self.conn.scheme == 'http':
-      connection_type = httplib.HTTPConnection
-      if self.conn.port is None:
-        self.port = 80
-    else:
-      connection_type = httplib.HTTPSConnection
-      if self.conn.port is None:
-        self.port = 443
-    self.connection_type = connection_type
-
-  def _connect(self):
-    self.connection = self.connection_type(self.conn.hostname + ':' +
-        str(self.conn.port or self.port))
-
-  def POST(self, path, parameter_dict, file_list=None):
-    self._connect()
-    parameter_dict.update(__ac_name=self.conn.username,
-                          __ac_password=self.conn.password)
-    header_dict = {'Content-type': "application/x-www-form-urlencoded"}
-    if file_list is None:
-      body = urllib.urlencode(parameter_dict)
-    else:
-      boundary = mimetools.choose_boundary()
-      header_dict['Content-type'] = 'multipart/form-data; boundary=%s' % (
-          boundary,)
-      body = ''
-      for k, v in parameter_dict.iteritems():
-        body += '--%s\r\n' % boundary
-        body += 'Content-Disposition: form-data; name="%s"\r\n' % k
-        body += '\r\n'
-        body += '%s\r\n' % v
-      for name, filename in file_list:
-        f = open(filename, 'r')
-        body += '--%s\r\n' % boundary
-        body += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'\
-                % (name, name)
-        body += 'Content-Type: %s\r\n' % get_content_type(f)
-        body += 'Content-Length: %d\r\n' % os.fstat(f.fileno())[stat.ST_SIZE]
-        body += '\r\n'
-        body += f.read()
-        f.close()
-        body += '\r\n'
-
-    self.connection.request("POST", self.conn.path + '/' + path,
-          body, header_dict)
-    self.response = self.connection.getresponse()
-
-class ERP5TestReportHandler:
-  def __init__(self, url, suite_name):
-    # random test id
-    self.test_id = "%s-%X" % (
-       ("%s" % datetime.date.today()).replace("-", ""),
-       randint(1, 10000000000000000),
-     )
-    self.connection_helper = ConnectionHelper(url)
-    self.suite_name = suite_name
-
-  def reportStart(self):
-    # report that test is running
-    print 'Starting test with id %s' % self.test_id
-    self.connection_helper.POST('TestResultModule_reportRunning', dict(
-      test_suite=self.suite_name,
-      test_report_id=self.test_id,
-      ))
-
-  def reportFinished(self, out_file, revision, success, duration, text):
-    # make file parsable by erp5_test_results
-    tempcmd = tempfile.mkstemp()[1]
-    tempcmd2 = tempfile.mkstemp()[1]
-    tempout = tempfile.mkstemp()[1]
-    templog = tempfile.mkstemp()[1]
-    log_lines = open(out_file, 'r').readlines()
-    tl = open(templog, 'w')
-    tl.write(TB_SEP + '\n')
-    if len(log_lines) > 900:
-      tl.write('...[truncated]... \n\n')
-    for log_line in log_lines[-900:]:
-      starts = log_line.startswith
-      if starts('Ran') or starts('FAILED') or starts('OK') or starts(TB_SEP):
-        continue
-      if starts('ERROR: ') or starts('FAIL: '):
-        tl.write('internal-test: ' + log_line)
-        continue
-      tl.write(log_line)
-
-    tl.write("----------------------------------------------------------------------\n")
-    tl.write('Ran 1 test in %.2fs\n' % duration)
-    if success:
-      tl.write('OK\n')
-    else:
-      tl.write('FAILED (failures=1)\n')
-    tl.write(TB_SEP + '\n')
-    tl.close()
-    open(tempcmd, 'w').write("""svn info dummy""")
-    open(tempcmd2, 'w').write(self.suite_name)
-    open(tempout, 'w').write("Revision: %s\n%s" % (revision, text))
-    # create nice zip archive
-    tempzip = tempfile.mkstemp()[1]
-    zip = zipfile.ZipFile(tempzip, 'w')
-    zip.write(tempcmd, 'dummy/001/cmdline')
-    zip.write(tempout, 'dummy/001/stdout')
-    zip.write(templog, 'dummy/001/stderr')
-    zip.write(tempout, '%s/002/stdout' % self.suite_name)
-    zip.write(templog, '%s/002/stderr' % self.suite_name)
-    zip.write(tempcmd2, '%s/002/cmdline' % self.suite_name)
-    zip.close()
-    os.unlink(templog)
-    os.unlink(tempcmd)
-    os.unlink(tempout)
-    os.unlink(tempcmd2)
-
-    # post it to ERP5
-    self.connection_helper.POST('TestResultModule_reportCompleted', dict(
-      test_report_id=self.test_id),
-      file_list=[('filepath', tempzip)]
-      )
-    os.unlink(tempzip)

slapos/recipe/testnode/profile_testnode.py

-import os
-import socket
-import signal
-import shutil
-import slapos.slap
-import subprocess
-import time
-import atexit
-from erp5testreporthandler import ERP5TestReportHandler
-
-process_group_pid_list = []
-def clean():
-  for pgpid in process_group_pid_list:
-    try:
-      os.killpg(pgpid, signal.SIGTERM)
-    except:
-      pass
-
-def sigterm_handler(signal, frame):
-  clean()
-
-def sigint_handler(signal, frame):
-  clean()
-  raise KeyboardInterrupt
-
-signal.signal(signal.SIGINT, sigint_handler)
-signal.signal(signal.SIGTERM, sigterm_handler)
-atexit.register(clean)
-
-def getCurrentBranchName(config, p):
-  r = subprocess.Popen([config['git_binary'], 'branch'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
-  for f in r.splitlines():
-    if f.startswith('*'):
-      return f.split()[1]
-  return ''
-
-def getRevision(config, p):
-  return subprocess.Popen([config['git_binary'], 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, cwd=p).communicate()[0].strip()
-
-def getCurrentFetchRemote(config, p):
-  r = subprocess.Popen([config['git_binary'], 'remote', '-v'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
-  remote = ''
-  for f in r.splitlines():
-    if f.startswith('origin') and f.endswith('(fetch)'):
-      if remote != '':
-        raise ValueError('Too many remotes: %s' % r)
-      remote = r.split()[1]
-  return remote
-
-def getMachineIdString():
-  """Returns machine identification string"""
-  kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  idstr = subprocess.Popen(["uname", "-m"], **kw).communicate()[0].strip()
-
-  # try to detect gcc version
-  try:
-    gcc_list = subprocess.Popen(["gcc", "-v"], **kw).communicate()[0].split(
-        '\n')
-    for gcc in gcc_list:
-      if gcc.startswith('gcc version'):
-        idstr += ' gcc:' + gcc.split()[2]
-        break
-  except IndexError:
-    pass
-
-  # try to detect libc version
-  try:
-    libdir = os.path.sep + 'lib'
-    for libso in os.listdir(libdir):
-      if libso.startswith('libc.') and os.path.islink(os.path.join(libdir,
-        libso)):
-        libc = os.readlink(os.path.join(libdir, libso))
-        if libc.endswith('.so'):
-          idstr += ' libc:' + libc.split('-')[1][:-3]
-        else:
-          idstr += ' ' + libc
-        break
-  except IndexError:
-    pass
-
-  return idstr
-
-def run(args):
-  config = args[0]
-  for k,v in config['environment'].iteritems():
-    os.environ[k] = v
-  proxy = None
-  slapgrid = None
-  last_revision_file = os.path.join(config['working_directory'],
-        'revision.txt')
-  if os.path.exists(last_revision_file):
-    os.unlink(last_revision_file)
-  # fetch repository from git
-  repository_clone = os.path.join(config['working_directory'], 'repository')
-  profile_path = os.path.join(repository_clone, config['profile_path'])
-  if os.path.exists(config['proxy_database']):
-    os.unlink(config['proxy_database'])
-  proxy = subprocess.Popen([config['slapproxy_binary'],
-    config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
-  process_group_pid_list.append(proxy.pid)
-  slap = slapos.slap.slap()
-  slap.initializeConnection(config['master_url'])
-  while True:
-    try:
-      slap.registerSupply().supply(profile_path,
-        computer_guid=config['computer_id'])
-    except socket.error:
-      time.sleep(1)
-      pass
-    else:
-      break
-  while True:
-    info_list = []
-    a = info_list.append
-    while True:
-      try:
-        if os.path.exists(repository_clone):
-          if getCurrentFetchRemote(config, repository_clone) != config['repository']:
-            shutil.rmtree(repository_clone)
-        if not os.path.exists(repository_clone):
-          subprocess.check_call([config['git_binary'], 'clone',
-            config['repository'], repository_clone])
-        # switch to branch
-        branch = getCurrentBranchName(config, repository_clone)
-        if branch != config['branch']:
-          subprocess.check_call([config['git_binary'], 'checkout', '--force',
-            '--track', '-b', config['branch'], 'origin/'+config['branch']],
-            cwd=repository_clone)
-        subprocess.check_call([config['git_binary'], 'reset', '--hard',
-          '@{upstream}'], cwd=repository_clone)
-      except Exception:
-        print 'Retrying git in 60s'
-        time.sleep(60)
-      else:
-        break
-    a('Tested repository: %s' % config['repository'])
-    a('Machine identification: %s' % getMachineIdString())
-    erp5_report = ERP5TestReportHandler(config['test_suite_master_url'],
-        '@'.join([config['suite_name'], branch]))
-    last_revision = ''
-    if os.path.exists(last_revision_file):
-      last_revision = open(last_revision_file).read().strip()
-    revision = getRevision(config, repository_clone)
-    open(last_revision_file, 'w').write(revision)
-    if revision != last_revision:
-      print 'Running for revision %r' % revision
-      while True:
-        try:
-          erp5_report.reportStart()
-        except Exception:
-          print 'Retrying in 5s'
-          time.sleep(5)
-        else:
-          break
-      if os.path.exists(config['software_root']):
-        shutil.rmtree(config['software_root'])
-      os.mkdir(config['software_root'])
-      out_file = os.path.join(config['working_directory'], 'slapgrid.out')
-      if os.path.exists(out_file):
-        os.unlink(out_file)
-      out = open(out_file, 'w')
-      begin = time.time()
-      slapgrid_environment = os.environ.copy()
-      for k, v in config['slapgrid_environment'].iteritems():
-        slapgrid_environment[k] = v
-      a('Slapgrid environment: %r'% config['slapgrid_environment'])
-      slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
-        config['slapos_config']], close_fds=True, preexec_fn=os.setsid,
-        stdout=out, stderr=subprocess.STDOUT, env=slapgrid_environment)
-      process_group_pid_list.append(slapgrid.pid)
-      slapgrid.communicate()
-      out.close()
-      while True:
-        try:
-          erp5_report.reportFinished(out_file,revision,
-              slapgrid.returncode == 0, time.time() - begin,
-              '\n'.join(info_list))
-        except Exception:
-          print 'Retrying in 5s'
-          time.sleep(5)
-        else:
-          break
-    print 'Sleeping for 600s'
-    time.sleep(600)

slapos/recipe/testnode/template/slapos.cfg.in

-[slapos]
-software_root = %(software_root)s
-instance_root = %(instance_root)s
-master_url = %(master_url)s
-computer_id = %(computer_id)s
-
-[slapproxy]
-host = %(proxy_host)s
-port = %(proxy_port)s
-database_uri = %(proxy_database)s

slapos/recipe/testnode/testnode.py

-from xml_marshaller import xml_marshaller
-import os
-import signal
-import slapos.slap
-import socket
-import subprocess
-import sys
-import time
-
-process_group_pid_list = []
-process_pid_file_list = []
-process_command_list = []
-def sigterm_handler(signal, frame):
-  for pgpid in process_group_pid_list:
-    try:
-      os.killpg(pgpid, signal.SIGTERM)
-    except:
-      pass
-  for pid_file in process_pid_file_list:
-    try:
-      os.kill(int(open(pid_file).read().strip()), signal.SIGTERM)
-    except:
-      pass
-  for p in process_command_list:
-    try:
-      subprocess.call(p)
-    except:
-      pass
-  sys.exit(1)
-
-signal.signal(signal.SIGTERM, sigterm_handler)
-
-def run(args):
-  config = args[0]
-  for k,v in config['environment'].iteritems():
-    os.environ[k] = v
-  proxy = None
-  slapgrid = None
-  supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
-        'supervisord.pid')
-  if os.path.exists(config['proxy_database']):
-    os.unlink(config['proxy_database'])
-  try:
-    proxy = subprocess.Popen([config['slapproxy_binary'],
-      config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
-    process_group_pid_list.append(proxy.pid)
-    slap = slapos.slap.slap()
-    slap.initializeConnection(config['master_url'])
-    while True:
-      try:
-        slap.registerSupply().supply(config['profile_url'],
-          computer_guid=config['computer_id'])
-      except socket.error:
-        time.sleep(1)
-        pass
-      else:
-        break
-    while True:
-      slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
-        config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
-      process_group_pid_list.append(slapgrid.pid)
-      slapgrid.wait()
-      if slapgrid.returncode == 0:
-        print 'Software installed properly'
-        break
-      print 'Problem with software installation, trying again'
-      time.sleep(600)
-    computer = slap.registerComputer(config['computer_id'])
-    partition_reference = config['partition_reference']
-    partition_path = os.path.join(config['instance_root'], partition_reference)
-    if not os.path.exists(partition_path):
-      os.mkdir(partition_path)
-      os.chmod(partition_path, 0750)
-    computer.updateConfiguration(xml_marshaller.dumps({
- 'address': config['ipv4_address'],
- 'instance_root': config['instance_root'],
- 'netmask': '255.255.255.255',
- 'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
-                                       'netmask': '255.255.255.255'},
-                                      {'addr': config['ipv6_address'],
-                                       'netmask': 'ffff:ffff:ffff::'},
-                      ],
-                     'path': partition_path,
-                     'reference': partition_reference,
-                     'tap': {'name': partition_reference},
-                     }
-                    ],
- 'reference': config['computer_id'],
- 'software_root': config['software_root']}))
-    slap.registerOpenOrder().request(config['profile_url'],
-        partition_reference='testing partition',
-        partition_parameter_kw=config['instance_dict'])
-    slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], '-vc',
-      config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
-    slapgrid.wait()
-    if slapgrid.returncode != 0:
-      raise ValueError('Slapgrid instance failed')
-    runUnitTest = os.path.join(partition_path, 'bin', 'runUnitTest')
-    if not os.path.exists(runUnitTest):
-      raise ValueError('No %r provided' % runUnitTest)
-  except:
-    try:
-      if os.path.exists(supervisord_pid_file):
-        os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
-    except:
-      pass
-    raise
-  finally:
-    # Nice way to kill *everything* generated by run process -- process
-    # groups working only in POSIX compilant systems
-    # Exceptions are swallowed during cleanup phase
-    if proxy is not None:
-      os.killpg(proxy.pid, signal.SIGTERM)
-    if os.path.exists(config['proxy_database']):
-      os.unlink(config['proxy_database'])
-    if slapgrid is not None and slapgrid.returncode is None:
-      os.killpg(slapgrid.pid, signal.SIGTERM)
-  try:
-    bot_env = os.environ.copy()
-    bot_env['PATH'] = ':'.join([config['bin_directory']] +
-        bot_env['PATH'].split(':'))
-    for l in config['bot_environment'].split():
-      k, v = l.split('=')
-      bot_env[k] = v
-    if subprocess.call([config['buildbot_binary'], 'create-slave', '-f',
-      config['working_directory'], config['buildbot_host'],
-      config['slave_name'], config['slave_password']]) != 0:
-      raise ValueError('Buildbot call failed')
-    process_command_list.append([config['buildbot_binary'], 'stop',
-      config['working_directory']])
-    if os.path.exists(os.path.join(config['working_directory'],
-      'buildbot.tac.new')):
-      tac = os.path.join(config['working_directory'], 'buildbot.tac')
-      if os.path.exists(tac):
-        os.unlink(tac)
-      os.rename(os.path.join(config['working_directory'],
-      'buildbot.tac.new'), tac)
-    if subprocess.call([config['buildbot_binary'], 'start',
-      config['working_directory']], env=bot_env) != 0:
-      raise ValueError('Issue during starting buildbot')
-    while True:
-      time.sleep(3600)
-  finally:
-    try:
-      subprocess.call([config['buildbot_binary'], 'stop',
-      config['working_directory']])
-    except:
-      pass
-    try:
-      if os.path.exists(supervisord_pid_file):
-        os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
-    except:
-      pass

slapos/recipe/tidstorage/__init__.py

@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
   def install(self):
-    path_list = []
-    a = path_list.append
-    configuration_file = self.createFile(self.options['configuration-path'], self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'), self.options))
-    a(configuration_file)
-    tidstorage_wrapper = self.createPythonScript(self.options['tidstorage-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstoraged-binary'], '--nofork', '--config', configuration_file])
-    a(tidstorage_wrapper)
-    repozo_wrapper = self.createPythonScript(self.options['repozo-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstorage-repozo-binary'], '--config', configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
-    a(repozo_wrapper)
+    configuration_file = self.createFile(
+      self.options['configuration-path'],
+      self.substituteTemplate(
+        self.getTemplateFilename('tidstorage.py.in'), self.options))
 
-    return path_list
+    tidstorage_wrapper = self.createPythonScript(
+      self.options['tidstorage-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstoraged-binary'],  '--nofork', '--config',
+        configuration_file])
+
+    repozo_wrapper = self.createPythonScript(
+      self.options['repozo-wrapper'],
+      'slapos.recipe.librecipe.execute.execute',
+      [self.options['tidstorage-repozo-binary'], '--config',
+        configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
+
+    return [configuration_file, tidstorage_wrapper, repozo_wrapper]

slapos/recipe/tidstorage/template/tidstorage.py.in

 known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s
-base_url = '%(base-url)s'
 address = '%(ip)s'
 port = %(port)s
 

slapos/recipe/xwiki/template/my.cnf.in

@@ -21,7 +21,7 @@ long_query_time = 5
 max_allowed_packet = 128M
 query_cache_size = 32M
 
-plugin-load = ha_mroonga.so;ha_sphinx.so
+plugin-load = ha_mroonga.so
 
 # The following are important to configure and depend a lot on to the size of
 # your database and the available resources.

software/agent/instance-agent.cfg

+[buildout]
+parts =
+  instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[instance]
+recipe = ${instance-recipe:egg}:${instance-recipe:module}
+agent_binary = ${buildout:directory}/bin/agent
+pidfile = $${directory:srv}/agent.pid
+log = $${directory:agentlog}/agent.log
+wrapper = $${directory:run}/agent
+config =  $${directory:etc}/agent.cfg
+master-url = $${slap-parameter:master-url}
+key = $${slap-parameter:userkey}
+cert = $${slap-parameter:usercertificate}
+configuration = $${slap-parameter:configuration}
+default_max_install_duration =  $${slap-parameter:default_max_install_duration}
+default_max_uninstall_duration = $${slap-parameter:default_max_uninstall_duration}
+default_max_request_duration = $${slap-parameter:default_max_request_duration}
+default_max_destroy_duration = $${slap-parameter:default_max_destroy_duration}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+run = $${:etc}/run
+agentlog = $${buildout:directory}/var/log/agent
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin

software/agent/instance.cfg

 [buildout]
 parts =
-  instance
+  switch_softwaretype
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
 
-[instance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-agent_binary = ${buildout:directory}/bin/agent
-report_start = ${buildout:directory}/bin/report_start
-report_stop = ${buildout:directory}/bin/report_stop
-dcrond_binary = ${dcron:location}/sbin/crond
-python_binary = ${python2.7:location}/bin/python
-pidfile = $${rootdirectory:run}/agent.pid
-log = $${rootdirectory:agentlog}/agent.log
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-run = $${buildout:directory}/etc/run
-agentlog = $${buildout:directory}/var/log/agent
-srv = $${buildout:directory}/srv
-bin = $${buildout:directory}/bin
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-agent:output}

software/agent/software.cfg

 [buildout]
 extends =
-  ../../component/dcron/buildout.cfg
-  ../../component/python-2.7/buildout.cfg
   ../../component/lxml-python/buildout.cfg
+  ../../component/git/buildout.cfg
   ../../stack/slapos.cfg
 
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
 parts =
   template
-  eggs
+  template-agent
+  slapos.cookbook-repository
+  check-recipe
+  slapos.toolbox-repository
+  check-recipe-toolbox
   instance-recipe-egg
-  dcron
-  python2.7
   script
 
+# Local development
+[slapos.cookbook-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+branch = agent
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = ${git:location}/bin/git clone --branch ${:branch} --quiet http://git.erp5.org/repos/slapos.git ${:location}
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[slapos.toolbox-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+branch = agent
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = ${git:location}/bin/git clone --branch ${:branch} --quiet http://git.erp5.org/repos/slapos.toolbox.git ${:location}
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+
+[check-recipe-toolbox]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
 [instance-recipe]
+# XXX-Cedric: it can use newest slapos.cfg to not have duplication
 egg = slapos.cookbook
 module = agent
 
 [instance-recipe-egg]
+# XXX-Cedric: it can use newest slapos.cfg to not have duplication
 recipe = zc.recipe.egg
-python = python2.7
-eggs = ${instance-recipe:egg}
+eggs =
+  ${lxml-python:egg}
+  ${instance-recipe:egg}
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = c7cb98594f394d05baedabe424643f6f
+md5sum = bcd3b3cb8a305c83bb048d5ac1c583fe
 mode = 0644
 
-[eggs]
-python = python2.7
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-  slapos.cookbook
-  slapos.toolbox
-  erp5.util
-
-[lxml-python]
-python = python2.7
+[template-agent]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-agent.cfg
+output = ${buildout:directory}/template-agent.cfg
+md5sum = fd9670d8473be402b10990398b553b00
+mode = 0644
 
 [script]
 recipe = z3c.recipe.scripts
-python = python2.7
 eggs =
   zc.buildout
-  ${lxml-python:egg}
   slapos.core
-  slapos.cookbook
   slapos.toolbox[agent]
+  erp5.util
 
 [networkcache]
 # signature certificates of the following uploaders.
 #   Romain Courteaud
+#   Cedric de Saint Martin
+#   Rafael Monnerat
 signature-certificate-list =
   -----BEGIN CERTIFICATE-----
   MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
@@ -68,3 +100,29 @@ signature-certificate-list =
   q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
   QUUGLQ==
   -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAOcKrOH/2Da6MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjk3MCAXDTEyMDYyNjAzMDU1MVoYDzIxMTIwNjAyMDMwNTUxWjAT
+  MREwDwYDVQQDDAhDT01QLTI5NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  xzbOGlcoin2q+Mtp52r26Njliz2aoxIXbnOBUyDc/OGtk9nWA5uBtTc2zwR17um6
+  KV0bGyvuBA78XcvU+AIV/5s0ohBAX7yjRKmEhAYcFvov3EyWSdjOrqqo4qFSzOrK
+  sVQBlxIDpjQBH4F3lf6dBv6/M+tCT3iSv3aOZbsG0E8CAwEAAaNQME4wHQYDVR0O
+  BBYEFLqtrfTu+BIVt+TFiRUkIoiWIYrxMB8GA1UdIwQYMBaAFLqtrfTu+BIVt+TF
+  iRUkIoiWIYrxMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc8N5P5gW
+  Jrdk9gF/3Cpp6THDiy93+WcuAm7zFwXPFNttJtFKMNObP2YRZvsQkvjezfrZoRBF
+  j8LgKB3tZCbBj+HDj+AeD+q9V+cqMFLKc6LezvQYUuum6bZdfUNnPv1K1ULYSPjq
+  /jsRBbabCWSXqxR6gYEM6ooauj3udBMXhHE=
+  -----END CERTIFICATE-----

software/apache-frontend/README.apache_frontend.txt

@@ -9,15 +9,22 @@ It means that a single main instance of Apache will be used to act as frontend
 for many slaves.
 
 
-How to use
-==========
+How to deploy a frontend server
+===============================
 
-First, you will need to request a "master" instance of Apache Frontend with
-"domain" parameter, like::
+This is to deploy an entire frontend server with a public IPv4.
+If you want to use an already deployed frontend to make your service available
+via ipv4, switch to the "Example" parts.
+
+First, you will need to request a "master" instance of Apache Frontend with:
+  * A "domain" parameter where the frontend will be available
+  * A "public-ipv4" parameter to state which public IPv4 will be used
+
+like::
   <?xml version='1.0' encoding='utf-8'?>
   <instance>
    <parameter id="domain">moulefrite.org</parameter>
-   <parameter id="port">443</parameter>
+   <parameter id="public-ipv4">xxx.xxx.xxx.xxx</parameter>
   </instance>
 
 Then, it is possible to request many slave instances
@@ -69,11 +76,11 @@ url of backend to use.
 "url" is a mandatory parameter.
 Example: http://mybackend.com/myresource
 
-cache
+enable_cache
 ~~~~~
 Specify if slave instance should use a varnish / stunnel to connect to backend.
 Possible values: "true", "false".
-"cache" is an optional parameter. Defaults to "false". 
+"enable_cache" is an optional parameter. Defaults to "false".
 Example: true
 
 type
@@ -81,30 +88,83 @@ type
 Specify if slave instance will redirect to a zope backend. If specified, Apache
 RewriteRule will use Zope's Virtual Host Daemon.
 Possible values: "zope", "default".
-"type" is an optional parameter. Defaults to "default". 
+"type" is an optional parameter. Defaults to "default".
 Example: zope
 
 custom_domain
 ~~~~~~~~~~~~~
 Domain name to use as frontend. The frontend will be accessible from this domain.
 "custom_domain" is an optional parameter. Defaults to
-[instancereference].[masterdomain]. 
+[instancereference].[masterdomain].
 Example: www.mycustomdomain.com
 
+path
+~~~~
+Only used if type is "zope".
+
+Will append the specified path to the "VirtualHostRoot" of the zope's
+VirtualHostMonster.
+
+"path" is an optional parameter, ignored if not specified.
+Example of value: "/erp5/web_site_module/hosting/"
+
+Examples
+========
+
+Here are some example of how to make your SlapOS service available through
+an already deployed frontend.
+
+Simple Example
+--------------
+
+Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
+redirected and accessible from the proxy::
+  instance = request(
+    software_release=apache_frontend,
+    software_type="RootSoftwareInstance",
+    partition_reference='my frontend',
+    shared=True,
+    partition_parameter_kw={
+        "url":"https://[1:2:3:4:5:6:7:8]:1234",
+    }
+  )
+
+
+Zope Example
+------------
+
+Request slave frontend instance using a Zope backend so that
+https://[1:2:3:4:5:6:7:8]:1234 will be redirected and accessible from the
+proxy::
+  instance = request(
+    software_release=apache_frontend,
+    software_type="RootSoftwareInstance",
+    partition_reference='my frontend',
+    shared=True,
+    partition_parameter_kw={
+        "url":"https://[1:2:3:4:5:6:7:8]:1234",
+        "type":"zope",
+    }
+  )
+
 
 Advanced example
-================
+----------------
 
 Request slave frontend instance using a Zope backend, with Varnish activated,
-listening to a custom domain::
+listening to a custom domain and redirecting to /erp5/ so that
+https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
+the proxy::
   instance = request(
     software_release=apache_frontend,
-    partition_reference='frontend2',
+    software_type="RootSoftwareInstance",
+    partition_reference='my frontend',
     shared=True,
     partition_parameter_kw={
-        "url":"https://[1:2:3:4]:1234/someresource",
-        "cache":"true",
+        "url":"https://[1:2:3:4:5:6:7:8]:1234",
+        "enable_cache":"true",
         "type":"zope",
+        "path":"/erp5",
         "custom_domain":"mycustomdomain.com",
     }
   )
@@ -115,5 +175,9 @@ Notes
 It is not possible with slapos to listen to port <= 1024, because process are
 not run as root. It is a good idea then to go on the node where the instance is
 and set some iptables rules like (if using default ports)::
-  iptables -t nat -A PREROUTING -p tcp -d {public ip} --dport 443 -j DNAT --to-destination {listening ip}:4443
-  iptables -t nat -A PREROUTING -p tcp -d {public_ip} --dport 80 -j DNAT --to-destination {listening ip}:8080
+
+  iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 443 -j DNAT --to-destination {listening_ipv4}:4443
+  iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 80 -j DNAT --to-destination {listening_ipv4}:8080
+
+Where {public ip} is the public IP of your server, or at least the LAN IP to where your NAT will forward to.
+{listening ip} is the private ipv4 (like 10.0.34.123) that the instance is using and sending as connection parameter.

software/apache-frontend/common.cfg

+[buildout]
+extends =
+  ../../component/binutils/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../component/stunnel/buildout.cfg
+  ../../component/varnish/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../stack/slapos.cfg
+
+parts =
+  template
+  binutils
+  apache-2.2
+  apache-antiloris-apache-2.2
+
+  stunnel
+  varnish-2.1
+
+  dcron
+  logrotate
+  rdiff-backup
+
+# Buildoutish
+  eggs
+  instance-recipe-egg
+
+[instance-recipe]
+# Note: In case if specific instantiation recipe is used this is the place to
+# put its name
+egg = slapos.cookbook
+module = apache.frontend
+
+[instance-recipe-egg]
+recipe = zc.recipe.egg
+eggs = ${instance-recipe:egg}
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+
+[template]
+# Default template for apache instance.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = fea902a2b9dbf8c80ff201bcf73f9396
+output = ${buildout:directory}/template.cfg
+mode = 0644
\ No newline at end of file

software/apache-frontend/development.cfg

+# Development profile of apache-frontend.
+# Exactly the same as software.cfg, but fetch the slapos.cookbook
+# from git repository instead of fetching stable version,
+# allowing to play with bleeding edge environment.
+
+# You'll need to run buildout twice for this profile.
+
+[buildout]
+extends =
+# Extend in this order, otherwise "parts" will be taken from git profile
+  ../../component/git/buildout.cfg
+  common.cfg
+
+parts +=
+  slapos.cookbook-repository
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = frontend
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&

software/apache-frontend/instance.cfg

 [buildout]
 parts =
+  directory
   instance
+  configtest
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 
+# Create all needed directories
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+service = $${:etc}/service
+
+
+# Deploy Apache (old way, with monolithic recipe)
 [instance]
 recipe = ${instance-recipe:egg}:${instance-recipe:module}
+httpd_home = ${apache-2.2:location}
 httpd_binary = ${apache-2.2:location}/bin/httpd
 logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
 openssl_binary = ${openssl:location}/bin/openssl
@@ -14,5 +28,12 @@ dcrond_binary = ${dcron:location}/sbin/crond
 varnishd_binary = ${varnish-2.1:location}/sbin/varnishd
 stunnel_binary = ${stunnel:location}/bin/stunnel
 rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-gcc_binary = ${gcc-java-minimal:location}/bin/gcc
+gcc_binary = gcc
 binutils_directory = ${binutils:location}/bin/
+
+
+# Create wrapper for "apachectl conftest" in bin
+[configtest]
+recipe = slapos.cookbook:wrapper
+command-line = $${instance:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
+output = $${directory:bin}/apache-configtest

software/apache-frontend/software.cfg

 [buildout]
-versions = versions
-
-extends =
-  ../../component/binutils/buildout.cfg
-  ../../component/gcc/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/apache/buildout.cfg
-  ../../component/stunnel/buildout.cfg
-  ../../component/varnish/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/logrotate/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../stack/slapos.cfg
-
-parts =
-  template
-  binutils
-  gcc-java-minimal
-  apache-2.2
-  apache-antiloris-apache-2.2
-
-  stunnel
-  varnish-2.1
-
-  dcron
-  logrotate
-  rdiff-backup
-
-# Buildoutish
-  eggs
-  instance-recipe-egg
-
-[instance-recipe]
-# Note: In case if specific instantiation recipe is used this is the place to
-# put its name
-egg = slapos.cookbook
-module = apache.frontend
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-
-[template]
-# Default template for apache instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 74c0f41246d167c020854a212e919ce4
-output = ${buildout:directory}/template.cfg
-mode = 0644
+extends = common.cfg
 
 [versions]
 Jinja2 = 2.6
 Werkzeug = 0.8.3
 buildout-versions = 1.7
-hexagonit.recipe.cmmi = 1.5.0
-meld3 = 0.6.8
+hexagonit.recipe.cmmi = 1.6
+meld3 = 0.6.10
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.52
-slapos.recipe.template = 2.3
+slapos.cookbook = 0.71.1
+slapos.recipe.build = 0.11.5
+slapos.recipe.template = 2.4.2
 
 # Required by:
-# slapos.core==0.24
-Flask = 0.8
+# slapos.core==0.33.1
+Flask = 0.9
 
 # Required by:
-# slapos.cookbook==0.52
-PyXML = 0.8.4
+# hexagonit.recipe.cmmi==1.6
+hexagonit.recipe.download = 1.6
 
 # Required by:
-# slapos.cookbook==0.52
+# slapos.cookbook==0.71.1
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.52
-# slapos.core==0.24
+# slapos.cookbook==0.71.1
+# slapos.core==0.33.1
 # xml-marshaller==0.9.7
-lxml = 2.3.4
+lxml = 3.0.2
 
 # Required by:
-# slapos.cookbook==0.52
-netaddr = 0.7.6
+# slapos.cookbook==0.71.1
+netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.24
+# slapos.core==0.33.1
 netifaces = 0.8
 
 # Required by:
-# slapos.cookbook==0.52
-# slapos.core==0.24
-# zc.buildout==1.6.0-dev-SlapOS-004
+# slapos.cookbook==0.71.1
+pytz = 2012j
+
+# Required by:
+# slapos.cookbook==0.71.1
+# slapos.core==0.33.1
+# zc.buildout==1.6.0-dev-SlapOS-010
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.52
-slapos.core = 0.24
+# slapos.cookbook==0.71.1
+slapos.core = 0.33.1
 
 # Required by:
-# slapos.core==0.24
-supervisor = 3.0a12
+# slapos.core==0.33.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.52
+# slapos.cookbook==0.71.1
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.52
-zc.recipe.egg = 1.3.2
+# slapos.core==0.33.1
+zope.interface = 4.0.3
 
-# Required by:
-# slapos.core==0.24
-zope.interface = 4.0.0
 
 [networkcache]
-# Cedric de Saint Martin signature certificate
+# signature certificates of the following uploaders.
+#   Cedric de Saint Martin
+#   Romain Courteaud
 signature-certificate-list =
   -----BEGIN CERTIFICATE-----
   MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
@@ -133,3 +84,16 @@ signature-certificate-list =
   Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
   If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
   -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
+  CBMQRGVmYXVsdCBQcm92aW5jZTEPMA0GA1UEChMGTmV4ZWRpMB4XDTExMDkxNTA5
+  MDAwMloXDTEyMDkxNTA5MDAwMlowOTELMAkGA1UEBhMCRlIxGTAXBgNVBAgTEERl
+  ZmF1bHQgUHJvdmluY2UxDzANBgNVBAoTBk5leGVkaTCBnzANBgkqhkiG9w0BAQEF
+  AAOBjQAwgYkCgYEApYZv6OstoqNzxG1KI6iE5U4Ts2Xx9lgLeUGAMyfJLyMmRLhw
+  boKOyJ9Xke4dncoBAyNPokUR6iWOcnPHtMvNOsBFZ2f7VA28em3+E1JRYdeNUEtX
+  Z0s3HjcouaNAnPfjFTXHYj4um1wOw2cURSPuU5dpzKBbV+/QCb5DLheynisCAwEA
+  ATANBgkqhkiG9w0BAQsFAAOBgQBCZLbTVdrw3RZlVVMFezSHrhBYKAukTwZrNmJX
+  mHqi2tN8tNo6FX+wmxUUAf3e8R2Ymbdbn2bfbPpcKQ2fG7PuKGvhwMG3BlF9paEC
+  q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
+  QUUGLQ==
+  -----END CERTIFICATE-----

software/dotclear/software.cfg

@@ -16,7 +16,7 @@ extends =
 [application]
 recipe = slapos.recipe.build:download-unpacked
 url = http://download.dotclear.net/latest-2.0.tar.gz
-md5sum = 98263b4734f93b7c54d3ab436bf344e3
+md5sum = 4ca12cbd12228ceee0b7c0f697e1ff09
 
 [application-template]
 recipe = slapos.recipe.download

software/drupal/settings.php.in

@@ -200,12 +200,23 @@
  *   );
  * @endcode
  */
+
+/*
++ * The apachephp recipe provided with both host+port in the same string.
++ * Split them again, php way.
++ * And it could be an ipv6 as well, so beware of colons.
++ */
+$mysql_host_port = '%(mysql_host)s';
+$mysql_port = substr(strrchr($mysql_host_port, ":"), 1);
+$mysql_host = substr($mysql_host_port, 0, strlen($mysql_host_port) - strlen($mysql_port) - 1);
+
 $databases['default']['default'] = array(
   'driver' => 'mysql',
   'database' => '%(mysql_database)s',
   'username' => '%(mysql_user)s',
   'password' => '%(mysql_password)s',
-  'host' => '%(mysql_host)s',
+  'host' => $mysql_host,
+  'port' => $mysql_port,
   'prefix' => '',
 );
 

software/drupal/software.cfg

@@ -3,15 +3,30 @@
 extends =
   ../../stack/lamp/buildout.cfg
 
+parts +=
+  patch-hide-dbsetup
+
+
 [application]
 recipe = slapos.recipe.build:download-unpacked
-url = http://ftp.drupal.org/files/projects/drupal-7.15.tar.gz
-md5sum = f42c9baccd74e1d035d61ff537ae21b4
+url = http://ftp.drupal.org/files/projects/drupal-7.16.tar.gz
+md5sum = 352497b2df94b5308e31cb8da020b631
+
+[download-patch-hide-dbsetup]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 44a1689173bc8c51cc384218296d826e
+download-only = true
+filename = system.theme.css.patch
+
+[patch-hide-dbsetup]
+recipe = cp.recipe.cmd
+install_cmd = patch ${application:location}/modules/system/system.theme.css <${download-patch-hide-dbsetup:location}/${download-patch-hide-dbsetup:filename}
 
 [application-template]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/settings.php.in
-md5sum = add9e4e96094a9d3fb7cf0f4d26ae544
+md5sum = b0ef5548c3ac1de8ca56882e51e6eb6d
 download-only = True
 filename = template.in
 mode = 0644

software/drupal/system.theme.css.patch

+--- a/system.theme.css  Mon Oct 29 11:20:30 2012 +0100
++++ b/system.theme.css  Mon Oct 29 11:20:53 2012 +0100
+@@ -1,3 +1,14 @@
++
++#install-settings-form .form-item-driver,
++#install-settings-form #edit-mysql * {
++  display: none;
++}
++
++#install-settings-form #edit-mysql:before {
++  content: "The database has been set up by SlapOS.";
++  font-weight: bold;
++  font-size: 120%;
++}
+ 
+ /**
+  * @file
+

software/erp5-util-testing/instance.cfg

+[buildout]
+parts =
+  test-runner
+  sh-environment
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+services = $${:etc}/run
+srv = $${buildout:directory}/srv
+source-code = $${:srv}/eggs-source-code
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+# Local development
+[erp5-util]
+<= download-source
+repository = ${erp5-util-repository:location}
+
+[test-runner]
+recipe = slapos.cookbook:egg_test
+run-test-suite = $${create-directory:bin}/runTestSuite
+run-test-suite-binary = ${buildout:bin-directory}/runTestSuite
+# The list of executables should be defined here and a combination
+# of tests should dynamically generated.
+#python-list = $${}
+test-list =
+  $${erp5-util:location}
+prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
+environment = environment
+
+[environment]
+CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
+LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib
+PYTHONPATH = ${python-setuptools:location}
+LD_LIBRARY_PATH = ${libxslt:location}/lib
+
+[sh-environment]
+# Section exposes testing default environment as sh file. It is thus easy
+# to directly develop and test the egg inside of this instance.
+recipe = collective.recipe.template
+input = inline:
+  export PATH="$${test-runner:prepend-path}:$PATH"
+  export CPPFLAGS="$${environment:CPPFLAGS}"
+  export LDFLAGS="$${environment:LDFLAGS}"
+  export PYTHONPATH="$${environment:PYTHONPATH}"
+  export PS1="[slapos-testing env Active] $PS1"
+output = $${create-directory:bin}/environment.sh
+mode = 755

software/erp5-util-testing/software.cfg

+[buildout]
+
+extends =
+  ../../component/git/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/libxml2/buildout.cfg
+  ../../component/libxslt/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/python-setuptools/buildout.cfg
+  ../../component/zlib/buildout.cfg
+  ../../stack/slapos.cfg
+
+parts =
+  slapos.cookbook-repository
+  erp5-util-repository
+  eggs
+  template
+
+[eggs]
+recipe = zc.recipe.egg
+eggs = 
+  ${lxml-python:egg}
+  erp5.util
+  slapos.cookbook
+  collective.recipe.template
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/slapos.git
+branch = master
+
+[erp5-util-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+repository = http://git.erp5.org/repos/erp5.git
+branch = master
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = 0cdfaa1df03885bd6004755d285e69b5
+output =  ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+Jinja2 = 2.6
+Werkzeug = 0.8.3
+buildout-versions = 1.7
+erp5.util = 0.4.7
+hexagonit.recipe.cmmi = 1.6
+lxml = 2.3.6
+meld3 = 0.6.8
+plone.recipe.command = 1.1
+slapos.cookbook = 0.65
+slapos.libnetworkcache = 0.13.2
+slapos.recipe.build = 0.11.2
+slapos.recipe.download = 1.0.dev-r4053
+slapos.recipe.template = 2.4.2
+
+# Required by:
+# slapos.core==0.28.5
+Flask = 0.9
+
+# Required by:
+# slapos.cookbook==0.65
+PyXML = 0.8.4
+
+# Required by:
+# hexagonit.recipe.cmmi==1.6
+hexagonit.recipe.download = 1.5.1
+
+# Required by:
+# slapos.cookbook==0.65
+inotifyx = 0.2.0
+
+# Required by:
+# slapos.cookbook==0.65
+netaddr = 0.7.10
+
+# Required by:
+# slapos.core==0.28.5
+netifaces = 0.8
+
+# Required by:
+# erp5.util==0.4.7
+# slapos.cookbook==0.65
+# slapos.core==0.28.5
+# zc.buildout==1.6.0-dev-SlapOS-006
+# zc.recipe.egg==1.3.2
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.cookbook==0.65
+slapos.core = 0.28.5
+
+# Required by:
+# slapos.core==0.28.5
+supervisor = 3.0a12
+
+# Required by:
+# slapos.cookbook==0.65
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.65
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.28.5
+zope.interface = 4.0.1

software/joomla/software.cfg

 [buildout]
-versions = versions
-
-
-  
 extends =
   ../../stack/lamp/buildout.cfg
-  ../../stack/shacache-client.cfg
 
 [application]
 recipe = slapos.recipe.build:download-unpacked
@@ -16,7 +11,7 @@ strip-top-level-dir = false
 [application-template]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/database.xml-in
-md5sum = 6ae063318a251b70ae7d037e3ae375d6 
+md5sum = 6ae063318a251b70ae7d037e3ae375d6
 download-only = True
 filename = template.in
 mode = 0644
@@ -24,15 +19,3 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
  
 [application-configuration]
 location = installation/models/forms/database.xml
-
-[downloadcache-workaround]
-# workaround irritating problem of hexagonit.recipe.cmmi which automatically
-# creates download cache, which in turn switches builout to "semi-offline" mode
-recipe = plone.recipe.command
-# in hexagonit.recipe.cmmi if there is no ${buildout:download-cache} set it resolves
-# to ${buildout:directory}/downloads but this variable is available late, that's
-# why it is hardcoded only for required case
-download-cache = ${buildout:directory}/downloads
-command = [ -d ${:download-cache} ] && rm -fr ${:download-cache}/* || exit 0
-update-command = ${:command}
-stop-on-error = True

software/kvm/README.txt

@@ -7,6 +7,8 @@ Introduction
 This software release is used to deploy KVM instances, NBD instances and
 Frontend instances of KVM.
 
+For extensive parameters definition, please look at parameter-input-schema.json.
+
 Examples
 --------
 

software/kvm/instance-kvm-input-schema.json

+{
+  "name": "Input Parameters",
+  "properties": {
+    "ram-size": {
+      "title": "RAM size",
+      "description": "RAM size, in MB.",
+      "type": "integer",
+      "default": 1024,
+      "minimum": 128,
+      "divisibleBy": 128,
+      "maximum": 16384
+    },
+    "disk-size": {
+      "title": "Disk size",
+      "description": "Disk size, in GB.",
+      "type": "integer",
+      "default": 10,
+      "minimum": 1,
+      "maximum": 80
+    },
+    "disk-type": {
+      "title": "Disk type",
+      "description": "Type of QEMU disk drive.",
+      "type": "string",
+      "default": "virtio",
+      "enum": ["ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio"]
+    },
+
+    "cpu-count": {
+      "title": "CPU count",
+      "description": "Number of CPU cores.",
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 8
+    },
+
+
+    "nbd-host": {
+      "title": "NBD hostname",
+      "description": "hostname (or IP) of the NBD server containing the boot image.",
+      "type": "string",
+      "format": ["host-name", "ip-address", "ipv6"],
+      "default": "debian.nbd.vifib.net"
+    },
+    "nbd-port": {
+      "title": "NBD port",
+      "description": "Port of the NBD server containing the boot image.",
+      "type": "integer",
+      "default": 1024,
+      "minimum": 1,
+      "maximum": 65535
+    },
+
+    "nbd2-host": {
+      "title": "Second NBD hostname",
+      "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
+      "type": "string",
+      "format": ["host-name", "ip-address", "ipv6"],
+    },
+    "nbd2-port": {
+      "title": "Second NBD port",
+      "description": "Port of the second NBD server containing the boot image.",
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 65535
+    },
+
+
+    "frontend-instance-guid": {
+      "title": "Frontend Instance ID",
+      "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+      "type": "string",
+      "default": "SOFTINST-11031"
+    },
+    "frontend-software-type": {
+      "title": "Frontend Software Type",
+      "description": "Type of the frontend instance, like \"frontend\".",
+      "type": "string",
+      "default": "frontend"
+    },
+    "frontend-software-url": {
+      "title": "Frontend Software URL",
+      "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+      "type": "string",
+      "format": "uri",
+      "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg"
+    }
+  }
+}

software/kvm/instance-kvm-output-schema.json

+{
+  "name": "Output Parameters",
+  "properties": {
+    "backend-url": {
+      "title": "Backend URL",
+      "description": "URL used to connect directly to backend without frontend. Requires IPv6.",
+      "type": "uri",
+      "required": true
+    },
+
+    "url": {
+      "title": "URL",
+      "description": "URL used to connect to the service.",
+      "type": "uri",
+      "required": false
+    },
+
+    "password": {
+      "title": "Password",
+      "description": "Password used to authenticate in the service webpage.",
+      "type": "uri",
+      "required": true
+    }
+  }
+}

software/kvm/instance-kvm.cfg → software/kvm/instance-kvm.cfg.in

@@ -17,46 +17,49 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[rootdirectory]
+[directory]
 recipe = slapos.cookbook:mkdirectory
 etc = $${buildout:directory}/etc
 bin = $${buildout:directory}/bin
 srv = $${buildout:directory}/srv
 var = $${buildout:directory}/var
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-services = $${rootdirectory:etc}/run
-promises = $${rootdirectory:etc}/promise
-novnc-conf = $${rootdirectory:etc}/novnc
-run = $${rootdirectory:var}/run
-ca-dir = $${rootdirectory:srv}/ssl
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+novnc-conf = $${:etc}/novnc
+run = $${:var}/run
+ca-dir = $${:srv}/ssl
 
 [create-mac]
 recipe = slapos.cookbook:generate.mac
-storage-path = $${rootdirectory:srv}/mac
+storage-path = $${directory:srv}/mac
 
 [gen-passwd]
 recipe = slapos.cookbook:generate.password
-storage-path = $${rootdirectory:srv}/passwd
+storage-path = $${directory:srv}/passwd
 bytes = 4
 
 [kvm-instance]
+# XXX-Cedric: change "KVM" recipe to simple "create wrappers". No need for this
+# Specific code
 recipe = slapos.cookbook:kvm
 vnc-ip = $${slap-network-information:local-ipv4}
 vnc-port = 5901
-nbd-ip = $${slap-parameter:nbd_ip}
-nbd-port = $${slap-parameter:nbd_port}
+nbd-host = $${slap-parameter:nbd-host}
+nbd-port = $${slap-parameter:nbd-port}
+nbd2-host = $${slap-parameter:nbd2-host}
+nbd2-port = $${slap-parameter:nbd2-port}
 tap = $${slap-network-information:network-interface}
-disk-path = $${rootdirectory:srv}/virtual.qcow2
-disk-size = 10
-socket-path = $${rootdirectory:var}/qmp_socket
-pid-path = $${basedirectory:run}/pid_file
-smp-count = 1
-ram-size = 1024
+disk-path = $${directory:srv}/virtual.qcow2
+disk-size = $${slap-parameter:disk-size}
+disk-type = $${slap-parameter:disk-type}
+socket-path = $${directory:var}/qmp_socket
+pid-path = $${directory:run}/pid_file
+smp-count = $${slap-parameter:cpu-count}
+ram-size = $${slap-parameter:ram-size}
 mac-address = $${create-mac:mac-address}
-runner-path = $${basedirectory:services}/kvm
-controller-path = $${basedirectory:services}/kvm_controller
+runner-path = $${directory:services}/kvm
+controller-path = $${directory:scripts}/kvm_controller
 shell-path = ${dash:location}/bin/dash
 qemu-path = ${kvm:location}/bin/qemu-system-x86_64
 qemu-img-path = ${kvm:location}/bin/qemu-img
@@ -64,7 +67,7 @@ passwd = $${gen-passwd:passwd}
 
 [kvm-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${basedirectory:promises}/vnc_promise
+path = $${directory:promises}/vnc_promise
 hostname = $${kvm-instance:vnc-ip}
 port = $${kvm-instance:vnc-port}
 
@@ -83,15 +86,15 @@ ssl-cert-path = $${ca-novnc:cert-file}
 
 [websockify-sighandler]
 recipe = slapos.cookbook:signalwrapper
-wrapper-path = $${basedirectory:services}/websockify
+wrapper-path = $${directory:services}/websockify
 wrapped-path = $${novnc-instance:path}
 
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${basedirectory:ca-dir}
+ca-dir = $${directory:ca-dir}
 requests-directory = $${cadirectory:requests}
-wrapper = $${basedirectory:services}/certificate_authority
+wrapper = $${directory:services}/certificate_authority
 ca-private = $${cadirectory:private}
 ca-certs = $${cadirectory:certs}
 ca-newcerts = $${cadirectory:newcerts}
@@ -99,30 +102,30 @@ ca-crl = $${cadirectory:crl}
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${basedirectory:ca-dir}/requests/
-private = $${basedirectory:ca-dir}/private/
-certs = $${basedirectory:ca-dir}/certs/
-newcerts = $${basedirectory:ca-dir}/newcerts/
-crl = $${basedirectory:ca-dir}/crl/
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
 
 [ca-novnc]
 <= certificate-authority
 recipe = slapos.cookbook:certificate_authority.request
-key-file = $${basedirectory:novnc-conf}/novnc.key
-cert-file = $${basedirectory:novnc-conf}/novnc.crt
-executable = $${rootdirectory:bin}/novnc
-wrapper = $${rootdirectory:bin}/websockify
+key-file = $${directory:novnc-conf}/novnc.key
+cert-file = $${directory:novnc-conf}/novnc.crt
+executable = $${directory:bin}/novnc
+wrapper = $${directory:bin}/websockify
 
 [novnc-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${basedirectory:promises}/novnc_promise
+path = $${directory:promises}/novnc_promise
 hostname = $${novnc-instance:ip}
 port = $${novnc-instance:port}
 
 
 [kvm-monitor]
 recipe = slapos.cookbook:generic.slapmonitor
-db-path = $${rootdirectory:srv}/slapmonitor_database
+db-path = $${directory:srv}/slapmonitor_database
 
 
 [request-slave-frontend]
@@ -145,7 +148,7 @@ sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 [publish-kvm-backend-connection-information]
 recipe = slapos.cookbook:publish
-backend_url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
+backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
 password = $${kvm-instance:passwd}
 
 [publish-kvm-frontend-connection-information]
@@ -154,7 +157,7 @@ url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-sl
 
 [frontend-promise]
 recipe = slapos.cookbook:check_url_available
-path = $${basedirectory:promises}/frontend_promise
+path = $${directory:promises}/frontend_promise
 url = $${publish-kvm-frontend-connection-information:url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
@@ -164,5 +167,14 @@ curl_path = ${curl:location}/bin/curl
 frontend-instance-guid = SOFTINST-11031
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
-nbd_port = 1024
-nbd_ip = debian.nbd.vifib.net
+
+nbd-port = 1024
+nbd-host = debian.nbd.vifib.net
+nbd2-port = 1024
+nbd2-host =
+
+ram-size = 1024
+disk-size = 10
+disk-type = virtio
+
+cpu-count = 1
\ No newline at end of file

software/kvm/instance-kvmplus.cfg → software/kvm/instance-kvmplus.cfg.in

@@ -3,10 +3,12 @@
 # Instanciate kvm+
 #
 #############################
+
+# Deprecated. Just specify amount of RAM / disk you want in instance parameter.
+
 [buildout]
 extends = ${template-kvm:output}
 
-[kvm-instance]
-disk-size = 20
-smp-count = 2
+[slap-parameter]
 ram-size = 2048
+disk-size = 20
\ No newline at end of file

software/kvm/software.cfg

 [buildout]
-extensions =
-  buildout-versions
-  
+
 extends =
-  ../../component/gzip/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
   ../../component/dcron/buildout.cfg
-  ../../component/logrotate/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/gnutls/buildout.cfg
+  ../../component/gzip/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
-  ../../component/dash/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/curl/buildout.cfg
   ../../stack/nodejs.cfg
+  ../../stack/slapos.cfg
 
 develop =
   ${:parts-directory}/websockify
 
 parts =
   template
-  dash
-  kvm
   eggs
   check-local-eggs
-  nodejs
-  http-proxy
-  proxy-by-url
-  npm-modules
-  dcron
-  logrotate
-
-versions = versions
 
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
 #             Problem is : no native crypto support in web browsers. So we have to disable ssl
-#             In qemu builtin vnc server, and make it available only for localhost 
+#             In qemu builtin vnc server, and make it available only for localhost
 #             so that only novnc can listen to it.
 
 #XXX-Cedric: Check status of https://github.com/kanaka/noVNC/issues/13 to see
-#            When qemu has builtin support for websockets in vnc server to get rid of 
+#            When qemu has builtin support for websockets in vnc server to get rid of
 #            Websockify (socket <-> websocket proxy server) when it is ready.
 #            May solve previous XXX depending on the implementation.
 
@@ -129,35 +118,35 @@ command =
 
 [template-kvm]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-kvm.cfg
-md5sum = 6aaa9b6ef059a2ed2f022834151086ab
+url = ${:_profile_base_location_}/instance-kvm.cfg.in
+md5sum = 5607cb6b6af58694d55b87411880d368
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
 [template-kvmplus]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-kvmplus.cfg
-md5sum = 301fbe4eaaab5648e1a933a4c853f5b9
+url = ${:_profile_base_location_}/instance-kvmplus.cfg.in
+md5sum = c8e92237eeda93caca1132b5202c3a02
 output = ${buildout:directory}/template-kvmplus.cfg
 mode = 0644
 
 [template-nbd]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-nbd.cfg
+url = ${:_profile_base_location_}/instance-nbd.cfg.in
 md5sum = c030e7be231aba25ee0f51703e60ce67
 output = ${buildout:directory}/template-nbd.cfg
 mode = 0644
 
 [template-frontend]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-frontend.cfg
+url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = 73359b52013b1b65f75005e8698ed180
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
 
 [template]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/instance.cfg.in
 md5sum = 68788763d23f70f24b9e575871c903a8
 output = ${buildout:directory}/template.cfg
 mode = 0644
@@ -210,50 +199,43 @@ signature-certificate-list =
 
 
 [versions]
+lxml = 2.3.6
 Jinja2 = 2.6
-Pygments = 1.5
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
-docutils = 0.9.1
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-ipython = 0.13
-lxml = 2.3.5
-meld3 = 0.6.8
+meld3 = 0.6.9
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.60
-slapos.recipe.build = 0.10.2
-slapos.recipe.template = 2.4.1
-slapos.toolbox = 0.28
+slapos.cookbook = 0.70
+slapos.recipe.build = 0.11.5
+slapos.recipe.template = 2.4.2
+slapos.toolbox = 0.31.1
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.27
-# slapos.toolbox==0.28
+# slapos.core==0.32.2
+# slapos.toolbox==0.31.1
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.28
+# slapos.toolbox==0.31.1
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.60
+# slapos.cookbook==0.70
 PyXML = 0.8.4
 
 # Required by:
-# netaddr==0.7.7
-Sphinx = 1.1.3
-
-# Required by:
-# slapos.toolbox==0.28
+# slapos.toolbox==0.31.1
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.28
+# slapos.toolbox==0.31.1
 feedparser = 5.1.2
 
 # Required by:
@@ -261,15 +243,15 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.60
+# slapos.cookbook==0.70
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.60
-netaddr = 0.7.7
+# slapos.cookbook==0.70
+netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.27
+# slapos.core==0.32.2
 netifaces = 0.8
 
 # Required by:
@@ -277,37 +259,41 @@ netifaces = 0.8
 numpy = 1.6.2
 
 # Required by:
-# slapos.toolbox==0.28
-paramiko = 1.7.7.2
+# slapos.toolbox==0.31.1
+paramiko = 1.8.0
+
+# Required by:
+# slapos.toolbox==0.31.1
+psutil = 0.6.1
 
 # Required by:
-# slapos.toolbox==0.28
-psutil = 0.6.0
+# slapos.cookbook==0.70
+pytz = 2012f
 
 # Required by:
-# slapos.cookbook==0.60
-# slapos.core==0.27
-# slapos.toolbox==0.28
+# slapos.cookbook==0.70
+# slapos.core==0.32.2
+# slapos.toolbox==0.31.1
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.60
-# slapos.toolbox==0.28
-slapos.core = 0.27
+# slapos.cookbook==0.70
+# slapos.toolbox==0.31.1
+slapos.core = 0.32.2
 
 # Required by:
-# slapos.core==0.27
-supervisor = 3.0a12
+# slapos.core==0.32.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.60
-# slapos.toolbox==0.28
+# slapos.cookbook==0.70
+# slapos.toolbox==0.31.1
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.60
+# slapos.cookbook==0.70
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.27
+# slapos.core==0.32.2
 zope.interface = 4.0.1

software/maarch/instance-custom.cfg.in

+[buildout]
+
+[maarch-configuration]
+recipe = slapos.recipe.maarch:default
+htdocs = $${apache-php:htdocs}
+db_host = $${postgres-urlparse:host}
+db_port = $${postgres-urlparse:port}
+db_dbname = $${postgres-urlparse:path}
+db_username = $${postgres-urlparse:username}
+db_password = $${postgres-urlparse:password}
+language = en
+php_ini_dir = $${directory:php-ini-dir}
+root_docservers = $${buildout:directory}/srv/docservers
+dependency = $${apache-php:recipe}
+

software/maarch/software.cfg

+[buildout]
+
+extends =
+  ../../stack/lapp/buildout.cfg
+
+# += since we need rdiff-backup and friends
+parts +=
+  slapos-cookbook
+  apache-php-postgres
+  pear-modules
+  eggs
+  instance
+  instance-apache-php
+
+#----------------
+#--
+#-- Main application part
+#-- XXX provide a better URL
+
+[application]
+url = http://downloads.sourceforge.net/project/maarch/Maarch%20Entreprise/MaarchEntreprise-1.3.zip?r=http%3A%2F%2Fwww.maarch.org%2Ftelecharger&ts=1347961624&use_mirror=ignum
+md5sum = 5c2c859dee9d0dde3ba959474fd5fc86
+
+
+#----------------
+#--
+#-- We don't need this static configuration file.
+#-- An empty file is provided because it is required by the lapp stack.
+#--
+
+[application-template]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/config.php.in
+md5sum = d41d8cd98f00b204e9800998ecf8427e
+download-only = True
+filename = template.in
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[application-configuration]
+location = config.php
+
+
+#----------------
+#--
+#-- Define parts that will be executed later, in the instance.
+#--
+
+[custom-application-deployment]
+path = ${custom-application-deployment-template:output}
+part-list = maarch-configuration
+
+[custom-application-deployment-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-custom.cfg.in
+output = ${buildout:directory}/instance-custom.cfg
+md5sum = 28f6ad42b73806901ac2cd4ec563d73b
+mode = 0644
+
+
+#----------------
+#--
+#-- Install PHP channels and modules (only if they are not already installed).
+#--
+
+[pear-modules]
+recipe = cp.recipe.cmd
+pear = ${apache-php-postgres:location}/bin/pear
+install_cmd =
+    ${:pear} channel-info pear.maarch.org >/dev/null || ${:pear} channel-discover pear.maarch.org
+    ${:pear} info maarch/CLITools-0.3.1 >/dev/null || ${:pear} install maarch/CLITools-0.3.1
+    ${:pear} info MIME_Type >/dev/null || ${:pear} install MIME_Type
+
+#----------------
+
+

software/mioga/instance.cfg

+[buildout]
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+parts = instance
+
+[instance]
+recipe = ${instance-recipe:egg}:${instance-recipe:module}
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+httpd_binary = ${apache:location}/bin/httpd
+mysql_binary = ${mariadb:location}/bin/mysql
+mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
+mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
+mysqld_binary = ${mariadb:location}/libexec/mysqld

software/mioga/software.cfg

+[buildout]
+extends = 
+  ../../component/postgresql/buildout.cfg
+  ../../component/apache-perl/buildout.cfg
+  ../../stack/slapos.cfg
+
+[networkcache]
+# Cedric de Saint Martin signature certificate
+signature-certificate-list =
+  -----BEGIN CERTIFICATE-----
+  MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+  BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
+  MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
+  wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
+  D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
+  P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
+  BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
+  cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
+  b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
+  Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
+  If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
+  -----END CERTIFICATE-----

software/phpmyadmin/software.cfg

@@ -22,7 +22,7 @@ mode = 0644
 
 [application]
 recipe = slapos.recipe.build:download-unpacked
-url = http://downloads.sourceforge.net/project/phpmyadmin/phpMyAdmin/3.3.10/phpMyAdmin-3.3.10-all-languages.tar.bz2?r=http%3A%2F%2Fwww.phpmyadmin.net%2Fhome_page%2Fdownloads.php&ts=1300959842&use_mirror=sunet
+url = http://downloads.sourceforge.net/project/phpmyadmin/phpMyAdmin/3.3.10/phpMyAdmin-3.3.10-all-languages.tar.bz2
 md5sum = cb7a632fb4f10a180ead15f7f90087f1
 
 [application-template]

software/phpnuke/software.cfg

@@ -16,7 +16,7 @@ extends =
 [application]
 recipe = slapos.recipe.build:download-unpacked
 url = http://phpnuke.org/modules/Release/files/phpnuke-release-8.2.4.tar.gz
-md5sum = 0f31895b141f50a5f2e6f083b0222fe2
+md5sum = d448586c651c811a137280b435c4e05c
 extract-directory = html
 
 [application-template]

software/postgres/instance.cfg.in

+[buildout]
+parts =
+  symlinks
+  publish
+  postgres-instance
+
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+[instance-parameters]
+# Fetches parameters defined in SlapOS Master for this instance
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+
+[directories]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+etc = $${buildout:directory}/etc
+services = $${directories:etc}/run/
+promises = $${directories:etc}/promise/
+var = $${buildout:directory}/var
+
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${directories:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+[postgres-instance]
+# create cluster, configuration files and a database
+recipe = slapos.cookbook:postgres
+
+# Options
+ipv6_host = $${instance-parameters:ipv6}
+ipv4_host = $${slap-network-information:local-ipv4}
+user = user
+port = 5432
+dbname = db
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${directories:var}/data
+services = $${directories:services}
+bin = $${directories:bin}
+
+
+[publish]
+recipe = slapos.cookbook:publishurl
+url = $${postgres-instance:url}
+
+
+[slap-connection]
+# part to migrate to new - separated words
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+

software/postgres/software.cfg

+[buildout]
+
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/postgresql/buildout.cfg
+
+parts =
+  eggs
+  slapos-cookbook
+  instance-template
+  postgresql
+
+[instance-template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/template.cfg
+#md5sum = 
+mode = 0644
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+    cns.recipe.symlink
+

software/prestashop/software.cfg

@@ -16,7 +16,7 @@ extends =
 [application]
 recipe = slapos.recipe.build:download-unpacked
 url = http://www.prestashop.com/download/prestashop_1.4.5.1.zip
-md5sum = 3d19305d728f7e8a19f81f9f326b9383
+md5sum = 6c0dcd301abe992a19dea1737d0adbe9
   
 [instance-recipe]
 egg = slapos.cookbook

software/punbb/software.cfg

@@ -16,7 +16,7 @@ extends =
 [application]
 recipe = slapos.recipe.build:download-unpacked
 url = http://punbb.informer.com/download/punbb-1.3.6.tar.gz
-md5sum = 9454ef78101028fd5acf2731f77545c2
+md5sum = 7bb3f570cb6a61f8c3e3e25e750fd29d
 
 [application-template]
 recipe = slapos.recipe.download

software/slapos-testing/instance.cfg

 [buildout]
 parts = 
   slapos-test-runner
+  sh-environment
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -47,5 +48,18 @@ environment = environment
 
 [environment]
 CPPFLAGS = -I${python2.7:location}/include/python2.7 -I${libxml2:location}/include -I${libxslt:location}/include
-LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib
+LDFLAGS = -L${python2.7:location}/lib -L${libxml2:location}/lib -L${libxslt:location}/lib -L${libxslt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${python2.7:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${libxslt:location}/lib -Wl,-rpath=${zlib:location}/lib
 PYTHONPATH = ${python-setuptools:location}
+
+[sh-environment]
+# Section exposes testing default environment as sh file. It is thus easy
+# to directly develop and test the egg inside of this instance.
+recipe = collective.recipe.template
+input = inline:
+  export PATH="$${slapos-test-runner:prepend-path}:$PATH"
+  export CPPFLAGS="$${environment:CPPFLAGS}"
+  export LDFLAGS="$${environment:LDFLAGS}"
+  export PYTHONPATH="$${environment:PYTHONPATH}"
+  export PS1="[slapos-testing env Active] $PS1"
+output = $${create-directory:bin}/environment.sh
+mode = 755

software/slapos-testing/software.cfg

@@ -24,24 +24,18 @@ eggs =
   ${lxml-python:egg}
   erp5.util
   slapos.cookbook
+  collective.recipe.template
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
 scripts =
   runTestSuite
 
-[erp5.util-repository]
-recipe = slapos.recipe.build:gitclone
-git-executable = ${git:location}/bin/git
-forbid-download-cache = true
-repository = http://git.erp5.org/repos/erp5.git
-branch = master
-
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
 forbid-download-cache = true
 repository = http://git.erp5.org/repos/slapos.git
-branch = slapos-testing
+branch = master
 
 [slapos.core-repository]
 recipe = slapos.recipe.build:gitclone
@@ -67,7 +61,7 @@ branch = master
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = d7dbd5da07c1170d0b80199d99f932eb
+md5sum = 47f5a8cf4d134a35b7f36a7442e5798a
 output =  ${buildout:directory}/template.cfg
 mode = 640
 
@@ -77,7 +71,7 @@ Werkzeug = 0.8.3
 buildout-versions = 1.7
 erp5.util = 0.4.7
 hexagonit.recipe.cmmi = 1.6
-lxml = 3.0alpha2
+lxml = 2.3.6
 meld3 = 0.6.8
 plone.recipe.command = 1.1
 slapos.cookbook = 0.65

software/slaprunner/common.cfg

+[buildout]
+extends =
+  ../../component/cloud9/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dropbear/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../stack/flask.cfg
+  ../../stack/shacache-client.cfg
+  ../../stack/slapos.cfg
+
+parts =
+  template
+  slapos-cookbook
+  eggs
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+mode = 0644
+md5sum = 5307e4200f044ae57b504ad68444491c
+
+[template-runner]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-runner.cfg
+output = ${buildout:directory}/template-runner.cfg
+md5sum = 91d6550c43b7a43a999724af4650ae40
+mode = 0644
+
+[eggs]
+recipe = z3c.recipe.scripts
+eggs =
+  ${lxml-python:egg}
+  slapos.libnetworkcache
+  slapos.toolbox[flask_auth]
+  slapos.core
+  cns.recipe.symlink

software/slaprunner/development.cfg

+# Development profile of slaprunner.
+# Exactly the same as software.cfg, but fetch the slapos.cookbook and
+# slapos.toolbox from git repository instead of fetching stable version,
+# allowing to play with bleeding edge environment.
+
+# You'll need to run buildout twice for this profile.
+
+[buildout]
+extends = common.cfg
+
+parts +=
+  slapos.cookbook-repository
+  slapos.core-repository
+  slapos.toolbox-repository
+  check-recipe
+
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.core-repository
+  ${:parts-directory}/slapos.toolbox-repository
+
+
+[slapos.toolbox-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.toolbox.git
+branch = slaprunner
+git-executable = ${git:location}/bin/git
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = slaprunner
+git-executable = ${git:location}/bin/git
+
+[slapos.core-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.core.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link

software/slaprunner/instance-runner.cfg

@@ -2,6 +2,7 @@
 parts =
   cloud9
   slaprunner
+  test-runner
   sshkeys-dropbear
   dropbear-server-add-authorized-key
   sshkeys-authority
@@ -10,46 +11,48 @@ parts =
   slaprunner-frontend-promise
   cloud9-promise
   dropbear-promise
+  symlinks
+
 
-  
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
 # Create all needed directories
-[rootdirectory]
+[directory]
 recipe = slapos.cookbook:mkdirectory
 etc = $${buildout:directory}/etc/
 var = $${buildout:directory}/var/
 srv = $${buildout:directory}/srv/
 bin = $${buildout:directory}/bin/
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-sshkeys = $${rootdirectory:srv}/sshkeys
-services = $${rootdirectory:etc}/run/
-ssh = $${rootdirectory:etc}/ssh/
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+sshkeys = $${:srv}/sshkeys
+services = $${:etc}/service/
+scripts = $${:etc}/run/
+ssh = $${:etc}/ssh/
+log = $${:var}/log/
+run = $${:var}/run/
+backup = $${:srv}/backup/
+promises = $${:etc}/promise/
+test = $${:etc}/test/
 
 [runnerdirectory]
 recipe = slapos.cookbook:mkdirectory
-home = $${rootdirectory:srv}/runner/
+home = $${directory:srv}/runner/
+test = $${directory:srv}/test/
 project = $${:home}/project
 software-root = $${:home}/software
 instance-root = $${:home}/instance
+project-test = $${:test}/project
+software-test = $${:test}/software
+instance-test = $${:test}/instance
+sessions = $${buildout:directory}/.sessions
 
 #Create password recovery code for slaprunner
 [recovery-code]
 recipe = slapos.cookbook:generate.password
-storage-path = $${rootdirectory:etc}/.rcode
+storage-path = $${directory:etc}/.rcode
 bytes = 4
 
 # Deploy cloud9 and slaprunner
@@ -57,7 +60,7 @@ bytes = 4
 recipe = slapos.cookbook:cloud9
 ip = $${slap-network-information:global-ipv6}
 port = 30000
-wrapper = $${basedirectory:services}/cloud9
+wrapper = $${directory:services}/cloud9
 working-directory = $${runnerdirectory:home}
 git-binary = ${git:location}/bin/git
 node-binary = ${nodejs:location}/bin/node
@@ -71,13 +74,14 @@ slapgrid_cp = ${buildout:directory}/bin/slapgrid-cp
 slapproxy = ${buildout:directory}/bin/slapproxy
 supervisor = ${buildout:directory}/bin/slapgrid-supervisorctl
 git-binary = ${git:location}/bin/git
-slapos.cfg = $${rootdirectory:etc}/slapos.cfg
+slapos.cfg = $${directory:etc}/slapos.cfg
 working-directory = $${runnerdirectory:home}
 project-directory = $${runnerdirectory:project}
 software-directory = $${runnerdirectory:software-root}
 instance-directory = $${runnerdirectory:instance-root}
-etc_dir = $${rootdirectory:etc}
-log_dir =  $${basedirectory:log}
+etc_dir = $${directory:etc}
+log_dir =  $${directory:log}
+run_dir = $${directory:run}
 ssh_client = $${sshkeys-dropbear:wrapper}
 public_key = $${sshkeys-dropbear:public-key}
 private_key = $${sshkeys-dropbear:private-key}
@@ -85,11 +89,26 @@ ipv4 = $${slap-network-information:local-ipv4}
 ipv6 = $${slap-network-information:global-ipv6}
 proxy_port = 50000
 runner_port = 50000
-partition-amount = 7
+partition-amount = $${slap-parameter:instance-amount}
 cloud9-url = $${cloud9:access-url}
-wrapper = $${basedirectory:services}/slaprunner
-
-# Deploy dropbear (minimalist ssh server)
+wrapper = $${directory:services}/slaprunner
+debug = $${slap-parameter:debug}
+
+[test-runner]
+<= slaprunner
+recipe = slapos.cookbook:slaprunner.test
+slaprunnertest = ${buildout:directory}/bin/slaprunnertest
+slapos.cfg = $${directory:etc}/slapos-test.cfg
+working-directory = $${runnerdirectory:test}
+project-directory = $${runnerdirectory:project-test}
+software-directory = $${runnerdirectory:software-test}
+instance-directory = $${runnerdirectory:instance-test}
+proxy_port = 8602
+wrapper = $${directory:bin}/runTestSuite
+etc_dir = $${directory:test}
+
+
+# Deploy dropbear (minimalist SSH server)
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
 requests = $${directory:sshkeys}/requests/
@@ -99,7 +118,7 @@ keys = $${directory:sshkeys}/keys/
 recipe = slapos.cookbook:sshkeys_authority
 request-directory = $${sshkeys-directory:requests}
 keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
+wrapper = $${directory:services}/sshkeys_authority
 keygen-binary = ${dropbear:location}/bin/dropbearkey
 
 [dropbear-server]
@@ -107,7 +126,7 @@ recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
 port = 2222
 home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
+wrapper = $${directory:bin}/raw_sshd
 shell = /bin/bash
 rsa-keyfile = $${directory:ssh}/server_key.rsa
 dropbear-binary = ${dropbear:location}/sbin/dropbear
@@ -120,7 +139,7 @@ type = rsa
 executable = $${dropbear-server:wrapper}
 public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
+wrapper = $${directory:services}/sshd
 
 [dropbear-server-add-authorized-key]
 <= dropbear-server
@@ -154,31 +173,38 @@ password_recovery_code = $${recovery-code:passwd}
 # Deploy promises scripts
 [slaprunner-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${basedirectory:promises}/slaprunner
+path = $${directory:promises}/slaprunner
 hostname = $${slaprunner:ipv6}
 port = $${slaprunner:runner_port}
 
 [slaprunner-frontend-promise]
 recipe = slapos.cookbook:check_url_available
-path = $${basedirectory:promises}/slaprunner_frontend
+path = $${directory:promises}/slaprunner_frontend
 url = $${request-frontend:connection-site_url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
 [cloud9-promise]
 recipe = slapos.cookbook:check_url_available
-path = $${basedirectory:promises}/cloud9
+path = $${directory:promises}/cloud9
 url = http://$${cloud9:ip}:$${cloud9:port}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
 [dropbear-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${basedirectory:promises}/dropbear
+path = $${directory:promises}/dropbear
 hostname = $${dropbear-server:host}
 port = $${dropbear-server:port}
 
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${directory:bin}
+symlink_base = ${buildout:directory}/bin
 
 [slap-parameter]
 # Default value if no ssh key is specified
 authorized-key =
+# Default value of instances number in slaprunner
+instance-amount = 10
+debug = false

software/slaprunner/software.cfg

-[buildout]
-
-extends =
-  ../../stack/slapos.cfg
-  ../../stack/flask.cfg
-  ../../stack/shacache-client.cfg
-  ../../component/dropbear/buildout.cfg
-  ../../component/git/buildout.cfg
-  ../../component/cloud9/buildout.cfg
-  ../../component/dash/buildout.cfg
-
-parts =
-  template
-  slapos-cookbook
-  eggs
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
-mode = 0644
-md5sum = 5307e4200f044ae57b504ad68444491c
+# Production profile of slaprunner.
+# Exactly the same as common.cfg, but:
+# 1/ Use a defined set of Python eggs instead of using the latest available
+#    ones from Pypi, to ensure stability;
+# 2/ Define list of trusted certificates for the cache.
 
-[template-runner]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-runner.cfg
-output = ${buildout:directory}/template-runner.cfg
-md5sum = b57ebff565595fa92cd7d5adc6a22c7e
-mode = 0644
-
-[eggs]
-recipe = z3c.recipe.scripts
-eggs =
-  slapos.libnetworkcache
-  slapos.toolbox[flask_auth]
-  slapos.core
+[buildout]
+extends = common.cfg
 
 [networkcache]
 # signature certificates of the following uploaders.
@@ -53,104 +25,82 @@ signature-certificate-list =
   If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
   -----END CERTIFICATE-----
 
+
 [versions]
-Flask-Auth = 0.8
+Flask-Auth = 0.85
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.4
 async = 0.6.1
 buildout-versions = 1.7
+cns.recipe.symlink = 0.2.3
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+inotifyx = 0.2.0
+lxml = 3.0.1
+meld3 = 0.6.10
+netaddr = 0.7.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.64.2
-slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11
+pytz = 2012h
+slapos.cookbook = 0.70
+slapos.core = 0.33.1
+slapos.libnetworkcache = 0.13.3
+slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.32
 smmap = 0.8.2
+xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.33.1
+# slapos.toolbox==0.32
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.32
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.64.2
-PyXML = 0.8.4
-
-# Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.32
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
-feedparser = 5.1.2
+# slapos.toolbox==0.32
+feedparser = 5.1.3
 
 # Required by:
 # hexagonit.recipe.cmmi==1.6
-hexagonit.recipe.download = 1.5.1
+hexagonit.recipe.download = 1.6
 
 # Required by:
-# slapos.cookbook==0.64.2
-inotifyx = 0.2.0
-
-# Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# xml-marshaller==0.9.7
-lxml = 3.0alpha2
-
-# Required by:
-# slapos.cookbook==0.64.2
-netaddr = 0.7.9
-
-# Required by:
-# slapos.core==0.28.5
+# slapos.core==0.33.1
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
-paramiko = 1.7.7.2
+# slapos.toolbox==0.32
+paramiko = 1.9.0
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.32
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.64.2
-# slapos.core==0.28.5
-# slapos.libnetworkcache==0.13.2
-# slapos.toolbox==0.30
-# supervisor==3.0a12
-# zc.buildout==1.6.0-dev-SlapOS-007
-# zc.recipe.egg==1.3.2
-# zope.interface==4.0.1
+# cns.recipe.symlink==0.2.3
+# slapos.cookbook==0.70
+# slapos.core==0.33.1
+# slapos.libnetworkcache==0.13.3
+# slapos.toolbox==0.32
+# supervisor==3.0b1
+# zc.buildout==1.6.0-dev-SlapOS-010
+# zope.interface==4.0.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.64.2
-slapos.core = 0.28.5
-
-# Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
-
-# Required by:
-# slapos.cookbook==0.64.2
-xml-marshaller = 0.9.7
-
-# Required by:
-# slapos.cookbook==0.64.2
-zc.recipe.egg = 1.3.2
+# slapos.core==0.33.1
+supervisor = 3.0b1
 
 # Required by:
-# slapos.core==0.28.5
-zope.interface = 4.0.1
+# slapos.core==0.33.1
+zope.interface = 4.0.2

software/sugar-crm/software.cfg

@@ -16,7 +16,7 @@ extends =
 [application]
 recipe = slapos.recipe.build:download-unpacked
 url = http://www.sugarforge.org/frs/download.php/8558/SugarCE-6.3.0RC3.zip
-md5sum = ff4cf7c6f673f9f3f6e082c62736f628
+md5sum = 40bcc6ff38ef31194d46864a9f3548f3
 
 [instance-recipe]
 egg = slapos.cookbook

software/testnode/instance.cfg

-[buildout]
-parts =
-  testnode
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-[testnode]
-recipe = slapos.cookbook:testnode
-
-buildbot_binary = ${buildout:bin-directory}/buildbot
-git_binary = ${git:location}/bin/git
-slapgrid_partition_binary = ${buildout:bin-directory}/slapgrid-cp
-slapgrid_software_binary = ${buildout:bin-directory}/slapgrid-sr
-slapproxy_binary = ${buildout:bin-directory}/slapproxy
-svn_binary = ${subversion:location}/bin/svn
-svnversion_binary = ${subversion:location}/bin/svnversion
-zip_binary = ${zip:location}/bin/zip

software/testnode/software.cfg

-[buildout]
-recipe_location = ${:parts-directory}/slapos.cookbook
-develop = ${:recipe_location}
-versions = versions
-extensions = slapos.rebootstrap
-rebootstrap-section = python2.6
-extends =
-  ../../component/python-2.6/buildout.cfg
-  ../../component/subversion/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../component/git/buildout.cfg
-  ../../component/zip/buildout.cfg
-
-parts =
-  template
-  bootstrap
-  eggs
-  subversion
-  git
-  zip
-  checkrecipe
-
-find-links +=
-  http://www.nexedi.org/static/packages/source/slapos.buildout/
-  http://www.nexedi.org/static/packages/source/
-
-[checkrecipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-
-[slapos.cookbook]
-recipe = plone.recipe.command
-stop-on-error = true
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = "${git:location}/bin/git" clone -b testnode --quiet http://git.erp5.org/repos/slapos.git "${:location}"
-update-command = cd "${:location}" && "${git:location}/bin/git" fetch --quiet && "${git:location}/bin/git" reset --hard @{upstream}
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 03451596826e487dc97d81e27a1e7a73
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[bootstrap]
-recipe = zc.recipe.egg
-eggs = zc.buildout
-suffix =
-scripts =
-    buildout=bootstrap2.6
-arguments = sys.argv[1:] + ["bootstrap"]
-
-[rebootstrap]
-section = python2.6
-version = 1
-
-[eggs]
-dummy = ${slapos.cookbook:location}
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-  Zope2
-  collective.recipe.template
-  netaddr
-  slapos.slap
-  xml_marshaller
-  PyXML
-  slapos.core
-  slapos.cookbook
-  buildbot
-  Twisted
-entry-points = buildbot=buildbot.scripts.runner:run
-
-[versions]
-zc.buildout = 1.5.3-dev-SlapOS-001
-# only those buildout with Twisted are known to work
-buildbot = 0.7.12
-Twisted = 8.2.0
-# locally fixed PyXML which supports python 2.6
-PyXML = 0.8.4nxd001

stack/cloudooo.cfg

@@ -69,6 +69,7 @@ parts =
 # fonts
   liberation-fonts
   ipaex-fonts
+  ocrb-fonts
 
 # Dependencies
   libpng12

stack/erp5/buildout.cfg

@@ -17,7 +17,7 @@ find-links =
 
 extends =
 # Exact version of Zope
-  http://svn.zope.org/repos/main/Zope/tags/2.12.23/versions.cfg
+  http://svn.zope.org/repos/main/Zope/tags/2.12.25/versions.cfg
   ../../stack/slapos.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -48,7 +48,6 @@ extends =
   ../../component/python-2.7/buildout.cfg
   ../../component/python-ldap-python/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
-  ../../component/sphinx/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/subversion/buildout.cfg
   ../../component/tesseract/buildout.cfg
@@ -85,7 +84,6 @@ parts =
   ghostscript
   mariadb
   mroonga-mariadb
-  sphinx
   imagemagick
   libdmtx
   dmtx-utils
@@ -119,6 +117,7 @@ parts =
 # fonts
   liberation-fonts
   ipaex-fonts
+  ocrb-fonts
 
 # Cloudooo specific part
   cloudooo
@@ -186,7 +185,7 @@ context =
 [template-mariadb]
 < = template-jinja2-base
 filename = instance-mariadb.cfg
-md5sum = 0e8e9bfc8c8a5f1bd71f5c4f346ef68b
+md5sum = 867bd8e3c4f56db2aab2fcad59f2efd8
 extra-context =
     key coreutils_location coreutils:location
     key dcron_location dcron:location
@@ -200,13 +199,6 @@ extra-context =
     key sed_location sed:location
     key xtrabackup_location xtrabackup:location
 
-[template-sphinx]
-< = template-jinja2-base
-filename = instance-sphinx.cfg
-md5sum = e3975afd5220f251d6b6e7024a2ca08b
-extra-context =
-    key sphinx_location sphinx:location
-
 [template-zope]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-zope.cfg.in
@@ -227,7 +219,7 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 520e39f2612a045b42c0ad914bfdc5a9
+md5sum = 1f3e7cfc719d3c2eee4860678f6a67df
 mode = 640
 
 [template-cloudooo]
@@ -239,7 +231,7 @@ mode = 640
 [template-zope-conf]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/zope.conf.in
-md5sum = 599e004c18fd3461aa846ae75bd112f2
+md5sum = bc2154161a1d5baddc4ed4dfaaf94fbe
 mode = 640
 
 [template]
@@ -247,7 +239,7 @@ mode = 640
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 26d26c357053af48602b5110120bd085
+md5sum = 700b8729a4d9a8423ab2a99de486cc31
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -292,7 +284,6 @@ extra-context =
     key template_kumofs template-kumofs:rendered
     key template_mariadb template-mariadb:rendered
     key template_memcached template-memcached:rendered
-    key template_sphinx template-sphinx:rendered
     key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
     key template_zope template-zope:target
@@ -320,7 +311,7 @@ mode = 640
 [template-erp5-cluster]
 < = template-jinja2-base
 filename = instance-erp5-cluster.cfg
-md5sum = 3051f3fea39d7409c417dcac26027b56
+md5sum = b4a9688ca7983fac5dd573e66c6e4263
 extra-context =
     key local_bt5_repository local-bt5-repository:list
 
@@ -473,9 +464,10 @@ eggs =
   huBarcode
   qrcode
 
-# Zope 2.12 with patched acquisition
+# Zope
   ZODB3
   Zope2
+# Zope acquisition patch
   Acquisition
 
 # Other Zope 2 packages
@@ -574,7 +566,7 @@ pysvn = 1.7.4nxd006
 
 # CMF 2.3 requries Zope 2.13.
 Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
+Products.CMFCore = 2.2.6
 Products.CMFDefault = 2.2.2
 Products.CMFTopic = 2.2.1
 Products.CMFUid = 2.2.1
@@ -583,7 +575,7 @@ Products.CMFUid = 2.2.1
 slapos.cookbook =
 
 # Pinned versions
-Flask = 0.8
+Flask = 0.9
 GitPython = 0.3.2.RC1
 MySQL-python = 1.2.3
 PIL = 1.1.7
@@ -591,68 +583,63 @@ Paste = 1.7.5.1
 PasteDeploy = 1.5.0
 PasteScript = 1.7.5
 Products.CMFActionIcons = 2.1.3
-Products.CMFCalendar = 2.2.2
-Products.CMFCore = 2.2.5
-Products.CMFDefault = 2.2.2
-Products.CMFTopic = 2.2.1
-Products.CMFUid = 2.2.1
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
-Products.GenericSetup = 1.7.1
+Products.GenericSetup = 1.7.2
 Products.MimetypesRegistry = 2.0.3
-Products.PluggableAuthService = 1.7.7
-Products.PluginRegistry = 1.3b1
-Products.TIDStorage = 5.4.7.dev-r45842
+Products.PluggableAuthService = 1.8.0
+Products.PluginRegistry = 1.3
+Products.TIDStorage = 5.4.8
 Products.Zelenium = 1.0.3
 StructuredText = 2.11.1
 WSGIUtils = 0.7
 Werkzeug = 0.8.3
-apache-libcloud = 0.9.1
-argparse = 1.1
+apache-libcloud = 0.11.1
+argparse = 1.2.1
 async = 0.6.1
 atomize = 0.1.1
 buildout-versions = 1.7
 cElementTree = 1.0.5-20051216
-chardet = 1.0.1
-coverage = 3.5.1
+chardet = 1.1
+coverage = 3.5.2
 csp-eventlet = 0.7.0
 elementtree = 1.2.7-20070827-preview
 erp5.recipe.cmmiforcei686 = 0.1.3
-erp5.util = 0.4.6
+erp5.util = 0.4.7
 erp5diff = 0.8.1.5
-eventlet = 0.9.16
-feedparser = 5.1.1
+eventlet = 0.9.17
+feedparser = 5.1.2
 five.localsitemanager = 2.0.5
 fpconst = 0.7.2
 gitdb = 0.5.4
-greenlet = 0.3.4
-hexagonit.recipe.cmmi = 1.5.0
-http-parser = 0.7.5
+greenlet = 0.4.0
+hexagonit.recipe.cmmi = 1.6
+http-parser = 0.7.8
 huBarcode = 0.63
 inotifyx = 0.2.0
-ipdb = 0.6.1
-ipython = 0.12
+ipdb = 0.7
+ipython = 0.13
 meld3 = 0.6.8
-netaddr = 0.7.6
+netaddr = 0.7.9
 netifaces = 0.8
 ordereddict = 1.1
-paramiko = 1.7.7.1
+paramiko = 1.7.7.2
 plone.recipe.command = 1.1
 ply = 3.4
 polib = 1.0.0
-psutil = 0.4.1
-python-ldap = 2.4.9
-python-magic = 0.4.2
+psutil = 0.6.1
+python-ldap = 2.4.10
+python-magic = 0.4.3
 python-memcached = 1.47
-restkit = 4.1.2
+restkit = 4.2.0
 rtjp-eventlet = 0.3.2
 setuptools = 0.6c12dev-r88846
-slapos.core = 0.24
-slapos.recipe.build = 0.7
-slapos.recipe.template = 2.4
-slapos.toolbox = 0.23
+slapos.core = 0.28
+slapos.recipe.build = 0.10
+slapos.recipe.template = 2.4.2
+slapos.toolbox = 0.29
 smmap = 0.8.2
-socketpool = 0.3.0
+socketpool = 0.5.2
 supervisor = 3.0a12
 threadframe = 0.2
 timerserver = 2.0.2
@@ -661,3 +648,6 @@ uuid = 1.30
 validictory = 0.8.3
 xml-marshaller = 0.9.7
 xupdate-processor = 0.4
+mr.developer = 1.21
+qrcode = 2.4.1
+requests = 0.13.2

stack/erp5/instance-erp5-cluster.cfg.in

@@ -24,13 +24,6 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 
-[request-sphinx]
-<=request-common
-name = Sphinx Search Engine
-software-type = sphinx
-sla-computer_guid = ${slap-parameter:sphinx-computer-guid}
-return = url-sphinx url-sphinx-sql
-
 [request-mariadb]
 <=request-common
 name = MariaDB DataBase
@@ -67,8 +60,6 @@ config =
   memcached-url
   cloudooo-url
   kumofs-url
-  sphinx-url-sphinx
-  sphinx-url-sphinx-sql
   smtp-url
   bt5
   bt5-repository-url
@@ -77,8 +68,6 @@ config-mysql-url = ${request-mariadb:connection-url}
 config-memcached-url = ${request-memcached:connection-url}
 config-cloudooo-url = ${request-cloudooo:connection-url}
 config-kumofs-url = ${request-kumofs:connection-url}
-config-sphinx-url-sphinx = ${request-sphinx:connection-url-sphinx}
-config-sphinx-url-sphinx-sql = ${request-sphinx:connection-url-sphinx-sql}
 config-bt5 = ${slap-parameter:bt5}
 config-bt5-repository-url = ${slap-parameter:bt5-repository-url}
 config-smtp-url = ${slap-parameter:smtp-url}
@@ -97,7 +86,6 @@ sla-computer_guid = ${slap-parameter:varnish-computer-guid}
 
 [slap-parameter]
 # Default value if no computer_guid is specified for each type
-sphinx-computer-guid = ${slap-connection:computer-id}
 mariadb-computer-guid = ${slap-connection:computer-id}
 cloudooo-computer-guid = ${slap-connection:computer-id}
 memcached-computer-guid = ${slap-connection:computer-id}

stack/erp5/instance-mariadb.cfg.in

@@ -30,7 +30,7 @@ recipe = slapos.cookbook:generic.mysql
 
 # Options
 user = user
-parallel-test-database-amount = 100
+parallel-test-database-amount = ${slap-parameter:test-database-amount}
 port = 45678
 ip = ${slap-network-information:local-ipv4}
 database = erp5
@@ -178,3 +178,6 @@ recipe = slapos.cookbook:check_port_listening
 path = ${basedirectory:promise}/mariadb
 hostname = ${mariadb-instance:ip}
 port = ${mariadb-instance:port}
+
+[slap-parameter]
+test-database-amount = 100

stack/erp5/instance-sphinx.cfg.in

-[buildout]
-parts =
-  publish-sphinx-url
-  sphinxd-instance
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[publish-sphinx-url]
-recipe = slapos.cookbook:publish
-url-sphinx = sphinx://${sphinxd-instance:ip}:${sphinxd-instance:sphinx-port}
-url-sphinx-sql = mysql://${sphinxd-instance:ip}:${sphinxd-instance:sql-port}
-
-[sphinxd-instance]
-recipe = slapos.cookbook:sphinx
-data-directory = ${directory:sphinx-data}
-configuration-file = ${rootdirectory:etc}/sphinx.conf
-searchd-log = ${basedirectory:log}/sphinx-searchd.log
-query-log = ${basedirectory:log}/sphinx-query.log
-pid = ${basedirectory:run}/sphinx-searchd.pid
-ip = ${slap-network-information:local-ipv4}
-sphinx-port = 9312
-sql-port = 9306
-wrapper = ${basedirectory:services}/sphinxd
-sphinx-searchd-binary = {{ sphinx_location }}/bin/searchd
-
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = ${buildout:directory}/etc
-var = ${buildout:directory}/var
-srv = ${buildout:directory}/srv
-bin = ${buildout:directory}/bin
-
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = ${rootdirectory:var}/log
-services = ${rootdirectory:etc}/run
-run = ${rootdirectory:var}/run
-
-[directory]
-recipe = slapos.cookbook:mkdirectory
-sphinx-data = ${rootdirectory:srv}/sphinx

stack/erp5/instance-tidstorage.cfg.in

@@ -485,7 +485,6 @@ recipe = slapos.cookbook:publish
 [tidstorage]
 recipe = slapos.cookbook:tidstorage
 known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }}
-base-url = http://${zope-admin:ip}:${zope-admin:port}/%s/serialize
 configuration-path = ${directory:etc}/tidstorage.py
 ip = {{ ipv4 }}
 port = 6001

stack/erp5/instance.cfg.in

@@ -197,7 +197,6 @@ memcached = {{ template_memcached }}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 zope = ${dynamic-template-zope:rendered}
 mariadb = {{ template_mariadb }}
-sphinx = {{ template_sphinx }}
 tidstorage = ${dynamic-template-tidstorage:rendered}
 varnish = ${dynamic-template-varnish:rendered}
 

stack/erp5/zope.conf.in

@@ -43,10 +43,11 @@ products {{ instance_products }}
 </product-config>
 
 {% endif -%}
-{% if 'timerserver-interval' in parameter_dict -%}
+{% set timerserver_interval = parameter_dict.get('timerserver-interval', '0') | int -%}
+{% if timerserver_interval -%}
 %import timerserver
 <timer-server>
-  interval {{ parameter_dict['timerserver-interval'] }}
+  interval {{ timerserver_interval }}
 </timer-server>
 
 {% endif -%}

stack/lamp/instance-apache-backup.cfg → stack/lamp/apache/instance-apache-backup.cfg.in

@@ -16,32 +16,92 @@ eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
-ssh-public-key = $${sshkeys-dropbear:public-key-value}
-ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[apache-proxy]
-recipe = slapos.cookbook:apacheproxy
-url = $${slap-parameter:proxy-url}
-pid-file = $${basedirectory:run}/apache.pid
-lock-file = $${basedirectory:run}/apache.lock
-ip = $${slap-network-information:global-ipv6}
-port = 8080
-error-log = $${directory:httpd-log}/error.log
-access-log = $${directory:httpd-log}/access.log
-httpd-conf = $${rootdirectory:etc}/apache.conf
-wrapper = $${basedirectory:services}/apache
+#----------------
+#--
+#-- Creation of all needed directories.
 
-promise = $${basedirectory:promises}/apache
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
-httpd-binary = ${apache:location}/bin/httpd
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- sshkeys
 
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -60,6 +120,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -75,6 +140,11 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
+
+#----------------
+#--
+#-- rdiff
+
 [rdiff-backup-server]
 recipe = slapos.cookbook:pbs
 client = false
@@ -82,18 +152,26 @@ path = $${directory:htdocs}
 wrapper = $${rootdirectory:bin}/rdiffbackup-server
 rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
+
+#----------------
+#--
+#-- Apache Proxy.
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
 
 [logrotate-entry-apache]
 <= logrotate
@@ -106,51 +184,14 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- Publish instance parameters.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[urls]
+recipe = slapos.cookbook:publish
+url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
+ssh-public-key = $${sshkeys-dropbear:public-key-value}
+ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-htdocs = $${rootdirectory:srv}/www/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-httpd-log = $${basedirectory:log}/apache/

stack/lamp/apache/instance-apache-import.cfg

+[buildout]
+extends = ${instance-apache-php:output}
+          ${pbsready-export:output}
+
+parts =
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- sshkeys
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+
+#----------------
+#--
+#-- rdiff
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+

stack/lamp/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg.in

@@ -10,7 +10,8 @@ parts =
   cron-entry-logrotate
   promise
   slapmonitor
-  slapreport
+  slapmonitor-xml
+
   frontend-promise
   content-promise
   publish-connection-informations
@@ -20,46 +21,98 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
 
-# Creation of all needed directories
+#----------------
+#--
+#-- Creation of all needed directories.
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
 
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-ca-dir = $${rootdirectory:srv}/ssl/
-httpd-log = $${basedirectory:log}/apache/
-php-ini-dir = $${rootdirectory:etc}/php/
-tmp-php = $${rootdirectory:tmp}/php/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+httpd-log = $${basedirectory:log}/apache
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
+requests = $${directory:ca-dir}/requests
+private = $${directory:ca-dir}/private
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts
+crl = $${directory:ca-dir}/crl
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
 
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- Deploy stunnel.
 
-# Deploy stunnel
 [stunnel]
 recipe = slapos.cookbook:stunnel
 client = true
@@ -76,8 +129,22 @@ pid-file = $${basedirectory:run}/stunnel.pid
 wrapper = $${rootdirectory:bin}/raw_stunnel
 post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+
+#----------------
+#--
+#-- Certificate stuff.
 
-# Certificate stuffs
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
 openssl-binary = ${openssl:location}/bin/openssl
@@ -98,7 +165,10 @@ key-file = $${stunnel:key-file}
 cert-file = $${stunnel:cert-file}
 
 
-# Request MariaDB instance and parse its URL
+#----------------
+#--
+#-- Request MariaDB instance and parse its URL.
+
 [request-mariadb]
 <= slap-connection
 recipe = slapos.cookbook:request
@@ -107,14 +177,17 @@ software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
 sla = computer_guid
-sla-computer_guid = ${slap-connection:computer-id}
+sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
 recipe = slapos.cookbook:urlparse
 url = $${request-mariadb:connection-url}
 
 
-# Deploy Apache + PHP application
+#----------------
+#--
+#-- Deploy Apache + PHP application.
+
 [apache-php]
 recipe = slapos.cookbook:apachephp
 source = ${application:location}
@@ -142,21 +215,6 @@ mysql-database = $${mariadb-urlparse:path}
 mysql-host = $${stunnel:local-host}
 mysql-port = $${stunnel:local-port}
 
-
-# Deploy logrotate, cron, configure it
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${rootdirectory:bin}/logrotate
-conf = $${rootdirectory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${rootdirectory:srv}/logrotate.status
-
 [logrotate-entry-apache]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
@@ -168,40 +226,11 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[logrotate-entry-stunnel]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = stunnel
-log = $${stunnel:log-file}
-frequency = daily
-rotate-num = 30
-notifempty = true
-create = true
-post = $${stunnel:post-rotate-script}
-
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
-
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
 
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
+#----------------
+#--
+#-- Request frontend.
 
-
-# Request frontend
 [request-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -215,37 +244,38 @@ return = site_url
 config-custom_domain = $${slap-parameter:domain}
 
 
-# Deploy slapmonitor
+#----------------
+#--
+#-- Deploy slapmonitor.
+
 [slapmonitor]
 recipe = slapos.cookbook:slapmonitor
 pid-file = $${basedirectory:run}/apache.pid
 database-path = $${basedirectory:log}/slapmonitor.db
-shell-path = ${dash:location}/bin/dash
 slapmonitor-path = ${buildout:bin-directory}/slapmonitor
 path = $${basedirectory:services}/slapmonitor
 
-[slapreport]
-recipe = slapos.cookbook:slapreport
-pid-file = $${basedirectory:run}/apache.pid
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
 database-path = $${basedirectory:log}/slapmonitor.db
-consumption-log-path = $${basedirectory:log}/instance_consumption.log
-logbox-ip = 87.98.152.12
-logbox-port = 5122
-logbox-user = admin
-logbox-passwd = passer
-shell-path = ${dash:location}/bin/dash
-slapreport-path = ${buildout:bin-directory}/slapreport
-path = $${basedirectory:services}/slapreport
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
 
 
-# Publish all instance parameters (url of instance)
+#----------------
+#--
+#-- Publish instance parameters.
+
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend_url = $${apache-php:url}
 url = $${request-frontend:connection-site_url}
 
 
-# Deploy promises scripts
+#----------------
+#--
+#-- Deploy promises scripts.
+
 [promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${basedirectory:promises}/apache
@@ -266,6 +296,9 @@ url = $${request-frontend:connection-site_url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
+
+
+
 [slap-parameter]
 # Default value if no domain is specified
 domain =
@@ -274,3 +307,4 @@ logbox-ip =
 logbox-port =
 logbox-user =
 logbox-passwd =
+

stack/lamp/buildout.cfg

 [buildout]
+
+ignore-existing = true
+
 parts =
+  slapos-cookbook
   apache-php
   mariadb
   mydumper
@@ -7,15 +11,20 @@ parts =
   rdiff-backup
   dropbear
   eggs
-  instance-recipe-egg
-  template
-  template-apache-php
-  template-mariadb
-  template-mariadb-pbsready
-  template-mariadb-pbsready-import
-  template-mariadb-pbsready-export
+  instance
+
+  instance-apache-php
+  instance-mariadb
+
+#Contains the importer and exporter recipes for mariadb
+  instance-mariadb-import
+  instance-mariadb-export
+
+  instance-default-root
+
 
 extends =
+  ../resilient/buildout.cfg
   ../../component/mariadb/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/apache-php/buildout.cfg
@@ -42,83 +51,85 @@ extends =
 # Compile dir is for plugins, there's no plugin in LAMP
 keep-compile-dir = false
 
-[instance-recipe]
-egg = slapos.cookbook
-module = lamp.request
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
 [application]
 recipe = hexagonit.recipe.download
 #If provided tarball does not containt top directory this option shall be changed to false
 strip-top-level-dir = true
 
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-output = ${buildout:directory}/template.cfg
-md5sum = 8117f10e814a13c5376af4c01e6546d4
-mode = 0644
 
-[template-apache-php]
+#----------------
+#-- Instance-level buildout profiles.
+
+[instance]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-php.cfg
-output = ${buildout:directory}/template-apache-php.cfg
-md5sum = 59cee571ea0fcdf2e6c9b3195e258738
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = 38bdcf0a8263d4a19bd6a35c0cd00340
 mode = 0644
 
-[template-apache-backup]
+[instance-apache-php]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-apache-backup.cfg
-output = ${buildout:directory}/template-apache-backup.cfg
-md5sum = cfb77ac8785e0d125a785f69a5339014
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/instance-apache-php.cfg
+md5sum = fc29d853dcd0802dd61c60b09e898c11
 mode = 0644
 
-[template-backuped]
+[instance-apache-backup]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-backuped.cfg
-output = ${buildout:directory}/template-backuped.cfg
-md5sum = f43d1c6412ea8dc83b75573dc00daf9e
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
+output = ${buildout:directory}/instance-apache-backup.cfg
+md5sum = 48f969d82319a9d145570f5f0fd27672
 mode = 0644
 
-[template-mariadb]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb.cfg
-output = ${buildout:directory}/template-mariadb.cfg
-md5sum = 767452bc503ff6c1b7af0ebfac590c9f
+[template-resilient-lamp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/template-resilient.cfg.in
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key templateapache instance-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 5605ad8896c2718854bf26148c4ae940
 mode = 0644
+ 
 
-[template-mariadb-pbsready]
+[instance-mariadb]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
-output = ${buildout:directory}/template-mariadb-pbsready.cfg
-md5sum = d2a580dcd7efdd528be45c5ffadfe760
+url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg.in
+output = ${buildout:directory}/instance-mariadb.cfg
+md5sum = ba8dd08dfd5e6a9dc614693d066eb21d
 mode = 0644
 
-[template-mariadb-pbsready-import]
+[instance-mariadb-import]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
-md5sum = 4a96ff02da3898fef7077fa8baec81ac
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg.in
+output = ${buildout:directory}/instance-mariadb-import.cfg
+md5sum = ea43b8ed38a55a11b027fc283c0e718a
 mode = 0644
 
-[template-mariadb-pbsready-export]
+[instance-mariadb-export]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
-output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
-md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
+url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg.in
+output = ${buildout:directory}/instance-mariadb-export.cfg
+md5sum = 685c8abf0f487c72273846002ec631a0
 mode = 0644
 
-[template-pull-backup]
+
+[instance-default-root]
 recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-pull-backup.cfg
-output = ${buildout:directory}/template-pull-backup.cfg
-md5sum = 9aab30ba5aa23a37d4b507e7c414be00
+url = ${:_profile_base_location_}/instance-default-root.cfg.in
+output = ${buildout:directory}/instance-default-root.cfg
+md5sum = 53c9020f7a0b5203f976e069e455787b
 mode = 0644
 
-# Dummy parts in case no application configuration file is needed
+
+#----------------
+#-- Dummy parts in case no application configuration file is needed
+
 [application-template]
 filename =
 location =
@@ -126,6 +137,8 @@ location =
 [application-configuration]
 location =
 
+#----------------
+
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -142,7 +155,7 @@ eggs =
 #   Cedric de Saint Martin
 #   Yingjie Xu
 #   Gabriel Monnerat
-#   Łukasz Nowak
+#   Łukasz Nowak
 #   Test Agent Signature
 #   Alain Takoudjou
 signature-certificate-list =
@@ -265,43 +278,45 @@ signature-certificate-list =
   -----END CERTIFICATE-----
 
 [versions]
+# 1.2.4b doesn't download
+MySQL-python = 1.2.3
+
 Jinja2 = 2.6
 Werkzeug = 0.8.3
-apache-libcloud = 0.11.1
+apache-libcloud = 0.11.3
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
 hexagonit.recipe.cmmi = 1.6
-meld3 = 0.6.8
+meld3 = 0.6.9
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.66
-slapos.libnetworkcache = 0.13.2
-slapos.recipe.build = 0.11.2
+slapos.cookbook = 0.69
+slapos.recipe.build = 0.11.5
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.30
+slapos.toolbox = 0.31
 smmap = 0.8.2
 
 # Required by:
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
 Flask = 0.9
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.69
 PyXML = 0.8.4
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 feedparser = 5.1.2
 
 # Required by:
@@ -309,58 +324,61 @@ feedparser = 5.1.2
 hexagonit.recipe.download = 1.5.1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.69
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
+# slapos.cookbook==0.69
+# slapos.core==0.31.2
 # xml-marshaller==0.9.7
-lxml = 3.0alpha2
+lxml = 3.0.1
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.69
 netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 netifaces = 0.8
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 paramiko = 1.7.7.2
 
 # Required by:
-# slapos.toolbox==0.30
+# slapos.toolbox==0.31
 psutil = 0.6.1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.core==0.28.5
-# slapos.toolbox==0.30
-# zc.buildout==1.6.0-dev-SlapOS-006
+# slapos.cookbook==0.69
+pytz = 2012f
+
+# Required by:
+# slapos.cookbook==0.69
+# slapos.core==0.31.2
+# slapos.toolbox==0.31
+# zc.buildout==1.6.0-dev-SlapOS-010
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
-slapos.core = 0.28.5
+# slapos.cookbook==0.69
+# slapos.toolbox==0.31
+slapos.core = 0.31.2
 
 # Required by:
-# slapos.core==0.28.5
-supervisor = 3.0a12
+# slapos.core==0.31.2
+supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.66
-# slapos.toolbox==0.30
+# slapos.cookbook==0.69
+# slapos.toolbox==0.31
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.66
+# slapos.cookbook==0.69
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.28.5
+# slapos.core==0.31.2
 zope.interface = 4.0.1
-

stack/lamp/instance-backuped.cfg

-[buildout]
-
-extends =
-  ${template-apache-php:output}
-
-parts =
-  request-mariadb
-  request-mariadb-pseudo-replicating
-  request-apache-backup
-  request-pull-backup-server
-  request-pull-backup-server-mariadb
-  request-pull-backup-server-mariadb-backup
-  request-pull-backup-server-apache
-  request-pull-backup-server-apache-backup
-  url
-  apache-php
-  stunnel
-  certificate-authority
-  ca-stunnel
-  logrotate
-  logrotate-entry-apache
-  logrotate-entry-stunnel
-  cron
-  cron-entry-logrotate
-  dropbear-server
-  sshkeys-authority
-  dropbear-server-pbs-authorized-key
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
-
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-wrapper = $${basedirectory:services}/sshkeys_authority
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = dropbear
-type = rsa
-executable = $${dropbear-server:wrapper}
-public-key = $${dropbear-server:rsa-keyfile}.pub
-private-key = $${dropbear-server:rsa-keyfile}
-wrapper = $${basedirectory:services}/sshd
-
-[dropbear-server]
-recipe = slapos.cookbook:dropbear
-host = $${slap-network-information:global-ipv6}
-port = 2222
-home = $${directory:ssh}
-wrapper = $${rootdirectory:bin}/raw_sshd
-shell = $${rdiff-backup-server:wrapper}
-rsa-keyfile = $${directory:ssh}/server_key.rsa
-dropbear-binary = ${dropbear:location}/sbin/dropbear
-
-[dropbear-server-pbs-authorized-key]
-<= dropbear-server
-recipe = slapos.cookbook:dropbear.add_authorized_key
-key = $${request-pull-backup-server:connection-ssh-key}
-
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:htdocs}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
-
-[request-mariadb]
-software-type = mariadb-pbsready-export
-config = authorized-key notify
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-return = url ssh-public-key ssh-url notification-id
-
-[request-mariadb-pseudo-replicating]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = MariaDB Backup
-software-url = $${slap-connection:software-release-url}
-software-type = mariadb-pbsready-import
-return = url ssh-public-key ssh-url notification-url
-config = authorized-key on-notification
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
-
-[request-apache-backup]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = Apache Backup
-software-url = $${slap-connection:software-release-url}
-software-type = apache-backup
-return = url ssh-url ssh-public-key
-config = authorized-key proxy-url
-config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
-config-proxy-url = $${url:url}
-
-[request-pull-backup-server-apache]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pulling from Apache
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key notify notification-id frequency
-config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
-config-type = pull
-config-server-key = $${sshkeys-dropbear:public-key-value}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
-config-frequency = 30 * * * *
-slave = true
-
-[request-pull-backup-server-apache-backup]
-<= request-pull-backup-server
-   slap-connection
-recipe = slapos.cookbook:request
-name = PBS pushing to $${request-apache-backup:name}
-software-url = $${slap-connection:software-release-url}
-config = url name type server-key on-notification
-config-url = $${request-apache-backup:connection-ssh-url}
-config-name = $${request-pull-backup-server-apache:config-name}
-config-type = push
-config-server-key = $${request-apache-backup:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
-slave = true
-
-[request-pull-backup-server-mariadb]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pulling from $${request-mariadb:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb:connection-ssh-url}
-config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
-config-type = pull
-config-server-key = $${request-mariadb:connection-ssh-public-key}
-config-on-notification = $${request-mariadb:connection-notification-id}
-config-notify = $${request-pull-backup-server:connection-notification-url}
-config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
-config-title = Pulling from MariaDB
-slave = true
-
-[request-pull-backup-server-mariadb-backup]
-<= request-pull-backup-server
-   slap-connection
-name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
-config = url name type server-key on-notification notify notification-id title
-config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
-config-name = $${request-pull-backup-server-mariadb:config-name}
-config-type = push
-config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
-config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
-config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
-config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
-config-title = Pushing to MariaDB backup
-slave = true
-
-[directory]
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys

stack/lamp/instance-default-root.cfg.in

+[buildout]
+parts = request-apache
+
+[request-apache]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = apache
+name = Apache
+

stack/lamp/instance-mariadb-pbsready-export.cfg

-[buildout]
-extends = ${template-mariadb-pbsready:output}
-
-parts += cron-entry-mariadb-backup
-
-[urls]
-notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
-
-[mydumper]
-recipe = slapos.cookbook:mydumper
-wrapper = $${rootdirectory:bin}/raw_mydumper
-backup-directory = $${directory:mariadb-backup}
-socket = $${mariadb:socket}
-user = root
-mydumper-binary = ${mydumper:location}/bin/mydumper
-database = $${mariadb:database}
-import = false
-
-[notifier-mydumper]
-<= notifier
-recipe = slapos.cookbook:notifier.notify
-name = mydumper
-title = Dumping MariaDB Database
-executable = $${mydumper:wrapper}
-wrapper = $${rootdirectory:bin}/mydumper
-notify = $${slap-parameter:notify}
-
-[cron-entry-mariadb-backup]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = backup
-frequency = 0 * * * *
-command = $${notifier-mydumper:wrapper}

stack/lamp/instance.cfg.in

+[buildout]
+
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${instance-apache-php:output}
+resilient = ${template-resilient-lamp:rendered}
+mariadb = ${instance-mariadb:output}
+mariadb-import = ${instance-mariadb-import:output}
+mariadb-export = ${instance-mariadb-export:output}
+apache-backup = ${instance-apache-backup:output}
+
+# To prepend an empty root instance:
+#default = ${instance-default-root:output}
+#apache = ${instance-apache-php:output}
+
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+

stack/lamp/mariadb/instance-mariadb-export.cfg.in

+[buildout]
+extends = ${instance-mariadb:output}
+          ${pbsready-export:output}
+
+parts += mariadb
+
+[exporter]
+recipe = slapos.cookbook:mydumper
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+backup-directory = $${directory:backup}
+socket = $${mariadb:socket}
+user = root
+mydumper-binary = ${mydumper:location}/bin/mydumper
+database = $${mariadb:database}
+import = false

stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg.in

 [buildout]
+extends = ${instance-mariadb:output}
+          ${pbsready-import:output}
 
-extends = ${template-mariadb-pbsready:output}
+parts += mariadb
 
-parts += mariadb-import-on-notification
-
-[urls]
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-
-[mydumper-import]
+[importer]
 recipe = slapos.cookbook:mydumper
 wrapper = $${rootdirectory:bin}/myloader
-backup-directory = $${directory:mariadb-backup}
+backup-directory = $${directory:backup}
 socket = $${mariadb:socket}
 user = root
 myloader-binary = ${mydumper:location}/bin/myloader
 database = $${mariadb:database}
 import = true
-
-[mariadb-import-on-notification]
-<= notifier
-recipe = slapos.cookbook:notifier.callback
-on-notification-id = $${slap-parameter:on-notification}
-callback = $${mydumper-import:wrapper}

stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg.in

@@ -12,109 +12,80 @@ parts =
   cron
   cron-entry-logrotate
   slapmonitor
-  slapreport
+  slapmonitor-xml
 
 gzip-binary = ${gzip:location}/bin/gzip
 
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[urls]
-recipe = slapos.cookbook:publish
-url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
 
-[mariadb]
-recipe = slapos.cookbook:mysql
-
-# Options
-recovering = false
-user = user
-port = 3306
-ip = $${slap-network-information:local-ipv4}
-database = db
+#----------------
+#--
+#-- Creation of all needed directories.
 
-# Paths
-wrapper = $${basedirectory:services}/mariadb
-update-wrapper = $${basedirectory:services}/mariadb_update
-logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
-data-directory = $${directory:mariadb-data}
-pid-file = $${basedirectory:run}/mariadb.pid
-socket = $${basedirectory:run}/mariadb.sock
-error-log = $${basedirectory:log}/mariadb_error.log
-conf-file = $${rootdirectory:etc}/mariadb.cnf
-promise = $${basedirectory:promises}/mysql
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
 
-# Binary information
-mysql-base-directory = ${mariadb:location}
-mysql-binary = ${mariadb:location}/bin/mysql
-mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
-mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
-mysqld-binary = ${mariadb:location}/bin/mysqld
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
 
-[slapmonitor]
-recipe = slapos.cookbook:slapmonitor
-pid-file = $${basedirectory:run}/mariadb.pid
-database-path = $${basedirectory:log}/slapmonitor.db
-shell-path = ${dash:location}/bin/dash
-slapmonitor-path = ${buildout:bin-directory}/slapmonitor
-path = $${basedirectory:services}/slapmonitor
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+mariadb-data = $${rootdirectory:srv}/mariadb
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
+
+
+#----------------
+#--
+#-- Deploy cron.
 
-[slapreport]
-recipe = slapos.cookbook:slapreport
-pid-file = $${basedirectory:run}/mariadb.pid
-consumption-log-path = $${basedirectory:log}/instance_consumption.log
-database-path = $${basedirectory:log}/slapmonitor.db
-logbox-ip = 87.98.152.12
-logbox-port = 5122
-logbox-user = admin
-logbox-passwd = passer
-shell-path = ${dash:location}/bin/dash
-slapreport-path = ${buildout:bin-directory}/slapreport
-path = $${basedirectory:services}/slapreport
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
 
-[certificate-authority]
-recipe = slapos.cookbook:certificate_authority
-openssl-binary = ${openssl:location}/bin/openssl
-ca-dir = $${directory:ca-dir}
-requests-directory = $${cadirectory:requests}
-wrapper = $${basedirectory:services}/ca
-ca-private = $${cadirectory:private}
-ca-certs = $${cadirectory:certs}
-ca-newcerts = $${cadirectory:newcerts}
-ca-crl = $${cadirectory:crl}
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
 
-[cadirectory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:ca-dir}/requests/
-private = $${directory:ca-dir}/private/
-certs = $${directory:ca-dir}/certs/
-newcerts = $${directory:ca-dir}/newcerts/
-crl = $${directory:ca-dir}/crl/
 
-[ca-stunnel]
-<= certificate-authority
-recipe = slapos.cookbook:certificate_authority.request
-executable = $${stunnel:wrapper}
-wrapper = $${basedirectory:services}/stunnel
-key-file = $${stunnel:key-file}
-cert-file = $${stunnel:cert-file}
+#----------------
+#--
+#-- Deploy logrotate.
 
-[stunnel]
-recipe = slapos.cookbook:stunnel
-stunnel-binary = ${stunnel:location}/bin/stunnel
-wrapper = $${rootdirectory:bin}/stunnel
-log-file = $${basedirectory:log}/stunnel.log
-config-file = $${directory:stunnel-conf}/stunnel.conf
-key-file = $${directory:stunnel-conf}/stunnel.key
-cert-file = $${directory:stunnel-conf}/stunnel.crt
-pid-file = $${basedirectory:run}/stunnel.pid
-local-host = $${mariadb:ip}
-local-port = $${mariadb:port}
-remote-host = $${slap-network-information:global-ipv6}
-remote-port = 6446
-client = false
-post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
 
 [logrotate]
 recipe = slapos.cookbook:logrotate
@@ -162,52 +133,119 @@ rotate-num = 30
 notifempty = true
 create = true
 
-[cron]
-recipe = slapos.cookbook:cron
-dcrond-binary = ${dcron:location}/sbin/crond
-cron-entries = $${directory:cron-entries}
-crontabs = $${directory:crontabs}
-cronstamps = $${directory:cronstamps}
-catcher = $${cron-simplelogger:wrapper}
-binary = $${basedirectory:services}/crond
+#----------------
+#--
+#-- Deploy stunnel.
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+[stunnel]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = ${stunnel:location}/bin/stunnel
+wrapper = $${rootdirectory:bin}/stunnel
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+local-host = $${mariadb:ip}
+local-port = $${mariadb:port}
+remote-host = $${slap-network-information:global-ipv6}
+remote-port = 6446
+client = false
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
 
-[cron-entry-logrotate]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = logrotate
-frequency = 0 0 * * *
-command = $${logrotate:wrapper}
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
+#----------------
+#--
+#-- Certificate stuff.
 
-[basedirectory]
-recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
 
-[directory]
+[cadirectory]
 recipe = slapos.cookbook:mkdirectory
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronstamps = $${rootdirectory:etc}/cronstamps/
-ca-dir = $${rootdirectory:srv}/ssl/
-mariadb-data = $${rootdirectory:srv}/mariadb/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-stunnel-conf = $${rootdirectory:etc}/stunnel/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+#----------------
+#--
+#-- Creates a MariaDB configuration file, and a database.
+
+[mariadb]
+recipe = slapos.cookbook:mysql
+
+# Options
+recovering = false
+user = user
+port = 3306
+ip = $${slap-network-information:local-ipv4}
+database = db
+
+# Paths
+wrapper = $${basedirectory:services}/mariadb
+update-wrapper = $${basedirectory:services}/mariadb_update
+logrotate-post = $${rootdirectory:bin}/mariadb_post_logrotate
+data-directory = $${directory:mariadb-data}
+pid-file = $${basedirectory:run}/mariadb.pid
+socket = $${basedirectory:run}/mariadb.sock
+error-log = $${basedirectory:log}/mariadb_error.log
+conf-file = $${rootdirectory:etc}/mariadb.cnf
+promise = $${basedirectory:promises}/mysql
+
+# Binary information
+mysql-base-directory = ${mariadb:location}
+mysql-binary = ${mariadb:location}/bin/mysql
+mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
+mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
+mysqld-binary = ${mariadb:location}/bin/mysqld
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/mariadb.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
+ip = $${slap-network-information:global-ipv6}
+
 
 [slap-parameter]
 #Default  value if no ssh parameters specified

stack/lamp/template-resilient.cfg.in

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+   {{templateapache}}
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("mariadb","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+
+{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
+
+
+[request-pull-backup-server]
+<= request-pbs-common
+name = PBS (Pull Backup Server)
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests
+keys = ${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pbs-common
+name = PBS pulling from Apache 1
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-2]
+<= request-pbs-common
+name = PBS pulling from Apache 2
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-1:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-2:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/lapp/README.txt

+
+LAPP stack
+==========
+
+This fork of the LAMP stack provides:
+
+ - a Postgres instance, with an empty database and a 'postgres' superuser.
+   Log rotation is handled by Postgres itself.
+
+ - symlinks to all the postgres binaries, usable through unix socket
+   with no further authentication, or through ipv6
+
+ - a psycopg2 (postgres driver) egg
+
+ - configuration for a maarch instance (this part should be brought outside the stack)
+

stack/lapp/apache/instance-apache-backup.cfg.in

+[buildout]
+
+parts =
+  urls
+  apache-proxy
+  logrotate
+  logrotate-entry-apache
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
+  dropbear-server
+  dropbear-server-pbs-authorized-key
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+htdocs = $${rootdirectory:srv}/www
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+ssh = $${rootdirectory:etc}/ssh
+sshkeys = $${rootdirectory:srv}/sshkeys
+httpd-log = $${basedirectory:log}/apache
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- sshkeys
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = $${dropbear-server:wrapper}
+public-key = $${dropbear-server:rsa-keyfile}.pub
+private-key = $${dropbear-server:rsa-keyfile}
+wrapper = $${basedirectory:services}/sshd
+
+
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = $${slap-network-information:global-ipv6}
+port = 2222
+home = $${directory:ssh}
+wrapper = $${rootdirectory:bin}/raw_sshd
+shell = $${rdiff-backup-server:wrapper}
+rsa-keyfile = $${directory:ssh}/server_key.rsa
+dropbear-binary = ${dropbear:location}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = $${slap-parameter:authorized-key}
+
+
+#----------------
+#--
+#-- rdiff
+
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:htdocs}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+
+#----------------
+#--
+#-- Apache Proxy.
+
+[apache-proxy]
+recipe = slapos.cookbook:apacheproxy
+url = $${slap-parameter:proxy-url}
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+promise = $${basedirectory:promises}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-proxy:error-log} $${apache-proxy:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = http://[$${apache-proxy:ip}]:$${apache-proxy:port}/
+ssh-public-key = $${sshkeys-dropbear:public-key-value}
+ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
+

stack/lapp/apache/instance-apache-php.cfg.in

+[buildout]
+extends = ${custom-application-deployment:path}
+
+parts =
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  promise
+  slapmonitor
+  slapmonitor-xml
+
+  frontend-promise
+  content-promise
+  publish-connection-informations
+  ${custom-application-deployment:part-list}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+httpd-log = $${basedirectory:log}/apache
+php-ini-dir = $${rootdirectory:etc}/php
+tmp-php = $${rootdirectory:tmp}/php
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+xml-report = $${rootdirectory:var}/xml_report
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests
+private = $${directory:ca-dir}/private
+certs = $${directory:ca-dir}/certs
+newcerts = $${directory:ca-dir}/newcerts
+crl = $${directory:ca-dir}/crl
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+
+#----------------
+#--
+#-- Deploy stunnel.
+
+[stunnel]
+recipe = slapos.cookbook:stunnel
+client = true
+stunnel-binary = ${stunnel:location}/bin/stunnel
+remote-host = $${postgres-urlparse:host}
+remote-port = $${postgres-urlparse:port}
+local-host = $${slap-network-information:local-ipv4}
+local-port = 33060
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+wrapper = $${rootdirectory:bin}/raw_stunnel
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+
+#----------------
+#--
+#-- Certificate stuff.
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+
+#----------------
+#--
+#-- Request Postgres instance and parse its URL.
+
+[request-postgres]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Postgres
+software-url = $${slap-connection:software-release-url}
+software-type = postgres
+return = url
+sla = computer_guid
+sla-computer_guid = $${slap-connection:computer-id}
+
+[postgres-urlparse]
+recipe = slapos.cookbook:urlparse
+url = $${request-postgres:connection-url}
+
+
+#----------------
+#--
+#-- Deploy Apache + PHP application.
+#-- Despite the names of mysql-* parameters, they are not really specific to mysql.
+
+[apache-php]
+recipe = slapos.cookbook:apachephp
+source = ${application:location}
+template = ${application-template:location}/${application-template:filename}
+configuration = ${application-configuration:location}
+
+htdocs = $${rootdirectory:srv}/www/
+pid-file = $${basedirectory:run}/apache.pid
+lock-file = $${basedirectory:run}/apache.lock
+ip = $${slap-network-information:global-ipv6}
+port = 8080
+url = http://[$${:ip}]:$${:port}/
+error-log = $${directory:httpd-log}/error.log
+access-log = $${directory:httpd-log}/access.log
+php-ini-dir = $${directory:php-ini-dir}
+tmp-dir = $${directory:tmp-php}
+httpd-conf = $${rootdirectory:etc}/apache.conf
+wrapper = $${basedirectory:services}/apache
+
+httpd-binary = ${apache:location}/bin/httpd
+
+mysql-username = $${postgres-urlparse:username}
+mysql-password = $${postgres-urlparse:password}
+mysql-database = $${postgres-urlparse:path}
+mysql-host = $${stunnel:local-host}
+mysql-port = $${stunnel:local-port}
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache-php:error-log} $${apache-php:access-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+
+
+#----------------
+#--
+#-- Request frontend.
+
+[request-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config = url custom_domain
+config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
+return = site_url
+config-custom_domain = $${slap-parameter:domain}
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${basedirectory:run}/apache.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend_url = $${apache-php:url}
+url = $${request-frontend:connection-site_url}
+
+
+#----------------
+#--
+#-- Deploy promises scripts.
+
+[promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/apache
+hostname = $${apache-php:ip}
+port = $${apache-php:port}
+
+[frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/frontend
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+[content-promise]
+recipe = slapos.cookbook:check_page_content
+path = $${basedirectory:promises}/content
+url = $${request-frontend:connection-site_url}
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+
+
+
+
+[slap-parameter]
+# Default value if no domain is specified
+domain =
+# Default value if no ssh parameter is specified
+logbox-ip =
+logbox-port =
+logbox-user =
+logbox-passwd =
+

stack/lapp/buildout.cfg

+[buildout]
+parts =
+  slapos-cookbook
+  apache-php-postgres
+  rdiff-backup
+  dropbear
+  eggs
+  instance
+  psycopg2
+
+  instance-apache-php
+  instance-postgres
+
+#Contains the importer and exporter recipes for postgres
+  instance-postgres-import
+  instance-postgres-export
+
+  instance-default-root
+
+
+extends =
+  ../resilient/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../component/apache-php/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/stunnel/buildout.cfg
+  ../../component/dropbear/buildout.cfg
+  ../slapos.cfg
+
+
+#----------------
+#-- Application-specific part (maarch, etc.)
+
+[application]
+recipe = hexagonit.recipe.download
+strip-top-level-dir = true
+
+
+#----------------
+#-- Instance-level buildout profiles.
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+md5sum = 1aaf3ea7b14e09e66904bdb80e3cfe2f
+mode = 0644
+
+[instance-apache-php]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
+output = ${buildout:directory}/instance-apache-php.cfg
+md5sum = bed286b680bd8cd494da080cdc229f1e
+mode = 0644
+
+[instance-apache-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg.in
+output = ${buildout:directory}/instance-apache-backup.cfg
+md5sum = 48f969d82319a9d145570f5f0fd27672
+mode = 0644
+
+[template-resilient-lapp]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/template-resilient.cfg.in
+rendered = ${buildout:directory}/instance-resilient.cfg
+
+context = key templateapache instance-apache-php:output
+          key dropbear dropbear:location
+          key buildout buildout:bin-directory
+
+import-list = file parts template-parts:destination
+              file replicated template-replicated:destination
+
+md5sum = 525f50e60d0a96557a552de6afa4ab88
+mode = 0644
+ 
+
+[instance-postgres]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres.cfg.in
+output = ${buildout:directory}/instance-postgres.cfg
+md5sum = c5cd2a644fcd8450bc5d13bf53ec9f7d
+mode = 0644
+
+[instance-postgres-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres-import.cfg.in
+output = ${buildout:directory}/instance-postgres-import.cfg
+md5sum = 1989ba2164dd5f79793a04e0a02ea515
+mode = 0644
+
+[instance-postgres-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/postgres/instance-postgres-export.cfg.in
+output = ${buildout:directory}/instance-postgres-export.cfg
+md5sum = 7bce31bc22a731a8fc6119aee96586f5
+mode = 0644
+
+
+#----------------
+#-- Postgres driver for Python recipes.
+
+[psycopg2-env]
+PATH = ${postgresql:location}/bin:%(PATH)s
+
+
+[psycopg2]
+recipe = zc.recipe.egg:custom
+egg = psycopg2
+define = PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+environment = psycopg2-env
+include-dirs =
+    ${postgresql:location}/include
+library-dirs =
+    ${postgresql:location}/lib
+rpath = 
+    ${postgresql:location}/lib
+
+[instance-default-root]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-default-root.cfg.in
+output = ${buildout:directory}/instance-default-root.cfg
+md5sum = 53c9020f7a0b5203f976e069e455787b
+mode = 0644
+
+#----------------
+#--
+#-- Optional part allowing applications using this stack to run a custom
+#-- instance.cfg at the end of Apache/PHP instance deployment.
+#-- To use it in your application, just override those two parameters, like:
+
+[custom-application-deployment]
+# path = /path/to/instance-custom.cfg
+# part-list = part1 part2
+# See software/maarch/software.cfg for an example.
+path =
+part-list =
+
+#----------------
+#-- Dummy parts in case no application configuration file is needed
+
+[application-template]
+filename =
+location =
+
+[application-configuration]
+location =
+
+#----------------
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${psycopg2:egg}
+  slapos.toolbox
+  cns.recipe.symlink
+
+
+[versions]
+meld3 = 0.6.10
+

stack/lapp/instance-default-root.cfg.in

+[buildout]
+parts = request-apache
+
+[request-apache]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = $${slap-connection:software-release-url}
+software-type = apache
+name = Apache
+

stack/lamp/instance.cfg → stack/lapp/instance.cfg.in

 [buildout]
+
 parts =
   switch_softwaretype
 
@@ -8,10 +9,16 @@ offline = true
 
 [switch_softwaretype]
 recipe = slapos.cookbook:softwaretype
-default = ${template-apache-php:output}
-backuped = ${template-backuped:output}
-mariadb = ${template-mariadb:output}
-mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
-mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
+default = ${instance-default-root:output}
+apache = ${instance-apache-php:output}
+resilient = ${template-resilient-lapp:rendered}
+postgres = ${instance-postgres:output}
+postgres-import = ${instance-postgres-import:output}
+postgres-export = ${instance-postgres-export:output}
+apache-backup = ${instance-apache-backup:output}
+
+#frozen creates a syntax error, meaning it can keep its data.
+#It's dirty as hell, it needs to be replaced.
+frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
-apache-backup = ${template-apache-backup:output}
+

stack/lapp/postgres/instance-postgres-export.cfg.in

+[buildout]
+extends = ${instance-postgres:output}
+          ${pbsready-export:output}
+
+
+parts +=
+  urls
+  postgres-instance
+  postgres-promise
+
+
+[exporter]
+recipe = slapos.cookbook:postgres.export
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
+bin = $${postgres-instance:bin}
+pgdata-directory = $${postgres-instance:pgdata-directory}
+backup-directory = $${postgres-instance:backup-directory}
+dbname = $${postgres-instance:dbname}

stack/lapp/postgres/instance-postgres-import.cfg.in

+[buildout]
+extends = ${instance-postgres:output}
+          ${pbsready-import:output}
+
+
+parts +=
+  postgres-instance
+
+[importer]
+recipe = slapos.cookbook:postgres.import
+wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-importer
+bin = $${postgres-instance:bin}
+pgdata-directory = $${postgres-instance:pgdata-directory}
+backup-directory = $${postgres-instance:backup-directory}
+dbname = $${postgres-instance:dbname}

stack/lapp/postgres/instance-postgres.cfg.in

+[buildout]
+parts =
+  urls
+  postgres-instance
+  postgres-promise
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-stunnel
+  logrotate-entry-cron
+  cron
+  cron-entry-logrotate
+  slapmonitor
+  slapmonitor-xml
+
+gzip-binary = ${gzip:location}/bin/gzip
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+script = $${rootdirectory:etc}/script
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronstamps = $${rootdirectory:etc}/cronstamps
+ca-dir = $${rootdirectory:srv}/ssl
+logrotate-backup = $${basedirectory:backup}/logrotate
+report = $${rootdirectory:etc}/report
+stunnel-conf = $${rootdirectory:etc}/stunnel
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+xml-report = $${rootdirectory:var}/xml_report
+
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${basedirectory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
+[cron-entry-logrotate]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = logrotate
+frequency = 0 0 * * *
+command = $${logrotate:wrapper}
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = $${buildout:gzip-binary}
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${rootdirectory:bin}/logrotate
+conf = $${rootdirectory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${rootdirectory:srv}/logrotate.status
+
+[logrotate-entry-stunnel]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = stunnel
+log = $${stunnel:log-file}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+post = $${stunnel:post-rotate-script}
+
+[logrotate-entry-cron]
+<= logrotate
+recipe =slapos.cookbook:logrotate.d
+name = crond
+log = $${cron-simplelogger:log}
+frequency = daily
+rotate-num = 30
+notifempty = true
+create = true
+
+#----------------
+#--
+#-- Deploy stunnel.
+#-- XXX This is actually not needed with Postgres.
+
+[stunnel]
+recipe = slapos.cookbook:stunnel
+stunnel-binary = ${stunnel:location}/bin/stunnel
+wrapper = $${rootdirectory:bin}/stunnel
+log-file = $${basedirectory:log}/stunnel.log
+config-file = $${directory:stunnel-conf}/stunnel.conf
+key-file = $${directory:stunnel-conf}/stunnel.key
+cert-file = $${directory:stunnel-conf}/stunnel.crt
+pid-file = $${basedirectory:run}/stunnel.pid
+local-host = $${postgres-instance:ipv4_host}
+local-port = $${postgres-instance:port}
+remote-host = $${slap-network-information:global-ipv6}
+remote-port = 6446
+client = false
+post-rotate-script = $${rootdirectory:bin}/stunnel_post_rotate
+
+
+#----------------
+#--
+#-- Certificate stuff.
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+#----------------
+#--
+#-- Creates a Postgres cluster, configuration files, and a database.
+
+[postgres-instance]
+recipe = slapos.cookbook:postgres
+ipv6_host = $${slap-network-information:global-ipv6}
+user = postgres
+port = 5432
+dbname = db
+# XXX the next line is required by stunnel, not by us
+ipv4_host = $${slap-network-information:local-ipv4}
+# pgdata_directory is created by initdb, and should not exist beforehand.
+pgdata-directory = $${rootdirectory:var}/data
+backup-directory = $${basedirectory:backup}/postgres
+services = $${basedirectory:services}
+bin = $${rootdirectory:bin}
+
+dependency-symlinks = $${symlinks:recipe}
+[ca-stunnel]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = $${stunnel:wrapper}
+wrapper = $${basedirectory:services}/stunnel
+key-file = $${stunnel:key-file}
+cert-file = $${stunnel:cert-file}
+
+#----------------
+#--
+#-- Creates symlinks from the instance to the software release.
+
+[symlinks]
+recipe = cns.recipe.symlink
+symlink_target = $${rootdirectory:bin}
+symlink_base = ${postgresql:location}/bin
+
+
+#----------------
+#--
+#-- Deploy slapmonitor.
+
+[slapmonitor]
+recipe = slapos.cookbook:slapmonitor
+pid-file = $${postgres-instance:pgdata-directory}/postmaster.pid
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-path = ${buildout:bin-directory}/slapmonitor
+path = $${basedirectory:services}/slapmonitor
+
+[slapmonitor-xml]
+recipe = slapos.cookbook:slapmonitor-xml
+database-path = $${basedirectory:log}/slapmonitor.db
+slapmonitor-xml-path = ${buildout:bin-directory}/slapmonitor-xml
+path = $${directory:report}/slapmonitor-xml
+
+
+#----------------
+#--
+#-- Deploy promise scripts.
+
+[postgres-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${basedirectory:promises}/postgres
+hostname = $${slap-network-information:global-ipv6}
+port = $${postgres-instance:port}
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[urls]
+recipe = slapos.cookbook:publish
+url = $${postgres-instance:url}
+ip = $${slap-network-information:global-ipv6}
+
+#----------------
+#--
+#-- Fetches parameters defined in SlapOS Master for this instance
+
+[instance-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+

stack/lapp/template-resilient.cfg.in

+{% import 'parts' as parts %}
+{% import 'replicated' as replicated %}
+
+[buildout]
+
+extends =
+   {{templateapache}}
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("postgres","3") }}
+  request-apache-backup-1
+  request-apache-backup-2
+
+  request-pull-backup-server-apache-1
+  request-pull-backup-server-apache-backup-1
+
+  request-pull-backup-server-apache-2
+  request-pull-backup-server-apache-backup-2
+
+  publish-connection-informations
+  apache-php
+  stunnel
+  certificate-authority
+  ca-stunnel
+  logrotate
+  logrotate-entry-apache
+  logrotate-entry-stunnel
+  cron
+  cron-entry-logrotate
+  dropbear-server
+  sshkeys-authority
+  dropbear-server-pbs-authorized-key
+
+  request-pull-backup-server
+
+{{ replicated.replicate("postgres", "3", "postgres-export", "postgres-import") }}
+
+
+[request-pull-backup-server]
+<= request-pbs-common
+name = PBS (Pull Backup Server)
+return = ssh-key notification-url feeds-url
+slave = false
+
+[sshkeys-directory]
+recipe = slapos.cookbook:mkdirectory
+requests = ${directory:sshkeys}/requests
+keys = ${directory:sshkeys}/keys
+
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = ${sshkeys-directory:requests}
+keys-directory = ${sshkeys-directory:keys}
+wrapper = ${basedirectory:services}/sshkeys_authority
+keygen-binary = {{dropbear}}/bin/dropbearkey
+
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = dropbear
+type = rsa
+executable = ${dropbear-server:wrapper}
+public-key = ${dropbear-server:rsa-keyfile}.pub
+private-key = ${dropbear-server:rsa-keyfile}
+wrapper = ${basedirectory:services}/sshd
+
+[dropbear-server]
+recipe = slapos.cookbook:dropbear
+host = ${slap-network-information:global-ipv6}
+port = 2222
+home = ${directory:ssh}
+wrapper = ${rootdirectory:bin}/raw_sshd
+shell = ${rdiff-backup-server:wrapper}
+rsa-keyfile = ${directory:ssh}/server_key.rsa
+dropbear-binary = {{dropbear}}/sbin/dropbear
+
+[dropbear-server-pbs-authorized-key]
+<= dropbear-server
+recipe = slapos.cookbook:dropbear.add_authorized_key
+key = ${request-pull-backup-server:connection-ssh-key}
+
+[rdiff-backup-server]
+<= apache-php
+recipe = slapos.cookbook:pbs
+client = false
+path = ${apache-php:htdocs}
+wrapper = ${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = {{buildout}}/rdiff-backup
+
+[request-apache-backup-1]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 1
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-apache-backup-2]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = Apache Backup 2
+software-url = ${slap-connection:software-release-url}
+software-type = apache-backup
+return = url ssh-url ssh-public-key
+config = authorized-key proxy-url
+config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
+config-proxy-url = ${publish-connection-informations:url}
+
+[request-pull-backup-server-apache-1]
+<= request-pbs-common
+name = PBS pulling from Apache 1
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-2]
+<= request-pbs-common
+name = PBS pulling from Apache 2
+config = url name type server-key notify notification-id frequency
+config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
+config-type = pull
+config-server-key = ${sshkeys-dropbear:public-key-value}
+config-notify = ${request-pull-backup-server:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
+config-frequency = 30 * * * *
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[request-pull-backup-server-apache-backup-1]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-1:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-1:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-1:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+[request-pull-backup-server-apache-backup-2]
+<= request-pbs-common
+name = PBS pushing to ${request-apache-backup-2:name}
+config = url name type server-key on-notification
+config-url = ${request-apache-backup-2:connection-ssh-url}
+config-name = ${request-pull-backup-server-apache-2:config-name}
+config-type = push
+config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
+config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pull-backup-server:instance_guid}
+
+
+[directory]
+ssh = ${rootdirectory:etc}/ssh
+sshkeys = ${rootdirectory:srv}/sshkeys

stack/resilient/buildout.cfg

+[buildout]
+
+parts =
+  pbsready
+  pbsready-import
+  pbsready-export
+  template-replicated
+  template-parts
+  instance-frozen
+  template-resilient
+
+
+#----------------
+#--
+#-- Profiles needed to setup automated backup and recovery.
+#--
+
+[pbsready]
+# Common parts for pbsready-import and pbsready-export.
+# Provides rdiff-backup, notification queue, ssh authentication,
+# dropbear server, and the bully script.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/pbsready.cfg.in
+output = ${buildout:directory}/pbsready.cfg
+md5sum = a2edaadfe652b4b131626b4801768f40
+mode = 0644
+
+[pbsready-import]
+# An import instance has an importer script, which is called
+# by the parent PBS instance when the dump content is propagated.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/pbsready-import.cfg.in
+output = ${buildout:directory}/pbsready-import.cfg
+md5sum = eda0c1574d8991f4f9e08e3707c2b04b
+mode = 0644
+
+[pbsready-export]
+# An export instance has an exporter script, and communicates
+# to parent PBS instances to deliver the exported dump.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/pbsready-export.cfg.in
+output = ${buildout:directory}/pbsready-export.cfg
+md5sum = dd56f9c74e580475a17a9afb1d220390
+mode = 0644
+
+[template-pull-backup]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
+output = ${buildout:directory}/instance-pull-backup.cfg
+md5sum = 18b88cd012e886fbaa457b03928c2d10
+mode = 0644
+
+[template-replicated]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-replicated.cfg.in
+md5sum = 63b5649f3cf1c9a77315382793d9593f
+mode = 0644
+destination = ${buildout:directory}/template-replicated.cfg.in
+
+[template-parts]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/template-parts.cfg.in
+md5sum = c942f82552fcb42fc74a5f896e0cd5f3
+mode = 0644
+destination = ${buildout:directory}/template-parts.cfg.in
+
+[instance-frozen]
+# When an instance is detected as broken, its software type is changed to "frozen".
+# On the next run of slapgrid-cp, the buildout profile is replaced by instance-frozen.cfg,
+# which will run without removing any content because it raises an error.
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-frozen.cfg.in
+output = ${buildout:directory}/instance-frozen.cfg
+
+[template-resilient]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/resilient.cfg.in
+output = ${buildout:directory}/resilient.cfg
+md5sum = 59e74d290d623de2c1e147e48f284fba
+mode = 0644
+

stack/resilient/instance-frozen.cfg.in

+[buildout]
+  parts = 
\ No newline at end of file

stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg.in

 [buildout]
 
 parts =
-    connection-dict
-    pbs
-    cron
-    cron-entry-logrotate
-    logrotate
-    sshkeys-authority
-    sshkeys-dropbear
+  connection-dict
+  pbs
+  logrotate
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  sshkeys-dropbear
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
-[connection-dict]
-recipe = slapos.cookbook:publish
-ssh-key = $${sshkeys-dropbear:public-key-value}
-notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
-feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+home = $${buildout:directory}/home
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+var = $${buildout:directory}/var
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+promises = $${rootdirectory:etc}/promise
+ssh-home = $${rootdirectory:home}/ssh
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+logrotate-entries = $${rootdirectory:etc}/logrotate.d
+logrotate-backup = $${basedirectory:backup}/logrotate
+cronstamps = $${rootdirectory:etc}/cronstamps
+cron-entries = $${rootdirectory:etc}/cron.d
+crontabs = $${rootdirectory:etc}/crontabs
+cronoutput = $${basedirectory:log}/cron-ouput
+pbs-backup = $${basedirectory:backup}/pbs
+sshkeys = $${rootdirectory:srv}/sshkeys
+pbs-wrappers = $${rootdirectory:bin}/pbs
+dot-ssh = $${basedirectory:ssh-home}/.ssh
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
+
+
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
 
 [equeue]
 recipe = slapos.cookbook:equeue
@@ -27,6 +66,8 @@ database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue
 equeue-binary = ${buildout:bin-directory}/equeue
 
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
 [notifier]
 recipe = slapos.cookbook:notifier
 feeds = $${directory:notifier-feeds}
@@ -39,34 +80,19 @@ wrapper = $${basedirectory:services}/notifier
 server-binary = ${buildout:bin-directory}/pubsubserver
 notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
-[dropbear-client]
-recipe = slapos.cookbook:dropbear.client
-dbclient-binary = ${dropbear:location}/bin/dbclient
-wrapper = $${rootdirectory:bin}/ssh
-home = $${basedirectory:ssh-home}
-identity-file = $${basedirectory:ssh-home}/id_rsa
-
-[sshkeys-directory]
-recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
 
-[sshkeys-authority]
-recipe = slapos.cookbook:sshkeys_authority
-request-directory = $${sshkeys-directory:requests}
-keys-directory = $${sshkeys-directory:keys}
-keygen-binary = ${dropbear:location}/bin/dropbearkey
-wrapper = $${basedirectory:services}/sshkeys_authority
 
-[sshkeys-dropbear]
-<= sshkeys-authority
-recipe = slapos.cookbook:sshkeys_authority.request
-name = pbs
-type = rsa
-executable = $${dropbear-client:wrapper}
-public-key = $${dropbear-client:identity-file}.pub
-private-key = $${dropbear-client:identity-file}
-wrapper = $${rootdirectory:bin}/do_backup
+#----------------
+#--
+#-- The pull-backup-server contains every backup (incremental)
+#-- to prevent a corrupt dump from destroying everything.
 
 [pbs]
 <= notifier
@@ -82,10 +108,10 @@ wrappers-directory = $${directory:pbs-wrappers}
 notifier-url = http://[$${notifier:host}]:$${notifier:port}/
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
-[cron-simplelogger]
-recipe = slapos.cookbook:simplelogger
-wrapper = $${rootdirectory:bin}/cron_simplelogger
-log = $${basedirectory:log}/crond.log
+
+#----------------
+#--
+#-- Deploy cron.
 
 [cron]
 recipe = slapos.cookbook:cron
@@ -96,6 +122,16 @@ cronstamps = $${directory:cronstamps}
 catcher = $${cron-simplelogger:wrapper}
 binary = $${basedirectory:services}/crond
 
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${rootdirectory:bin}/cron_simplelogger
+log = $${basedirectory:log}/crond.log
+
+
+#----------------
+#--
+#-- Deploy logrotate.
+
 [cron-entry-logrotate]
 <= cron
 recipe = slapos.cookbook:cron.d
@@ -106,7 +142,7 @@ command = $${logrotate:wrapper}
 [logrotate]
 recipe = slapos.cookbook:logrotate
 # Binaries
-logrotate-binary = ${logrotate:location}/sbin/logrotate
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
 gzip-binary = ${gzip:location}/bin/gzip
 gunzip-binary = ${gzip:location}/bin/gunzip
 # Directories
@@ -116,15 +152,7 @@ logrotate-entries = $${directory:logrotate-entries}
 backup = $${directory:logrotate-backup}
 state-file = $${rootdirectory:srv}/logrotate.status
 
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[logrotate-entry-equeue]
+[logrotate-entry-cron]
 <= logrotate
 recipe = slapos.cookbook:logrotate.d
 name = cron
@@ -133,40 +161,61 @@ frequency = daily
 rotate-num = 30
 
 
-[rootdirectory]
-recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+#----------------
+#--
+#-- sshkeys
 
-[basedirectory]
+[sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-log = $${rootdirectory:var}/log/
-services = $${rootdirectory:etc}/run/
-run = $${rootdirectory:var}/run/
-backup = $${rootdirectory:srv}/backup/
-promises = $${rootdirectory:etc}/promise/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-cronstamps = $${rootdirectory:etc}/cronstamps/
-cron-entries = $${rootdirectory:etc}/cron.d/
-crontabs = $${rootdirectory:etc}/crontabs/
-cronoutput = $${basedirectory:log}/cron-ouput/
-pbs-backup = $${basedirectory:backup}/pbs/
-logrotate-entries = $${rootdirectory:etc}/logrotate.d/
-logrotate-backup = $${basedirectory:backup}/logrotate/
-sshkeys = $${rootdirectory:srv}/sshkeys
-pbs-wrappers = $${rootdirectory:bin}/pbs/
-dot-ssh = $${basedirectory:ssh-home}/.ssh/
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/
+[sshkeys-authority]
+recipe = slapos.cookbook:sshkeys_authority
+request-directory = $${sshkeys-directory:requests}
+keys-directory = $${sshkeys-directory:keys}
+wrapper = $${basedirectory:services}/sshkeys_authority
+keygen-binary = ${dropbear:location}/bin/dropbearkey
 
-[basedirectory]
-ssh-home = $${rootdirectory:home}/ssh
-notifier = $${rootdirectory:etc}/notifier/
+[sshkeys-dropbear]
+<= sshkeys-authority
+recipe = slapos.cookbook:sshkeys_authority.request
+name = pbs
+type = rsa
+executable = $${dropbear-client:wrapper}
+public-key = $${dropbear-client:identity-file}.pub
+private-key = $${dropbear-client:identity-file}
+wrapper = $${rootdirectory:bin}/do_backup
+
+
+#----------------
+#--
+#-- Dropbear.
+
+[dropbear-client]
+recipe = slapos.cookbook:dropbear.client
+dbclient-binary = ${dropbear:location}/bin/dbclient
+wrapper = $${rootdirectory:bin}/ssh
+home = $${basedirectory:ssh-home}
+identity-file = $${basedirectory:ssh-home}/id_rsa
+
+
+#----------------
+#--
+#-- Slave instance list (empty default).
+
+[slap-parameter]
+slave_instance_list = []
+
+
+
+#----------------
+#--
+#-- Publish instance parameters.
+
+[connection-dict]
+recipe = slapos.cookbook:publish
+ssh-key = $${sshkeys-dropbear:public-key-value}
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
 
-[rootdirectory]
-home = $${buildout:directory}/home/

stack/resilient/pbsready-export.cfg.in

+[buildout]
+
+extends = ${pbsready:output}
+
+parts += cron-entry-backup
+
+[urls]
+notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
+
+[notifier-exporter]
+# notifier.notify launches an (exporter) executable, and when finished,
+# notifies the the pull-backup-servers.
+<= notifier
+recipe = slapos.cookbook:notifier.notify
+name = exporter
+title = Dumping $${slap-parameter:namebase}
+executable = $${exporter:wrapper}
+wrapper = $${rootdirectory:bin}/exporter
+notify = $${slap-parameter:notify}
+
+[cron-entry-backup]
+# Schedule the periodic database dump.
+# Through notifications, this triggers (one or more) incremental backups on PBS instances.
+<= cron
+recipe = slapos.cookbook:cron.d
+name = backup
+frequency = 0 * * * *
+command = $${notifier-exporter:wrapper}

stack/resilient/pbsready-import.cfg.in

+[buildout]
+
+extends = ${pbsready:output}
+
+parts += import-on-notification
+
+[urls]
+notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+
+[import-on-notification]
+# notifier.callback runs a script when a notification (sent by a parent PBS)
+# is received
+<= notifier
+recipe = slapos.cookbook:notifier.callback
+on-notification-id = $${slap-parameter:on-notification}
+callback = $${importer:wrapper}

stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/pbsready.cfg.in

 [buildout]
 
-extends = ${template-mariadb:output}
-
-parts =
+parts +=
+  resiliency
   urls
-  mariadb
   stunnel
   certificate-authority
   ca-stunnel
   logrotate
-  logrotate-entry-mariadb
   logrotate-entry-stunnel
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -21,14 +18,99 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+
+#----------------
+#--
+#-- Creation of all needed directories.
+
+[basedirectory]
+services = $${rootdirectory:etc}/run
+cache = $${rootdirectory:var}/cache
+notifier = $${rootdirectory:etc}/notifier
+
+[directory]
+backup = $${basedirectory:backup}/$${slap-parameter:namebase}
+ssh = $${rootdirectory:etc}/ssh/
+sshkeys = $${rootdirectory:srv}/sshkeys
+notifier-feeds = $${basedirectory:notifier}/feeds
+notifier-callbacks = $${basedirectory:notifier}/callbacks
+
+
+#----------------
+#--
+#-- Resiliency script for the bully algorithm
+
+[resiliency]
+# If enable-bully-service is true, the scripts will be run automatically.
+# If false, they can be run with bin/bullly for all the PBSReady instances.
+enable-bully-service = False
+recipe = slapos.cookbook:addresiliency
+wrapper-bully = bully
+wrapper-takeover = takeover
+services = $${basedirectory:services}
+bin = $${rootdirectory:bin}
+etc = $${rootdirectory:etc}
+
+
+#----------------
+#--
+#-- Sets up an rdiff-backup server (with a dropbear server for ssh)
+
 [urls]
 ssh-public-key = $${sshkeys-dropbear:public-key-value}
 ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
 
+[rdiff-backup-server]
+recipe = slapos.cookbook:pbs
+client = false
+path = $${directory:backup}
+wrapper = $${rootdirectory:bin}/rdiffbackup-server
+rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
+
+
+#----------------
+#--
+#-- Set up the equeue and notifier.
+
+[equeue]
+recipe = slapos.cookbook:equeue
+socket = $${basedirectory:run}/equeue.sock
+log = $${basedirectory:log}/equeue.log
+database = $${rootdirectory:srv}/equeue.db
+wrapper = $${basedirectory:services}/equeue
+equeue-binary = ${buildout:bin-directory}/equeue
+
+# notifier.notify adds the [exporter, notifier] to the execution queue
+# notifier.notify.callback sets up a callback
+[notifier]
+recipe = slapos.cookbook:notifier
+feeds = $${directory:notifier-feeds}
+callbacks = $${directory:notifier-callbacks}
+id-file = $${rootdirectory:etc}/notifier.id
+equeue-socket = $${equeue:socket}
+host = $${slap-network-information:global-ipv6}
+port = 8080
+wrapper = $${basedirectory:services}/notifier
+server-binary = ${buildout:bin-directory}/pubsubserver
+notifier-binary = ${buildout:bin-directory}/pubsubnotifier
+
+[logrotate-entry-equeue]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = equeue
+log = $${equeue:log}
+frequency = daily
+rotate-num = 30
+
+
+#----------------
+#--
+#-- sshkeys
+
 [sshkeys-directory]
 recipe = slapos.cookbook:mkdirectory
-requests = $${directory:sshkeys}/requests/
-keys = $${directory:sshkeys}/keys/
+requests = $${directory:sshkeys}/requests
+keys = $${directory:sshkeys}/keys
 
 [sshkeys-authority]
 recipe = slapos.cookbook:sshkeys_authority
@@ -47,6 +129,11 @@ public-key = $${dropbear-server:rsa-keyfile}.pub
 private-key = $${dropbear-server:rsa-keyfile}
 wrapper = $${basedirectory:services}/sshd
 
+
+#----------------
+#--
+#-- Dropbear.
+
 [dropbear-server]
 recipe = slapos.cookbook:dropbear
 host = $${slap-network-information:global-ipv6}
@@ -62,49 +149,5 @@ dropbear-binary = ${dropbear:location}/sbin/dropbear
 recipe = slapos.cookbook:dropbear.add_authorized_key
 key = $${slap-parameter:authorized-key}
 
-[rdiff-backup-server]
-recipe = slapos.cookbook:pbs
-client = false
-path = $${directory:mariadb-backup}
-wrapper = $${rootdirectory:bin}/rdiffbackup-server
-rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
-
-[logrotate-entry-equeue]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = equeue
-log = $${equeue:log}
-frequency = daily
-rotate-num = 30
-
-[equeue]
-recipe = slapos.cookbook:equeue
-socket = $${basedirectory:run}/equeue.sock
-log = $${basedirectory:log}/equeue.log
-database = $${rootdirectory:srv}/equeue.db
-wrapper = $${basedirectory:services}/equeue
-equeue-binary = ${buildout:bin-directory}/equeue
-
-[notifier]
-recipe = slapos.cookbook:notifier
-feeds = $${directory:notifier-feeds}
-callbacks = $${directory:notifier-callbacks}
-id-file = $${rootdirectory:etc}/notifier.id
-equeue-socket = $${equeue:socket}
-host = $${slap-network-information:global-ipv6}
-port = 8080
-wrapper = $${basedirectory:services}/notifier
-server-binary = ${buildout:bin-directory}/pubsubserver
-notifier-binary = ${buildout:bin-directory}/pubsubnotifier
 
 
-[basedirectory]
-cache = $${rootdirectory:var}/cache/
-notifier = $${rootdirectory:etc}/notifier/
-
-[directory]
-mariadb-backup = $${basedirectory:backup}/mariadb/
-ssh = $${rootdirectory:etc}/ssh/
-sshkeys = $${rootdirectory:srv}/sshkeys
-notifier-feeds = $${basedirectory:notifier}/feeds/
-notifier-callbacks = $${basedirectory:notifier}/callbacks/

stack/resilient/resilient.cfg.in

+## not used at the moment
+
+[buildout]
+
+parts =
+  request-pull-backup-server
+
+[request-pull-backup-server]
+<= slap-connection
+recipe = slapos.cookbook:request
+name = PBS (Pull Backup Server)
+software-url = $${slap-connection:software-release-url}
+software-type = pull-backup
+return = ssh-key notification-url feeds-url
+slave = false
\ No newline at end of file

stack/resilient/template-parts.cfg.in

+## Parts Needed for the resiliency
+
+  {% macro replicate(namebase, nbbackup) %}
+
+  request-{{namebase}}
+  request-{{namebase}}-2
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-{{namebase}}-pseudo-replicating-{{i}}
+  request-{{namebase}}-pseudo-replicating-{{i}}-2
+  {% endfor %}
+  
+  {% for i in range(1,nbbackup|int) %}
+  request-pbs-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-{{i}}
+  request-pull-backup-server-{{namebase}}-backup-{{i}}
+  {% endfor %}
+  
+
+
+  {% endmacro %}

stack/resilient/template-replicated.cfg.in

+{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
+
+## Tells the Backupable recipe that we want a backup
+[resilient]
+config-namebase = {{namebase}}
+
+## Every request is double to provide the 3 IPs.
+[request-{{namebase}}]
+<= resilient
+   slap-connection
+   {{heriteLeader}}
+software-type = {{typeexport}}
+name = {{namebase}}0
+return = url ssh-public-key ssh-url notification-id ip
+
+config = number authorized-key notify ip-list namebase
+config-number = 0
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+config-ip-list =
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-{{namebase}}-pseudo-replicating-{{id}}]
+<= slap-connection
+   resilient
+   {{heriteBackup}}
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url ip
+
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+config = number authorized-key on-notification ip-list namebase
+config-number = {{id}}
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+config-ip-list =
+
+{% endfor %}
+
+[iplist]
+config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
+
+[request-{{namebase}}-2]
+<= resilient
+   slap-connection
+   iplist
+   {{heriteLeader}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}0
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeexport}}
+return = url ssh-public-key ssh-url notification-id ip
+
+config = number authorized-key notify ip-list namebase
+config-number = 0
+config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
+config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
+
+{% for id in range(1,nbbackup|int) %}
+[request-{{namebase}}-pseudo-replicating-{{id}}-2]
+<= slap-connection
+   resilient
+   iplist
+   {{heriteBackup}}
+
+recipe = slapos.cookbook:request
+name = {{namebase}}{{id}}
+
+software-url = ${slap-connection:software-release-url}
+software-type = {{typeimport}}
+return = url ssh-public-key ssh-url notification-url
+
+pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
+
+config = number authorized-key on-notification ip-list namebase
+config-number = {{id}}
+config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
+
+{% endfor %}
+
+
+
+## The PBS and their push / pull slaves
+## Adding a PBS provides resiliency
+## Adding a backup server provides availability
+
+## Having 3 backups pulling from the same PBS provides
+##only availability, not resiliency
+
+## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
+
+[request-pbs-common]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = pull-backup
+
+{% for id in range(1,nbbackup|int) %}
+
+[request-pbs-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS ({{namebase}} / {{id}})
+return = ssh-key notification-url feeds-url
+slave = false
+
+[request-pull-backup-server-{{namebase}}-{{id}}]
+<= request-pbs-common
+name = PBS {{id}} pulling from ${request-{{namebase}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}:connection-ssh-url}
+config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
+config-type = pull
+config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
+config-on-notification = ${request-{{namebase}}:connection-notification-id}
+config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
+config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
+config-title = Pulling from {{namebase}}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
+
+[request-pull-backup-server-{{namebase}}-backup-{{id}}]
+<= request-pbs-common
+name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
+config = url name type server-key on-notification notify notification-id title
+config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
+config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
+config-type = push
+config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
+config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
+config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
+config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+config-title = Pushing to {{namebase}} backup {{id}}
+slave = true
+sla = instance_guid
+sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
+{% endfor %}
+
+{% endmacro %}

stack/slapos.cfg

@@ -25,6 +25,7 @@ find-links +=
 
 # Use only quite well working sites.
 allow-hosts +=
+  *.googlecode.com
   *.nexedi.org
   *.python.org
   *.sourceforge.net
@@ -35,7 +36,6 @@ allow-hosts +=
   github.com
   launchpad.net
   peak.telecommunity.com
-  psutil.googlecode.com
   sourceforge.net
   www.dabeaz.com
   www.owlfish.com
@@ -52,15 +52,25 @@ versions = versions
 networkcache-section = networkcache
 
 # Install slapos.cookbook containing all officials recipes
+# Explicitely define dependencies as well, because of buildout limitation
+# if using "develop"
 [slapos-cookbook]
 recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   slapos.cookbook
+  inotifyx
+  netaddr
+  slapos.core
+  xml_marshaller
+  pytz
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-007
+zc.buildout = 1.6.0-dev-SlapOS-010
+
+# zc.recipe.egg 2.0.0a is buggy
+zc.recipe.egg = 1.3.2
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache