Merge branch 'openstack' into grid-computing

CHANGES.txt

 Changes
 =======
 
-0.78.4.dev (2013-07-18)
------------------------
+0.84.2 (2013-10-04)
+-------------------
+
+ * sshkeys_authority: don't allow to return None as parameter. [9e340a0]
+
+0.84.1 (2013-10-03)
+-------------------
+
+ * Resiliency: PBS: promise should NOT bang. [64886cd]
+
+0.84 (2013-09-30)
+-----------------
+
+ * Request.py: improve instance-state handling. [ba5f160]
+ * Resilient recipe: remove hashing of urls/names. [ee2aec8]
+ * Resilient pbs recipe: recover from rdiff-backup failures. [be7f2fc, 92ee0c3]
+ * Resilience: add pidfiles in PBS. [0b3ad5c]
+ * Resilient: don't hide exception, print it. [05b3d64, d2b0494]
+ * Resiliency: Only keep 10 increments of backup. [4e89e33]
+ * KVM SR: add fallback in case of download exception. [de8d796]
+ * slaprunner: don't check certificate for importer. [53dc772]
+
+0.83.1 (2013-09-10)
+------------------
+
+ * slapconfiguration: fixes previous releasei (don't encode tap_set because it's not a string). [Cedric de Saint Martin]
+
+0.83 (2013-09-10)
+-----------------
+
+ * slaprunner recipe: remove trailing / from master_url. [Cedric de Saint Martin]
+ * librecipe: add pidfile option for singletons. [Cedric de Saint Martin]
+ * Resiliency: Use new pidfile option. [Cedric de Saint Martin]
+ * Fix request.py for slave instances. [Cedric de Saint Martin]
+ * slapconfiguration recipe: cast some parameters from unicode to str. [Cedric de Saint Martin]
+
+0.82 (2013-08-30)
+-----------------
+
+ * Certificate Authority: Can receice certificate to install. [Cedric Le Ninivin]
+ * Squid: Add squid recipe. [Romain Courteaud]
+ * Request: Trasmit instace state to requested instances. [Benjamin Blanc / Cédric Le Ninivin]
+ * Slapconfiguration: Now return instance state. [Cédric Le Ninivin]
+ * Apache Frontend: Remove recipe
+
+0.81 (2013-08-12)
+-----------------
+
+ * KVM SR: implement resiliency test. [Cedric de Saint Martin]
+
+0.80 (2013-08-06)
+----------------
+
+ * Add a simple readline recipe. [f4fce7e]
+
+0.79 (2013-08-06)
+-----------------
+
+ * KVM SR: Add support for NAT based networking (User Mode Network). [627895fe35]
+ * KVM SR: add virtual-hard-drive-url support. [aeb5df40cd, 8ce5a9aa1d0, a5034801aa9]
+ * Fix regression in GenericBaseRecipe.generatePassword. [3333b07d33c]
+
+0.78.5 (2013-08-06)
+-------------------
+
+ * check_url_available: add option to check secure links [6cbce4d8231]
+
+0.78.4 (2013-08-06)
+-------------------
 
+ * slapos.cookbook:slaprunner: Update to use https. [Cedric Le Ninivin]
 
 
 0.78.3 (2013-07-18)

component/cloud9/buildout.cfg

@@ -44,6 +44,13 @@ filename = cloud9-session-directory.patch
 download-only = true
 md5sum = 5dc8cc28447ed3747b8a53c768d872aa
 
+[cloud9-socket.patch]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:filename}
+filename = cloud9-socket.patch
+download-only = true
+#md5sum = 5dc8cc28447ed3747b8a53c768d872aa
+
 [cloud9-git]
 # Online IDE written in javascript/node.js
 # URL : c9.io
@@ -55,7 +62,7 @@ commit = f7d102bc225c922f116d2cea52a746d64343ea59
 repository = https://github.com/ajaxorg/cloud9.git
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
-command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1)
+command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-socket.patch:location}/${cloud9-socket.patch:filename}) || (rm -fr ${:location}; exit 1)
 update-command =
 executable = ${:location}/server.js
 

component/cloud9/cloud9-socket.patch

+diff --git a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+index fa7e54a..14b8e67 100644
+--- a/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
++++ b/node_modules/smith.io/node_modules/engine.io/node_modules/engine.io-client/dist/engine.io-dev.js
+@@ -2126,7 +2126,7 @@ Polling.prototype.uri = function () {
+     query = '?' + query;
+   }
+ 
+-  return schema + '://' + this.host + port + this.path + query;
++  return this.path + query;
+ };
+ 
+ });require.register("transports/websocket.js", function(module, exports, require, global){

component/coreutils/buildout.cfg

@@ -19,7 +19,11 @@ environment =
 recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
-command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm}
+command = ${:test} -x ${:test} -a -x ${:cat} -a -x ${:rm} -a -x ${:echo} -a -x ${:date} -a -x ${:md5sum} -a -x ${:basename}
 test = ${coreutils:location}/bin/test
 cat = ${coreutils:location}/bin/cat
 rm = ${coreutils:location}/bin/rm
+echo = ${coreutils:location}/bin/echo
+date = ${coreutils:location}/bin/date
+md5sum = ${coreutils:location}/bin/md5sum
+basename = ${coreutils:location}/bin/basename

component/faketime/buildout.cfg

+#  faketime - report faked system time to programs without having to change the system-wide time
+#  http://www.code-wizards.com/projects/libfaketime
+
+[buildout]
+parts = faketime
+
+[faketime]
+recipe = slapos.recipe.cmmi
+url = http://www.code-wizards.com/projects/libfaketime/libfaketime-0.9.1.tar.gz
+md5sum = ce3f996dfd5826b4ac62f1a7cc36ea27
+configure-command = true
+make-options =
+  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+make-binary = make -e -C src
+make-targets = install
+post-install = sed -i -e "16c\FTPL_PATH=${buildout:parts-directory}/${:_buildout_section_name_}/lib/faketime" ${buildout:parts-directory}/${:_buildout_section_name_}/bin/faketime

component/gnutls/buildout.cfg

 [buildout]
 extends =
-  ../readline/buildout.cfg
+  ../gmp/buildout.cfg
+  ../nettle/buildout.cfg
   ../ncurses/buildout.cfg
+  ../readline/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = gnutls
@@ -22,14 +24,13 @@ environment =
   LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
 
 [gnutls]
-# XXX-Cedric : update to latest gnutls
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2
-md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
+md5sum = e0cba4ddd923420026ff9739b3bc069a
 configure-options =
   --with-libgcrypt-prefix=${gcrypt:location}
   --disable-static
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
-  LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
-  PKG_CONFIG=${zlib:location}/lib/pkgconfig
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
+  LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
+

component/groonga/buildout.cfg

@@ -11,9 +11,9 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-version = 3.0.5
+version = 3.0.9
 url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = 2894bbdd2275cb3c62aea14446dc2561
+md5sum = 2e9f7090f40876233c1f077fd25c26ee
 configure-options =
   --disable-static
   --disable-glibtest

component/gtk-2/buildout.cfg

@@ -22,12 +22,12 @@ depends =
   ${libpng:so_version}
 configure-options = 
   --enable-tee=yes
-  --enable-xlib=no
+  --enable-xlib=yes
 environment = 
   PATH=${freetype:location}/bin:${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig
   CPPFLAGS=-I${libpng:location}/include/ -I${zlib:location}/include -I${libX11:location}/include/ -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${render:location}/include
-  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libXrender:location}/lib -Wl,-rpath=${libXrender:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libXrender:location}/lib -Wl,-rpath=${libXrender:location}/lib -L${libX11:location}/lib
   LD_LIBRARY_PATH=${render:location}/lib:${libX11:location}/lib:${libXrender:location}/lib
 
 [pango]

component/libpng/buildout.cfg

@@ -28,6 +28,6 @@ md5sum = fd85af68f84cbdf549147811006488c1
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.2.tar.xz
-md5sum = 9d838f6fca9948a9f360a0cc1b516d5f
+url = http://download.sourceforge.net/libpng/libpng-1.6.6.tar.xz
+md5sum = 3a41dcd58bcac7cc191c2ec80c7fb2ac
 so_version = 16

component/make/buildout.cfg

+[buildout]
+extends =
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+
+[make]
+# make 3.82 breaks too many things. Stick with 3.81.
+# See http://lists.gnu.org/archive/html/make-alpha/2010-07/msg00025.html
+# for all incompatible changes.
+# Moreover, vanilla 3.81 does some seg faults, so use Debian patched version.
+<= make3.81-debian
+
+[make-dfsg_3.81-8.2.diff]
+# Debian patch coming from:
+# http://ftp.de.debian.org/debian/pool/main/m/make-dfsg/make-dfsg_3.81-8.2.diff.gz
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/${:_buildout_section_name_}
+md5sum = fa77bb989a096fafbe7c78582e9415e3
+download-only = true
+
+[make3.81-debian]
+recipe = slapos.recipe.cmmi
+url = http://ftp.de.debian.org/debian/pool/main/m/make-dfsg/make-dfsg_3.81.orig.tar.gz
+md5sum = 7c93b1ab4680eb21c2c13f4f47741e2d
+patches =
+  ${make-dfsg_3.81-8.2.diff:location}/make-dfsg_3.81-8.2.diff
+patch-options = -p1

component/mariadb/buildout.cfg

@@ -25,10 +25,10 @@ download-only = true
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 5.5.32
+version = 5.5.33a
 revision = 1
 url = http://downloads.askmonty.org/f/mariadb-${:version}/kvm-tarbake-jaunty-x86/mariadb-${:version}.tar.gz/from/http://ftp.osuosl.org/pub/mariadb
-md5sum = 565c2dce6a2fb027c9d0ffbae4934135
+md5sum = 00449a034b88490f16bd679b800bb850
 # compile directory is required to build mysql plugins.
 keep-compile-dir = true
 patch-options = -p0
@@ -56,14 +56,14 @@ environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
   CMAKE_INCLUDE_PATH=${libaio:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${zlib:location}/include
   CMAKE_LIBRARY_PATH=${libaio:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${libaio:location}/lib
+  LDFLAGS=-L${libaio:location}/lib -L${zlib:location}/lib
 
 [mroonga-mariadb]
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-3.05.tar.gz
-md5sum = ba4cbd79274d832b9343a0b2fe7d0787
+url = http://packages.groonga.org/source/mroonga/mroonga-3.09.tar.gz
+md5sum = bdcd1d86185a64520881d33e12b7d7a7
 configure-options =
   --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
   --with-mysql-config=${mariadb:location}/bin/mysql_config

component/nano/buildout.cfg

+[buildout]
+parts =
+  nano
+  
+extends = 
+  ../ncurses/buildout.cfg
+
+[nano]
+recipe = slapos.recipe.cmmi
+version = 2.2.6
+url = http://www.nano-editor.org/dist/v2.2/nano-2.2.6.tar.gz
+md5sum = 03233ae480689a008eb98feb1b599807
+environment=
+    CFLAGS=-I${ncurses:location}/include
+    LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
\ No newline at end of file

component/nettle/buildout.cfg

+[buildout]
+extends =
+  ../gmp/buildout.cfg
+  ../m4/buildout.cfg
+
+[nettle-lib-location.patch]
+recipe = hexagonit.recipe.download
+download-only = true
+filename = ${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 41dd0ce2a73487929bdc637b75dd62c9
+
+[nettle]
+recipe = slapos.recipe.cmmi
+url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
+md5sum = 003d5147911317931dd453520eb234a5
+patches =
+  ${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
+configure-option =
+  --disable-static
+  --disable-assembler
+  --disable-openssl
+environment =
+  PATH=${m4:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gmp:location}/include
+  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/nettle/nettle-lib-location.patch

+--- configure.orig	2013-07-05 15:37:28.000000000 +0200
++++ configure	2013-07-05 15:47:48.000000000 +0200
+@@ -4680,52 +4680,6 @@
+ if test "x$ABI" != xstandard ; then
+   { $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
+ $as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
+-  if test "$libdir" = '${exec_prefix}/lib' ; then
+-    # Try setting a better default
+-    case "$host_cpu:$host_os:$ABI" in
+-      *:solaris*:32|*:sunos*:32)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *:solaris*:64|*:sunos*:64)
+-	libdir='${exec_prefix}/lib/64'
+-	;;
+-      # Linux conventions are a mess... According to the Linux File
+-      # Hierarchy Standard, all architectures except IA64 puts 32-bit
+-      # libraries in lib, and 64-bit in lib64. Some distributions,
+-      # e.g., Fedora and Gentoo, adhere to this standard, while at
+-      # least Debian has decided to put 64-bit libraries in lib and
+-      # 32-bit libraries in lib32.
+-
+-      # We try to figure out the convention, except if we're cross
+-      # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
+-      # appears to not be a symlink to a different name.
+-      *:linux*:32|*:linux*:64)
+-	if test "$cross_compiling" = yes ; then
+-          { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
+-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; 	else
+-	  # The dash builtin pwd tries to be "helpful" and remember
+-	  # symlink names. Use -P option, and hope it's portable enough.
+-	  test -d /usr/lib${ABI} \
+-	    && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
+-	    && libdir='${exec_prefix}/'"lib${ABI}"
+-	fi
+-	;;
+-      # On freebsd, it seems 32-bit libraries are in lib32,
+-      # and 64-bit in lib. Don't know about "kfreebsd", does
+-      # it follow the Linux fhs conventions?
+-      *:freebsd*:32)
+-	libdir='${exec_prefix}/lib32'
+-	;;
+-      *:freebsd*:64)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *)
+-        { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
+-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
+-    esac
+-    { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
+-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
+-  fi
+ fi
+ 
+ # Select assembler code

component/nginx/buildout.cfg

@@ -3,11 +3,14 @@ extends =
   ../pcre/buildout.cfg
   ../zlib/buildout.cfg
   ../openssl/buildout.cfg
+  ../coreutils/buildout.cfg
+
+parts = nginx-output
 
 [nginx]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.2.7.tar.gz
-md5sum = d252f5c689a14a668e241c744ccf5f06
+url = http://nginx.org/download/nginx-1.5.3.tar.gz
+md5sum = 1e735dd6a6ade2b5c20e924b67c3d355
 configure-options=
   --with-ipv6
   --with-http_ssl_module
@@ -16,6 +19,15 @@ configure-options=
   --with-ld-opt="-L ${zlib:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib  -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib"
   --with-cc-opt="-I ${pcre:location}/include -I ${openssl:location}/include -I ${zlib:location}/include"
 
+[nginx-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:nginx} -a -f ${:mime}
+nginx = ${nginx:location}/sbin/nginx
+mime = ${nginx:location}/conf/mime.types
+
 [nginx-unstable]
 <= nginx
 url = http://nginx.org/download/nginx-1.3.15.tar.gz

component/openssl/buildout.cfg

@@ -8,9 +8,10 @@ extends =
   ../ca-certificates/buildout.cfg
   ../zlib/buildout.cfg
   ../patch/buildout.cfg
+  ../coreutils/buildout.cfg
 
 parts =
-  openssl
+  openssl-output
 
 [openssl-nodoc.patch]
 # Disable doc generation part in Makefile 
@@ -52,3 +53,11 @@ make-options =
   -j1
 make-targets =
   all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
+
+[openssl-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:openssl}
+openssl = ${openssl:location}/bin/openssl

component/percona-toolkit/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 recipe = slapos.recipe.cmmi
 depends =
   ${perl:version}
-version = 2.1.9
+version = 2.2.5
 url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/percona-toolkit-${:version}.tar.gz
-md5sum = 94545d0fe6a4893dcad8a3411531107d
+md5sum = 66165271fc3ddf8311e5ff3b1a954595
 configure-command =
   ${perl:location}/bin/perl Makefile.PL

component/python-2.7/buildout.cfg

@@ -10,6 +10,7 @@ extends =
   ../sqlite3/buildout.cfg
   ../zlib/buildout.cfg
   ../file/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 parts =
     python2.7
@@ -25,9 +26,9 @@ python = python2.7
 
 [python2.7]
 recipe = slapos.recipe.cmmi
-package_version = 2.7.5
-package_version_suffix =
-md5sum = 6334b666b7ff2038c761d7b27ba699c1
+package_version = 2.7.6
+package_version_suffix = rc1
+md5sum = 74e1f6abe529350ac0d239387ee98616
 
 depends =
   ${gdbm:version}
@@ -38,7 +39,7 @@ version = 2.7
 executable = ${:prefix}/bin/python${:version}
 
 url =
-  http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.bz2
+  http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =
   --enable-ipv6
   --enable-unicode=ucs4
@@ -51,5 +52,6 @@ make-targets = make profile-opt && make install
 # the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
 # which would otherwise load the system libmagic.so with ctypes
 environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include  -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
   LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib

component/python-openssl/buildout.cfg

@@ -8,6 +8,8 @@ parts =
 [python-openssl]
 recipe = zc.recipe.egg:custom
 egg = pyOpenSSL
+include-dirs =
+  ${openssl:location}/include/
 library-dirs =
   ${openssl:location}/lib/
 rpath =

component/qemu-kvm/buildout.cfg

@@ -3,22 +3,26 @@ extends =
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
+  ../../component/pkgconfig/buildout.cfg
   ../../component/xorg/buildout.cfg
   ../../component/zlib/buildout.cfg
 
-[kvm]
-# Backward compatibility
-<= qemu-kvm
+# XXX Change all reference to kvm section to qemu section, then
+# use qemu as main name section.
+[qemu]
+<= kvm
 
-[qemu-kvm]
+[kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.4.1.tar.bz2
-md5sum = eb2d696956324722b5ecfa46e41f9a75
+url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
+md5sum = 3a897d722457c5a895cd6ac79a28fda0
 depends =
   ${libpng:so_version}
 configure-options =
-  --target-list=""
+  --target-list=x86_64-softmmu
+  --enable-system
+  --with-system-pixman
   --disable-sdl
   --disable-xen
   --enable-vnc-tls
@@ -36,4 +40,26 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
+  LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
+
+# The following is only available in buildout2, which we don't use yet.
+[kvm-bits64]
+configure-options =
+  --target-list=x86_64-softmmu
+  ${kvm:configure-options}
+
+[kvm-bits32]
+configure-options =
+  --target-list=i386-softmmu
+  ${kvm:configure-options}
+
 
+[debian-amd64-netinst.iso]
+# Download the installer of Debian 7 (Wheezy)
+recipe = hexagonit.recipe.download
+url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso 
+filename = ${:_buildout_section_name_}
+md5sum = b86774fe4de88be6378ba3d71b8029bd 
+download-only = true
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}

component/slapos/buildout.cfg

@@ -17,6 +17,7 @@ extends =
   ../pkgconfig/buildout.cfg
   ../popt/buildout.cfg
   ../python-2.7/buildout.cfg
+  ../python-openssl/buildout.cfg
   ../readline/buildout.cfg
   ../sqlite3/buildout.cfg
   ../swig/buildout.cfg
@@ -90,10 +91,14 @@ output = ${buildout:directory}/environment.sh
 [lxml-python]
 python = python2.7
 
+[python-openssl]
+python = python2.7
+
 [slapos]
 recipe = z3c.recipe.scripts
 python = python2.7
 eggs =
+  ${python-openssl:egg}
   slapos.libnetworkcache
   zc.buildout
   ${lxml-python:egg}
@@ -131,46 +136,64 @@ scripts = py
 
 [versions]
 # Use our own buildout version
-zc.buildout = 1.6.0-dev-SlapOS-010
+zc.buildout = 1.6.0-dev-SlapOS-012
 
 # Force to use zc.recipe.egg 1.x
 zc.recipe.egg = 1.3.2
 
 # Use own version of h.r.download to be able to open archives not supported by python2.x: .xz
-hexagonit.recipe.download = 1.6nxd002
+hexagonit.recipe.download = 1.7nxd002
 
-Jinja2 = 2.7
+Jinja2 = 2.7.1
 MarkupSafe = 0.18
-Werkzeug = 0.8.3
+Pygments = 1.6
+Werkzeug = 0.9.4
 buildout-versions = 1.7
+cmd2 = 0.6.7
 collective.recipe.template = 1.10
-lxml = 3.1.2
+itsdangerous = 0.23
+lxml = 3.2.3
 meld3 = 0.6.10
 netaddr = 0.7.10
+prettytable = 0.7.2
+pyOpenSSL = 0.13.1
+pyparsing = 2.0.1
+setuptools = 1.1.6
+slapos.core = 1.0.0rc6
 slapos.libnetworkcache = 0.13.4
+slapos.recipe.cmmi = 0.1.1
 xml-marshaller = 0.9.7
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.35.2-dev
-Flask = 0.9
+# slapos.core==1.0.0rc6
+Flask = 0.10.1
 
 # Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
+bpython = 0.12
+
+# Required by:
+# slapos.core==1.0.0rc6
+cliff = 1.4.5
+
+# Required by:
+# slapos.core==1.0.0rc6
+ipython = 1.1.0
+
+# Required by:
+# slapos.core==1.0.0rc6
 netifaces = 0.8
 
 # Required by:
-# slapos.core==0.35.2-dev
-# slapos.libnetworkcache==0.13.3
-# supervisor==3.0b1
-# zc.buildout==1.6.0-dev-SlapOS-010
-# zope.interface==4.0.5
-setuptools = 0.6c12dev-r88846
+# slapos.core==1.0.0rc6
+requests = 2.0.0
 
 # Required by:
-# slapos.core==0.35.2-dev
-supervisor = 3.0b2
+# slapos.core==1.0.0rc6
+supervisor = 3.0
 
 # Required by:
-# slapos.core==0.35.2-dev
+# slapos.core==1.0.0rc6
 zope.interface = 4.0.5
+

component/slapos/testing.cfg

+# This file is used to install testing, not-stable-yet, version of SlapOS Node
+[buildout]
+extends =
+  buildout.cfg
+
+# Add hosting location of testing version of slapos.core
+find-links +=
+  http://www.nexedi.org/static/packages/source/slapos.core-testing/
+
+[versions]
+slapos.core =

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2013/sqlite-autoconf-3071700.tar.gz
-md5sum = 18c285053e9562b848209cb0ee16d4ab
+url = http://www.sqlite.org/2013/sqlite-autoconf-3080100.tar.gz
+md5sum = 8b5a0a02dfcb0c7daf90856a5cfd485a
 configure-options =
   --disable-static
   --enable-readline

component/squid/buildout.cfg

+# Squid: Optimising Web Delivery
+# http://squid-cache.org
+
+[buildout]
+parts =
+  squid
+extends =
+  ../pkgconfig/buildout.cfg
+
+[squid]
+recipe = hexagonit.recipe.cmmi
+url = http://www.squid-cache.org/Versions/v3/3.2/squid-3.2.1.tar.gz
+md5sum = 3fb81acc6b70a432e3f0d8a0491056dc
+configure-options =
+  --disable-dependency-tracking
+  --disable-translation
+  --disable-htcp
+  --disable-snmp
+  --disable-loadable-modules
+  --disable-icmp
+  --disable-esi
+  --disable-icap-client
+  --disable-wccp
+  --disable-wccpv2
+  --disable-eui
+  --enable-http-violations
+  --disable-ipfw-transparent 
+  --disable-ipf-transparent
+  --disable-pf-transparent
+  --disable-linux-netfilter
+  --enable-follow-x-forwarded-for
+  --disable-auth
+  --disable-url-rewrite-helpers
+  --disable-auto-locale
+  --disable-kerberos
+  --enable-x-accelerator-vary
+  --disable-external-acl-helpers
+  --disable-auth-ntlm
+  --with-krb5-config=no
+Environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s

component/texinfo/buildout.cfg

+[buildout]
+extends =
+  ../ncurses/buildout.cfg
+
+[texinfo]
+# Most other components are not happy with texinfo 5, because it treats some
+# used-to-be-warnings as errors.
+<= texinfo4
+
+[texinfo4]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/texinfo/texinfo-4.13.tar.gz
+md5sum = 71ba711519209b5fb583fed2b3d86fcb
+configure-options =
+  --disable-static
+environment =
+  CFLAGS=-I${ncurses:location}/include
+  LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib

component/zeromq/buildout.cfg

+[buildout]
+extends =
+  ../libtool/buildout.cfg
+  ../libuuid/buildout.cfg
+
+[zeromq]
+<= zeromq3
+
+[zeromq3]
+recipe = slapos.recipe.cmmi
+url = http://download.zeromq.org/zeromq-3.2.3.tar.gz
+md5sum = 1abf8246363249baf5931a065ee38203
+configure-options = --without-documentation
+environment =
+  PATH=${libtool:location}/bin:%(PATH)s
+  LDFLAGS=-L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib
+
+[zeromq2]
+recipe = slapos.recipe.cmmi
+url = http://download.zeromq.org/zeromq-2.2.0.tar.gz
+md5sum = 1b11aae09b19d18276d0717b2ea288f6
+configure-options =
+  --without-documentation
+environment =
+  PATH=${libtool:location}/bin:%(PATH)s
+  CXXFLAGS=-I${libuuid:location}/include
+  LDFLAGS=-L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.78.4.dev'
+version = '0.84.2'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -70,7 +70,6 @@ setup(name=name,
         'zc.buildout': [
           'addresiliency = slapos.recipe.addresiliency:Recipe',
           'agent = slapos.recipe.agent:Recipe',
-          'apache.frontend = slapos.recipe.apache_frontend:Recipe',
           'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
           'apacheperl = slapos.recipe.apacheperl:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
@@ -166,8 +165,7 @@ setup(name=name,
           'publish.serialised = slapos.recipe.publish:Serialised',
           'publishsection = slapos.recipe.publish:PublishSection',
           'publishurl = slapos.recipe.publishurl:Recipe',
-          'pwgen = slapos.recipe.pwgen:Recipe',
-          'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
+          'readline = slapos.recipe.readline:Recipe',
           'redis.server = slapos.recipe.redis:Recipe',
           'request = slapos.recipe.request:Recipe',
           'request.serialised = slapos.recipe.request:Serialised',
@@ -193,6 +191,7 @@ setup(name=name,
           'slaprunner.import = slapos.recipe.slaprunner.backup:ImportRecipe',
           'softwaretype = slapos.recipe.softwaretype:Recipe',
           'sphinx= slapos.recipe.sphinx:Recipe',
+          'squid = slapos.recipe.squid:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
           'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
           'stunnel = slapos.recipe.stunnel:Recipe',

slapos/recipe/addresiliency/renamer.py

-# -*- coding: utf-8 -*-
-
-import logging
-import time
-
-import slapos
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-
-class Renamer(object):
-    def __init__(self, server_url, key_file, cert_file, computer_guid,
-                 partition_id, software_release, namebase):
-        self.server_url = server_url
-        self.key_file = key_file
-        self.cert_file = cert_file
-        self.computer_guid = computer_guid
-        self.partition_id = partition_id
-        self.software_release = software_release
-        self.namebase = namebase
-
-
-    def _failover(self):
-        """\
-        This method does
-
-        - retrieve the broken computer partition
-        - change its reference to 'broken-...' and its software type to 'frozen'
-        - retrieve the winner computer partition (attached to this process)
-        - change its reference to replace the broken one.
-          later, slapgrid will change its software_type as well.
-
-        Then, after running slapgrid-cp a few times, the winner takes over and
-        a new cp is created to replace it as an importer.
-        """
-
-        slap = slapos.slap.slap()
-        slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
-
-        # partition that will take over.
-        cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
-                                                   partition_id=self.partition_id)
-        # XXX although we can already rename cp_winner, to change its software type we need to
-        # get hold of the root cp as well
-
-        cp_exporter_ref = self.namebase + '0'       # this is ok. the boss is always number zero.
-
-        # partition to be deactivated
-        cp_broken = cp_winner.request(software_release=self.software_release,
-                                      software_type='frozen',
-                                      state='stopped',
-                                      partition_reference=cp_exporter_ref)
-
-        broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
-
-        log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
-
-        cp_broken.rename(new_name=broken_new_ref)
-
-        cp_broken.stopped()
-
-        log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
-
-        # update name (and later, software type) for the partition that will take over
-
-        cp_winner.rename(new_name=cp_exporter_ref)
-        cp_winner.bang(message='partitions have been renamed!')
-
-
-
-    def failover(self):
-        try:
-            self._failover()
-            log.info('Renaming done')
-        except slapos.slap.ServerError:
-            log.info('Internal server error')
-

slapos/recipe/addresiliency/takeover.py

 # -*- coding: utf-8 -*-
+import logging
+import time
+import traceback
 
-import slapos.recipe.addresiliency.renamer
+import slapos
+from slapos.slap.slap import NotFoundError
+
+log = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+def takeover(server_url, key_file, cert_file, computer_guid,
+             partition_id, software_release, namebase,
+             winner_instance_suffix = None):
+  """
+  This function does
+
+  - retrieve the broken computer partition
+  - change its reference to 'broken-...' and its software type to 'frozen'
+  - retrieve the winner computer partition (attached to this process)
+  - change its reference to replace the broken one.
+    later, slapgrid will change its software_type as well.
+
+  Then, after running slapgrid-cp a few times, the winner takes over and
+  a new cp is created to replace it as an importer.
+  """
+
+  slap = slapos.slap.slap()
+  slap.initializeConnection(server_url, key_file, cert_file)
+  current_partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                     partition_id=partition_id)
+
+  # partition that will take over.
+  if winner_instance_suffix:
+    winner_instance_name = namebase + winner_instance_suffix
+    # XXX: we hardcode a lot of values here, because request is a settergetter, all at once.
+    cp_winner = current_partition.request(software_release=software_release,
+                                  software_type='%s-import' % namebase,
+                                  partition_reference=winner_instance_name)
+  else:
+    # This script is run in the winning partition: use this one as winner
+    cp_winner = current_partition
+  # XXX although we can already rename cp_winner, to change its software type we need to
+  # get hold of the root cp as well
+
+  cp_exporter_ref = namebase + '0'       # this is ok. the boss is always number zero.
+
+  # partition to be deactivated
+  cp_broken = cp_winner.request(software_release=software_release,
+                                software_type='frozen',
+                                state='stopped',
+                                partition_reference=cp_exporter_ref)
+
+  broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
+
+  log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
+
+  cp_broken.rename(new_name=broken_new_ref)
+
+  log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
+
+  # update name (and later, software type) for the partition that will take over
+  while True:
+    time.sleep(10)
+    try:
+      cp_winner.rename(new_name=cp_exporter_ref)
+      break
+    except NotFoundError:
+      traceback.print_exc()
+      log.warning('Impossible to rename. Retrying in a few seconds...')
+  log.debug('Renamed.')
+
+  cp_winner.bang(message='partitions have been renamed!')
+  # Note: Root instance will reconfigure itself the winning instance (software_type
+  # and parameters.)
 
 def run(args):
-    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
+  slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
                                                 key_file = args.pop('key_file'),
                                                 cert_file = args.pop('cert_file'),
                                                 computer_guid = args.pop('computer_id'),
@@ -11,6 +83,3 @@ def run(args):
                                                 software_release = args.pop('software'),
                                                 namebase = args.pop('namebase'))
 
-    renamer.failover()
-
-

slapos/recipe/apache_frontend/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-import uuid
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions', 'v3_ca',
-          '-keyout', self.key, '-out', self.certificate, '-days', '10950'],
-          # Authority name will be random, so no instance has the same issuer
-          'Certificate Authority %s\n' % uuid.uuid1())
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/apache_frontend/template/apache.conf.path-protected.in

-<Directory %(path)s>
-  Order Deny,Allow
-  Allow from %(access_control_string)s
-</Directory>
-
-<Directory %(document_root)s>
-  Order Allow,Deny
-  Allow from All
-</Directory>

slapos/recipe/apache_frontend/template/apache.location-snippet.conf.in

-<Location %(location)s>
-  Order Deny,Allow
-  Deny from all
-  Allow from %(allow_string)s
-</Location>

slapos/recipe/apache_frontend/template/apache.ssl-snippet.conf.in

-SSLCertificateFile %(login_certificate)s
-SSLCertificateKeyFile %(login_key)s
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLSessionCache shmcb:/%(httpd_mod_ssl_cache_directory)s/ssl_scache(512000)
-SSLSessionCacheTimeout  300
-SSLRandomSeed startup /dev/urandom 256
-SSLRandomSeed connect builtin
-SSLProtocol -ALL +SSLv3 +TLSv1
-SSLHonorCipherOrder On
-SSLCipherSuite RC4-SHA:HIGH:!ADH
-<FilesMatch "\.(cgi|shtml|phtml|php)$">
-      SSLOptions +StdEnvVars
-</FilesMatch>
-# Accept proxy to sites using self-signed SSL certificates
-SSLProxyCheckPeerCN off
-SSLProxyCheckPeerExpire off

slapos/recipe/apache_frontend/template/logrotate_entry.in

-%(file_list)s {
-  daily
-  dateext
-  rotate 30
-  compress
-  notifempty
-  sharedscripts
-  create
-  postrotate
-    %(postrotate)s
-  endscript
-  olddir %(olddir)s
-}

slapos/recipe/apache_frontend/template/stunnel.conf.entry.in

-[%(name)s]
-accept = %(public_ip)s:%(public_port)s
-connect = %(private_ip)s:%(private_port)s

slapos/recipe/apache_frontend/template/stunnel.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-client = yes
-CApath = %(ca_path)s
-key = %(key)s
-CRLpath = %(ca_crl)s
-cert = %(cert)s
-sslVersion = SSLv3
-socket = l:TCP_NODELAY=1
-socket = r:TCP_NODELAY=1
-
-%(entry_str)s

slapos/recipe/apache_frontend/template/varnish.vcl.in

-# This is a basic VCL configuration file for varnish.  See the vcl(7)
-# man page for details on VCL syntax and semantics.
-# 
-# Default backend definition.  Set this to point to your content
-# server.
-# 
-backend default {
-    .host = "%(backend_host)s";
-    .port = "%(backend_port)s";
-    .probe = { 
-        .url = "/";
-        .timeout = 10s;
-        .interval = 10s;
-        .window = 4;
-        .threshold = 3;
-    }
-}
-# 
-# Below is a commented-out copy of the default VCL logic.  If you
-# redefine any of these subroutines, the built-in logic will be
-# appended to your code.
-# 
-# sub vcl_recv {
-#     if (req.http.x-forwarded-for) {
-#   set req.http.X-Forwarded-For =
-#       req.http.X-Forwarded-For ", " client.ip;
-#     } else {
-#   set req.http.X-Forwarded-For = client.ip;
-#     }
-#     if (req.request != "GET" &&
-#       req.request != "HEAD" &&
-#       req.request != "PUT" &&
-#       req.request != "POST" &&
-#       req.request != "TRACE" &&
-#       req.request != "OPTIONS" &&
-#       req.request != "DELETE") {
-#         /* Non-RFC2616 or CONNECT which is weird. */
-#         return (pipe);
-#     }
-#     if (req.request != "GET" && req.request != "HEAD") {
-#         /* We only deal with GET and HEAD by default */
-#         return (pass);
-#     }
-#     if (req.http.Authorization || req.http.Cookie) {
-#         /* Not cacheable by default */
-#         return (pass);
-#     }
-#     return (lookup);
-# }
-sub vcl_recv {
-    if (req.http.cache-control ~ "no-cache") {
-        purge_url(req.url);
-    }
-
-    if (req.url ~ "\.(css|js|ico)$") {
-        unset req.http.cookie;
-    }
-
-    # remove bogus cookies
-    if (req.http.Cookie) {
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__utm.=[^;]+;? *", "\1");
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac_name=\x22\x22;? *", "\1");
-        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *__ac=\x22Og.3D.3D\x22;? *", "\1");
-    }
-    if (req.http.Cookie == "") {
-        remove req.http.Cookie;
-    }
-
-    if (req.http.x-forwarded-for) {
-  set req.http.X-Forwarded-For =
-      req.http.X-Forwarded-For ", " client.ip;
-    } else {
-  set req.http.X-Forwarded-For = client.ip;
-    }
-    if (req.request != "GET" &&
-      req.request != "HEAD" &&
-      req.request != "PUT" &&
-      req.request != "POST" &&
-      req.request != "TRACE" &&
-      req.request != "OPTIONS" &&
-      req.request != "DELETE") {
-        /* Non-RFC2616 or CONNECT which is weird. */
-        return (pipe);
-    }
-    if (req.request != "GET" && req.request != "HEAD") {
-        /* We only deal with GET and HEAD by default */
-        return (pass);
-    }
-    if (req.http.Authorization) {
-        /* Not cacheable by default */
-        return (pass);
-    }
-    if (req.http.Cookie && req.http.Cookie ~ "(^|; ) *__ac=") {
-        /* Not cacheable for authorised users,
-     but KM images are cacheable */
-  if (!(req.url ~ "/km_img/.*\.(png|gif)$")) {
-    return (pass);
-  }
-    }
-    # XXX login form can defer based on __ac_name cookie value
-    if (req.url ~ "/(login_form|WebSite_viewLoginDialog)($|\?)") {
-        return (pass);
-    }
-    if (req.backend.healthy) {
-        set req.grace = 1h;
-    } else {
-        set req.grace = 1w;
-    }
-    return (lookup);
-}
-# 
-# sub vcl_pipe {
-#     # Note that only the first request to the backend will have
-#     # X-Forwarded-For set.  If you use X-Forwarded-For and want to
-#     # have it set for all requests, make sure to have:
-#     # set req.http.connection = "close";
-#     # here.  It is not set by default as it might break some broken web
-#     # applications, like IIS with NTLM authentication.
-#     return (pipe);
-# }
-# 
-# sub vcl_pass {
-#     return (pass);
-# }
-# 
-# sub vcl_hash {
-#     set req.hash += req.url;
-#     if (req.http.host) {
-#         set req.hash += req.http.host;
-#     } else {
-#         set req.hash += server.ip;
-#     }
-#     return (hash);
-# }
-# 
-# sub vcl_hit {
-#     if (!obj.cacheable) {
-#         return (pass);
-#     }
-#     return (deliver);
-# }
-# 
-# sub vcl_miss {
-#     return (fetch);
-# }
-# 
-# sub vcl_fetch {
-#     if (!beresp.cacheable) {
-#         return (pass);
-#     }
-#     if (beresp.http.Set-Cookie) {
-#         return (pass);
-#     }
-#     return (deliver);
-# }
-sub vcl_fetch {
-    # we only cache 200 (OK) and 304 (Not Modified) responses.
-    if (beresp.status != 200 && beresp.status != 304) {
-        set beresp.cacheable = false;
-    }
-
-    if (beresp.http.cache-control ~ "no-cache") {
-        set beresp.cacheable = false;
-    }
-
-    if (!beresp.cacheable) {
-        unset beresp.http.expires;
-        set beresp.http.cache-control = "no-cache";
-        return (pass);
-    }
-
-    # we don't care haproxy's cookie.
-    if (beresp.http.Set-Cookie && beresp.http.Set-Cookie !~ "^SERVERID=[^;]+; path=/$") {
-        return (pass);
-    }
-
-    if (req.url ~ "\.(css|js|ico)$") {
-        unset beresp.http.set-cookie;
-        set beresp.http.cache-control = regsub(beresp.http.cache-control, "^", "public,");
-        set beresp.http.cache-control = regsub(beresp.http.cache-control, ",$", "");
-    }
-
-    # remove some headers added by caching policy manager to avoid
-    # '304 Not Modified' in case of login <-> logout switching.
-    if (beresp.http.content-type ~ "^text/html") {
-  unset beresp.http.last-modified;
-    }
-
-    if (beresp.cacheable) {
-        /* Remove Expires from backend, it's not long enough */
-        unset beresp.http.expires;
-        /* Set the clients TTL on this object */
-        set beresp.http.cache-control = "max-age = 900";
-        /* Set how long Varnish will keep it */
-        set beresp.ttl = 1w;
-        /* marker for vcl_deliver to reset Age: */
-        set beresp.http.magicmarker = "1";
-    }
-
-    set beresp.grace = 1w;
-
-    return (deliver);
-}
-# 
-# sub vcl_deliver {
-#     return (deliver);
-# }
-sub vcl_deliver {
-    if (resp.http.magicmarker) {
-        /* Remove the magic marker */
-        unset resp.http.magicmarker;
-        /* By definition we have a fresh object */
-        set resp.http.age = "0";
-    }
-    if (obj.hits > 0) {
-        set resp.http.X-Cache = obj.hits;
-    } else {
-      set resp.http.X-Cache = "MISS";
-    }
-    return (deliver);
-}
-
-# 
-# sub vcl_error {
-#     set obj.http.Content-Type = "text/html; charset=utf-8";
-#     synthetic {"
-# <?xml version="1.0" encoding="utf-8"?>
-# <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-#  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-# <html>
-#   <head>
-#     <title>"} obj.status " " obj.response {"</title>
-#   </head>
-#   <body>
-#     <h1>Error "} obj.status " " obj.response {"</h1>
-#     <p>"} obj.response {"</p>
-#     <h3>Guru Meditation:</h3>
-#     <p>XID: "} req.xid {"</p>
-#     <hr>
-#     <p>Varnish cache server</p>
-#   </body>
-# </html>
-# "};
-#     return (deliver);
-# }

slapos/recipe/certificate_authority/__init__.py

@@ -103,10 +103,21 @@ class Request(Recipe):
     key_file = self.options['key-file']
     cert_file = self.options['cert-file']
 
+    key_content = self.options.get('key-content', None)
+    cert_content = self.options.get('cert-content', None)
+    request_needed = True
+
     name = self.options['name']
     hash_ = hashlib.sha512(name).hexdigest()
     key = os.path.join(self.ca_private, hash_ + self.ca_key_ext)
     certificate = os.path.join(self.ca_certs, hash_ + self.ca_crt_ext)
+
+    # XXX Ugly hack to quickly provide custom certificate/key to everyone using the recipe
+    if key_content and cert_content:
+      open(key, 'w').write(key_content)
+      open(certificate, 'w').write(cert_content)
+      request_needed = False
+    else:
       parser = ConfigParser.RawConfigParser()
       parser.add_section('certificate')
       parser.set('certificate', 'name', name)
@@ -123,11 +134,14 @@ class Request(Recipe):
     os.symlink(key, key_file)
     os.symlink(certificate, cert_file)
 
+    path_list = [key_file, cert_file]
+    if request_needed:
       wrapper = self.createPythonScript(
         self.options['wrapper'],
         'slapos.recipe.librecipe.execute.execute_wait',
         [ [self.options['executable']],
           [certificate, key] ],
       )
+      path_list.append(wrapper)
 
-    return [key_file, cert_file, wrapper]
+    return path_list

slapos/recipe/check_url_available/__init__.py

@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
       'url': self.options['url'],
       'shell_path': self.options['dash_path'],
       'curl_path': self.options['curl_path'],
+      'check_secure': self.options.get('check-secure', 0)
     }
 
     # XXX-Cedric in this script, curl won't check certificate

slapos/recipe/check_url_available/template/check_url.in

@@ -31,6 +31,13 @@ if [ $CODE -eq 000 ]; then
   exit 1
 fi
 
+if [ %(check_secure)s -eq 1 ]; then
+  if [ $CODE -eq 401 ]; then
+    echo "$URL is protected (returned $CODE)." >&2
+    exit 0
+  fi
+fi
+
 if ! [ $CODE -eq 200 ]; then
   echo "$URL is not available (returned $CODE)." >&2
   exit 2

slapos/recipe/dcron.py

@@ -26,7 +26,11 @@
 ##############################################################################
 import os
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+if __name__ == '__main__': # Hack to easily run test below.
+  GenericBaseRecipe = object
+else:
+  from slapos.recipe.librecipe import GenericBaseRecipe
+  from zc.buildout import UserError
 
 class Recipe(GenericBaseRecipe):
 
@@ -51,18 +55,117 @@ class Recipe(GenericBaseRecipe):
     return [script]
 
 
-
 class Part(GenericBaseRecipe):
 
   def install(self):
+    try:
+      periodicity = self.options['frequency']
+    except KeyError:
+      periodicity = self.options['time']
+      try:
+        periodicity = systemd_to_cron(periodicity)
+      except Exception:
+        raise UserError("Invalid systemd calendar spec %r" % periodicity)
     cron_d = self.options['cron-entries']
     name = self.options['name']
     filename = os.path.join(cron_d, name)
 
     with open(filename, 'w') as part:
-      part.write('%(frequency)s %(command)s\n' % {
-        'frequency': self.options['frequency'],
-        'command': self.options['command'],
-      })
+      part.write('%s %s\n' % (periodicity, self.options['command']))
 
     return [filename]
+
+
+day_of_week_dict = dict((name, dow) for dow, name in enumerate(
+    "sunday monday tuesday wednesday thursday friday saturday".split())
+  for name in (name, name[:3]))
+symbolic_dict = dict(hourly  = '0 * * * *',
+                     daily   = '0 0 * * *',
+                     monthly = '0 0 1 * *',
+                     weekly  = '0 0 * * 0')
+
+def systemd_to_cron(spec):
+  """Convert from systemd.time(7) calendar spec to crontab spec"""
+  try:
+    return symbolic_dict[spec]
+  except KeyError:
+    pass
+  if not spec.strip():
+    raise ValueError
+  spec = spec.split(' ')
+  try:
+    dow = ','.join(sorted('-'.join(str(day_of_week_dict[x.lower()])
+                                   for x in x.split('-', 1))
+                          for x in spec[0].split(',')
+                          if x))
+    del spec[0]
+  except KeyError:
+    dow = '*'
+  day = spec.pop(0) if spec else '*-*'
+  if spec:
+    time, = spec
+  elif ':' in day:
+    time = day
+    day = '*-*'
+  else:
+    time = '0:0'
+  day = day.split('-')
+  time = time.split(':')
+  if (# years not supported
+      len(day) > 2 and day.pop(0) != '*' or
+      # some crons ignore day of month if day of week is given, and dcron
+      # treats day of month in a way that is not compatible with systemd
+      dow != '*' != day[1] or
+      # seconds not supported
+      len(time) > 2 and int(time.pop())):
+    raise ValueError
+  month, day = day
+  hour, minute = time
+  spec = minute, hour, day, month, dow
+  for x, (y, z) in zip(spec, ((0, 60), (0, 24), (1, 31), (1, 12))):
+    if x != '*':
+      for x in x.split(','):
+        x = map(int, x.split('/', 1))
+        x[0] -= y
+        if x[0] < 0 or len(x) > 1 and x[0] >= x[1] or z <= sum(x):
+          raise ValueError
+  return ' '.join(spec)
+
+def test(self):
+  def _(systemd, cron):
+    self.assertEqual(systemd_to_cron(systemd), cron)
+  _("Sat,Mon-Thu,Sun", "0 0 * * 0,1-4,6")
+  _("mon,sun *-* 2,1:23", "23 2,1 * * 0,1")
+  _("Wed, 17:48", "48 17 * * 3")
+  _("Wed-Sat,Tue 10-* 1:2", "2 1 * 10 2,3-6")
+  _("*-*-7 0:0:0", "0 0 7 * *")
+  _("10-15", "0 0 15 10 *")
+  _("monday *-12-* 17:00", "00 17 * 12 1")
+  _("12,14,13,12:20,10,30", "20,10,30 12,14,13,12 * * *") # TODO: sort
+  _("*-1/2-1,3 *:30", "30 * 1,3 1/2 *")
+  _("03-05 08:05", "05 08 05 03 *")
+  _("08:05:00", "05 08 * * *")
+  _("05:40", "40 05 * * *")
+  _("Sat,Sun 12-* 08:05", "05 08 * 12 0,6")
+  _("Sat,Sun 08:05", "05 08 * * 0,6")
+
+  def _(systemd):
+    self.assertRaises(Exception, systemd_to_cron, systemd)
+  _("test")
+  _("")
+  _("7")
+  _("121212:1:2")
+
+  _("Wed *-1")
+  _("08:05:40")
+  _("2003-03-05")
+
+  _("0-1"); _("13-1"); _("6/4-1"); _("5/8-1")
+  _("1-0"); _("1-32"); _("1-4/3"); _("1-14/18")
+  _("24:0");_("9/9:0"); _("8/16:0")
+  _("0:60"); _("0:22/22"); _("0:15/45")
+
+if __name__ == '__main__':
+  import unittest
+  unittest.TextTestRunner().run(type('', (unittest.TestCase,), {
+    'runTest': test})())

slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in

@@ -12,7 +12,8 @@ erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
 mysql_url = "%(sql_connection_string)s"
 
 header_dict = {'Authorization': 'Basic %%s' %% \
-  base64.encodestring('%%s:%%s' %% (user, password)).strip()}
+  base64.encodestring('%%s:%%s' %% (user, password)).strip(),
+               'Referer':'http://%%s/manage_addProduct/ERP5/addERP5Site' %% host}
 zope_connection = httplib.HTTPConnection(host)
 
 # Check if an ERP5 site is already created, as ERP5 does support having

slapos/recipe/erp5_update/erp5.py

@@ -50,6 +50,7 @@ class ERP5Updater(object):
     base64string = base64.encodestring(authentication_string).strip()
 
     self.header_dict['Authorization'] = 'Basic %s' % base64string
+    self.header_dict['Referer'] = 'http://%s/manage_addProduct/ERP5/addERP5Site' % host
 
     self.host = host
     self.site_id = site_id

slapos/recipe/generatepassword.py

@@ -26,27 +26,69 @@
 #
 ##############################################################################
 
-import binascii
+import errno
 import os
+import random
+import string
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+def generatePassword(length):
+  return ''.join(random.SystemRandom().sample(string.ascii_lowercase, length))
 
 
-class Recipe(GenericBaseRecipe):
+class Recipe(object):
+  """Generate a password that is only composed of lowercase letters
+
+    This recipe only makes sure that ${:passwd} does not end up in `.installed`
+    file, which is world-readable by default. So be careful not to spread it
+    throughout the buildout configuration by referencing it directly: see
+    recipes like slapos.recipe.template:jinja2 to safely process the password.
+
+    Options:
+    - bytes: password length (default: 8 characters)
+    - storage-path: plain-text persistent storage for password,
+                    that can only be accessed by the user
+      (default: ${buildout:parts-directory}/${:_buildout_section_name_})
+  """
 
   def __init__(self, buildout, name, options):
-    if os.path.exists(options['storage-path']):
-      open_file = open(options['storage-path'], 'r')
-      options['passwd'] = open_file.read()
-      open_file.close()
+    options_get = options.get
+    try:
+      self.storage_path = options['storage-path']
+    except KeyError:
+      self.storage_path = options['storage-path'] = os.path.join(
+        buildout['buildout']['parts-directory'], name)
+    try:
+      with open(self.storage_path) as f:
+        passwd = f.read()
+    except IOError, e:
+      if e.errno != errno.ENOENT:
+        raise
+      passwd = None
+    if not passwd:
+      passwd = self.generatePassword(int(options_get('bytes', '8')))
+      self.update = self.install
+    self.passwd = passwd
+    # Password must not go into .installed file, for 2 reasons:
+    # security of course but also to prevent buildout to always reinstall.
+    options.get = lambda option, *args, **kw: passwd \
+      if option == 'passwd' else options_get(option, *args, **kw)
 
-    if options.get('passwd', '') == '':
-      options['passwd'] = binascii.hexlify(os.urandom(
-                            int(options.get('bytes', '24'))))
-    return GenericBaseRecipe.__init__(self, buildout, name, options)
+  generatePassword = staticmethod(generatePassword)
 
   def install(self):
-    with open(self.options['storage-path'], 'w') as fout:
-      fout.write(self.options['passwd'])
-    return [self.options['storage-path']]
+    if self.storage_path:
+      try:
+        os.unlink(self.storage_path)
+      except OSError, e:
+        if e.errno != errno.ENOENT:
+          raise
+      fd = os.open(self.storage_path,
+        os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
+      try:
+        os.write(fd, self.passwd)
+      finally:
+        os.close(fd)
+    return self.storage_path
 
+  def update(self):
+    return ()

slapos/recipe/kvm/__init__.py

@@ -41,39 +41,44 @@ class Recipe(GenericBaseRecipe):
           '"virtio" value.'
       self.options['disk-type'] = 'virtio'
 
-    config = dict(
-      tap_interface=self.options['tap'],
-      vnc_ip=self.options['vnc-ip'],
-      vnc_port=self.options['vnc-port'],
-      nbd_ip=self.options['nbd-host'],
-      nbd_port=self.options['nbd-port'],
-      nbd2_ip=self.options.get('nbd2-host', ''),
-      nbd2_port=self.options.get('nbd2-port', 1024),
-      disk_path=self.options['disk-path'],
-      disk_size=self.options['disk-size'],
-      disk_type=self.options['disk-type'],
-      mac_address=self.options['mac-address'],
-      smp_count=self.options['smp-count'],
-      ram_size=self.options['ram-size'],
-      socket_path=self.options['socket-path'],
-      pid_file_path=self.options['pid-path'],
-      python_path=sys.executable,
-      shell_path=self.options['shell-path'],
-      qemu_path=self.options['qemu-path'],
-      qemu_img_path=self.options['qemu-img-path'],
-      vnc_passwd=self.options['passwd']
+    self.options['python-path'] = sys.executable
+
+    path_list = []
+
+    if not self.isTrueValue(self.options.get('use-tap')):
+      # XXX This could be done using Jinja.
+      for item in self.options['nat-rules'].split():
+        ports = item.split(':')
+        if len(ports) == 1:
+          tunnel_port = int(ports[0]) + 10000
+        else:
+          tunnel_port = int(ports[1])
+        tunnel_path = self.createExecutable(
+            '%s-%s' % (self.options['6tunnel-wrapper-path'], tunnel_port),
+            self.substituteTemplate(
+                self.getTemplateFilename('6to4.in'),
+                {
+                    'ipv6': self.options['ipv6'],
+                    'ipv6_port': tunnel_port,
+                    'ipv4': self.options['ipv4'],
+                    'ipv4_port': tunnel_port,
+                    'shell_path': self.options['shell-path'],
+                    '6tunnel_path': self.options['6tunnel-path'],
+                },
+            ),
         )
+        path_list.append(tunnel_path)
 
-    # Runners
     runner_path = self.createExecutable(
         self.options['runner-path'],
         self.substituteTemplate(self.getTemplateFilename('kvm_run.in'),
-                              config))
+                                self.options))
+    path_list.append(runner_path)
 
     controller_path = self.createExecutable(
       self.options['controller-path'],
       self.substituteTemplate(self.getTemplateFilename('kvm_controller_run.in'),
-                              config))
+                              self.options))
 
 
-    return [runner_path, controller_path]
+    return path_list

slapos/recipe/kvm/template/6to4.in

+#!%(shell_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(6tunnel_path)s -6 -4 -d -l %(ipv6)s %(ipv6_port)s %(ipv4)s %(ipv4_port)s
+

slapos/recipe/kvm/template/kvm_controller_run.in

-#!%(python_path)s
+#!%(python-path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
@@ -6,12 +6,17 @@
 import socket
 import time
 
+# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
+
+socket_path = '%(socket-path)s'
+vnc_password = '%(vnc-passwd)s'
+
 # Connect to KVM qmp socket
 so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 connected = False
 while not connected:
   try:
-    so.connect('%(socket_path)s')
+    so.connect(socket_path)
   except socket.error:
     time.sleep(1)
   else:
@@ -25,7 +30,7 @@ data = so.recv(1024)
 # Set VNC password
 so.send('{ "execute": "change", ' \
         '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "%(vnc_passwd)s" } }')
+        '               "arg": "' + vnc_password + '" } }')
 data = so.recv(1024)
 
 # Finish

slapos/recipe/kvm/template/kvm_run.in

-#!%(python_path)s
+#!%(python-path)s
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
 
-# Echo client program
+import hashlib
 import os
 import socket
 import subprocess
+import urllib
+
+# XXX: give all of this through parameter, don't use this as template, but as module
+qemu_img_path = '%(qemu-img-path)s'
+qemu_path = '%(qemu-path)s'
+disk_size = '%(disk-size)s'
+disk_type = '%(disk-type)s'
+socket_path = '%(socket-path)s'
+nbd_list = (('%(nbd-host)s', %(nbd-port)s), ('%(nbd2-host)s', %(nbd2-port)s))
+default_disk_image = '%(default-disk-image)s'
+disk_path = '%(disk-path)s'
+virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
+virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
+nat_rules = '%(nat-rules)s'.strip()
+use_tap = '%(use-tap)s'
+tap_interface = '%(tap-interface)s'
+listen_ip = '%(ipv4)s'
+mac_address = '%(mac-address)s'
+smp_count = '%(smp-count)s'
+ram_size = '%(ram-size)s'
+pid_file_path = '%(pid-file-path)s'
+
+def md5Checksum(file_path):
+    with open(file_path, 'rb') as fh:
+        m = hashlib.md5()
+        while True:
+            data = fh.read(8192)
+            if not data:
+                break
+            m.update(data)
+        return m.hexdigest()
 
 def getSocketStatus(host, port):
   s = None
@@ -26,27 +57,62 @@ def getSocketStatus(host, port):
     break
   return s
 
-# create disk if doesn't exist
-disk_path = '%(disk_path)s'
+# Download existing hard drive if needed at first boot
+if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
+  print('Downloading virtual hard drive...')
+  try:
+    urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+  except:
+    os.remove(disk_path)
+    raise
+  md5sum = virtual_hard_drive_md5sum.strip()
+  if md5sum:
+    print('Checking MD5 checksum...')
+    local_md5sum = md5Checksum(disk_path)
+    if local_md5sum != md5sum:
+      os.remove(disk_path)
+      raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
+          local_md5sum, md5sum))
+    print('MD5sum check passed.')
+  else:
+    print('Warning: not checksum specified.')
+
+# Create disk if doesn't exist
+# XXX: move to Buildout profile
 if not os.path.exists(disk_path):
-  subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
-      disk_path, '%(disk_size)sG'])
-
-kvm_argument_list = ['%(qemu_path)s',
-  '-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s',
-  '-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no',
-  '-smp', '%(smp_count)s',
-  '-m', '%(ram_size)s',
-  '-drive', 'file=%(disk_path)s,if=%(disk_type)s',
-  '-vnc', '%(vnc_ip)s:1,ipv4,password',
+  print('Creating virtual hard drive...')
+  subprocess.Popen([qemu_img_path, 'create' ,'-f', 'qcow2',
+      disk_path, '%%sG' %% disk_size])
+  print('Done.')
+
+# Generate network parameters
+# XXX: use_tap should be a boolean
+if use_tap == 'True':
+  qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
+else:
+  qemu_network_parameter = 'user'
+  for item in nat_rules.split():
+    ports = item.split(':')
+    if len(ports) == 1:
+      ports.append(int(ports[0]) + 10000)
+    qemu_network_parameter += ',hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, ports[1], ports[0])
+
+kvm_argument_list = [qemu_path,
+  '-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
+  '-net', qemu_network_parameter,
+  '-smp', smp_count,
+  '-m', ram_size,
+  '-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
+  '-vnc', '%%s:1,ipv4,password' %% listen_ip,
   '-boot', 'menu=on',
-  '-qmp', 'unix:%(socket_path)s,server',
-  '-pidfile', '%(pid_file_path)s',
+  '-qmp', 'unix:%%s,server' %% socket_path,
+  '-pidfile', pid_file_path,
 ]
 
-# Try to connect to NBD server (and second nbd if defined)
-for nbd_ip, nbd_port in (
-    ('%(nbd_ip)s', %(nbd_port)s), ('%(nbd2_ip)s', %(nbd2_port)s)):
+# Try to connect to NBD server (and second nbd if defined).
+# If not available, don't even specify it in qemu command line parameters.
+# Reason: if qemu starts with unavailable NBD drive, it will just crash.
+for nbd_ip, nbd_port in nbd_list:
   if nbd_ip and nbd_port:
     s = getSocketStatus(nbd_ip, nbd_port)
     if s is None:
@@ -57,5 +123,10 @@ for nbd_ip, nbd_port in (
       kvm_argument_list.extend([
           '-drive',
           'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
+# If no NBD is specified/available: use internal disk image
+else:
+  kvm_argument_list.extend([
+      '-drive', 'file=%%s,media=cdrom' %% default_disk_image
+  ])
 
-os.execv('%(qemu_path)s', kvm_argument_list)
+os.execv(qemu_path, kvm_argument_list)

slapos/recipe/librecipe/generic.py

@@ -33,6 +33,7 @@ import sys
 import inspect
 import re
 import shutil
+from textwrap import dedent
 import urllib
 import urlparse
 
@@ -129,10 +130,14 @@ class GenericBaseRecipe(object):
     return script
 
   def createWrapper(self, name, command, parameters, comments=[],
-        parameters_extra=False, environment=None):
+      parameters_extra=False, environment=None,
+      pidfile=None
+  ):
     """
     Creates a very simple (one command) shell script for process replacement.
     Takes care of quoting.
+    if pidfile parameter is specified, then it will make the wrapper a singleton,
+    accepting to run only if no other instance is running.
     """
 
     lines = [ '#!/bin/sh' ]
@@ -144,6 +149,21 @@ class GenericBaseRecipe(object):
       for key in environment:
         lines.append('export %s=%s' % (key, environment[key]))
 
+    if pidfile:
+      lines.append(dedent("""\
+          # Check for other instances
+          pidfile=%s
+          if [ -e $pidfile ]; then
+            pid=$(cat $pidfile)
+            if [[ ! -z $(ps -p "$pid" | grep $(basename %s)) ]]; then
+              echo "Already running with pid $pid."
+              exit 1
+            else
+              rm $pidfile
+            fi
+          fi
+          echo $$ > $pidfile""" % (pidfile, command)))
+
     lines.append('exec %s' % shlex.quote(command))
 
     for param in parameters:
@@ -183,17 +203,13 @@ class GenericBaseRecipe(object):
         'template/%s' % template_name)
 
   def generatePassword(self, len_=32):
-    """
-    The purpose of this method is to generate a password which doesn't change
-    from one execution to the next, so the generated password doesn't change
-    on each slapgrid-cp execution.
-
-    Currently, it returns a hardcoded password because no decision has been
-    taken on where a generated password should be kept (so it is generated
-    once only).
-    """
-    # TODO: implement a real password generator which remember the last
-    # call.
+    # TODO: Consider having generate.password recipe inherit this class,
+    #       so that it can be easily inheritable.
+    #       In the long-term, it's probably better that passwords are provided
+    #       by software requesters, to avoid keeping unhashed secrets in
+    #       partitions when possible.
+    self.logger.warning("GenericBaseRecipe.generatePassword is deprecated."
+                        " Use generate.password recipe instead.")
     return "insecure"
 
   def isTrueValue(self, value):
@@ -247,7 +263,8 @@ class GenericBaseRecipe(object):
       destination = self.location
     if os.path.exists(destination):
         # leftovers from a previous failed attempt, removing it.
-        log.warning('Removing already existing directory %s' % destination)
+        self.logger.warning('Removing already existing directory %s',
+                            destination)
         shutil.rmtree(destination)
     os.mkdir(destination)
 

slapos/recipe/notifier.py

@@ -50,6 +50,9 @@ class Recipe(GenericBaseRecipe):
 class Callback(GenericBaseRecipe):
 
   def createCallback(self, notification_id, callback):
+    # XXX: hashing the name here and in
+    # slapos.toolbox/slapos/pubsub/__init__.py is completely messed up and
+    # prevent any debug.
     callback_id = sha512(notification_id).hexdigest()
 
     filepath = os.path.join(self.options['callbacks'], callback_id)
@@ -64,7 +67,7 @@ class Callback(GenericBaseRecipe):
 class Notify(GenericBaseRecipe):
 
   def createNotifier(self, notifier_binary, wrapper, executable,
-                     log, title, notification_url, feed_url):
+                     log, title, notification_url, feed_url, pidfile=None):
 
     if not os.path.exists(log):
       # Just a touch
@@ -82,6 +85,7 @@ class Notify(GenericBaseRecipe):
     return self.createWrapper(name=wrapper,
                               command=notifier_binary,
                               parameters=parameters,
+                              pidfile=pidfile,
                               comments=[
                                   '',
                                   'Call an executable and send notification(s).',
@@ -101,6 +105,7 @@ class Notify(GenericBaseRecipe):
                                  executable=options['executable'],
                                  log=log,
                                  title=options['title'],
+                                 pidfile=options['pidfile'],
                                  notification_url=options['notify'],
                                  feed_url=feed_url)
     return [script]

slapos/recipe/pwgen.py → slapos/recipe/readline.py

+# vim: set et sts=2:
 ##############################################################################
 #
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,37 +25,39 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import subprocess
-import os
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+import errno
 
-class Recipe(GenericBaseRecipe):
+class Recipe(object):
+  """Read the first line of a file.
 
-  def _options(self, options):
-    if not os.path.exists(self.options['file']):
-      password = subprocess.check_output([self.options['pwgen-binary'], '-1']).strip()
-      with open(self.options['file'], 'w') as password_file:
-        password_file.write(password)
-    else:
-      with open(self.options['file'], 'r') as password_file:
-        password = password_file.read()
-    options['password'] = password
+  As the result has to be provided as an options, it is mandatory that the
+  buildout profile fills the file content (if needed) before trying to read it.
 
-  def install(self):
-    os.chmod(self.options['file'], 0600)
-    return []
+    Options:
+    - storage-path: file to read
 
-class StablePasswordGeneratorRecipe(GenericBaseRecipe):
+    Result set in options:
+    - readline: first line of the file
   """
-  The purpose of this class is to generate a password which doesn't change
-  from one execution to the next (hence "stable"), so the generated password
-  doesn't change on each slapgrid-cp execution.
 
-  See GenericBaseRecipe.generatePassword .
-  """
+  def __init__(self, buildout, name, options):
+    storage_path = options['storage-path']
+    try:
+      with open(storage_path) as f:
+        readline = f.readline()
+    except IOError, e:
+      if e.errno != errno.ENOENT:
+        raise
+      readline = None
 
-  def _options(self, options):
-    options['password'] = self.generatePassword()
+    self.readline = readline
+    options['readline'] = readline
+
+  def install(self):
+    if self.readline is None:
+      raise ValueError('Unable to read the file content.')
+    return ()
 
-  update = install = lambda self: []
+  def update(self):
+    return ()

slapos/recipe/request.py

@@ -88,8 +88,15 @@ class Recipe(object):
       installation of request section will fail.
       Possible names depend on requested partition's software type.
 
+    state (optional)
+     Requested state, default value is the state of the requester.
+
     Output:
       See "return" input key.
+      "instance-state"
+          The current state of the instance.
+      "requested-state"
+          The requested state of the instance.
   """
   failed = None
 
@@ -112,6 +119,11 @@ class Recipe(object):
     ))
     slave = options.get('slave', 'false').lower() in \
       librecipe.GenericBaseRecipe.TRUE_VALUES
+
+    # By default XXXX Way of doing it is ugly and dangerous
+    requested_state = options.get('state', buildout['slap-connection'].get('requested','started'))
+    options['requested-state'] = requested_state
+
     slap = slapmodule.slap()
     slap.initializeConnection(
       options['server-url'],
@@ -125,21 +137,25 @@ class Recipe(object):
     self._raise_request_exception = None
     self._raise_request_exception_formatted = None
     self.instance = None
+
     # Try to do the request and fetch parameter dict...
     try:
       self.instance = request(software_url, software_type,
           name, partition_parameter_kw=partition_parameter_kw,
-          filter_kw=filter_kw, shared=slave)
+          filter_kw=filter_kw, shared=slave, state=requested_state)
       return_parameter_dict = self._getReturnParameterDict(self.instance,
           return_parameters)
+      # Fetch the instance-guid and the instance-state
+      # Note: SlapOS Master does not support it for slave instances
       if not slave:
         try:
           options['instance-guid'] = self.instance.getInstanceGuid()
           # XXX: deprecated, to be removed
           options['instance_guid'] = self.instance.getInstanceGuid()
+          options['instance-state'] = self.instance.getState()
         except (slapmodule.ResourceNotReady, AttributeError):
           # Backward compatibility. Old SlapOS master and core don't know this.
-          self.logger.warning("Impossible to fetch instance GUID.")
+          self.logger.warning("Impossible to fetch instance GUID nor state.")
     except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
       self._raise_request_exception = exc
       self._raise_request_exception_formatted = traceback.format_exc()

slapos/recipe/slapconfiguration.py

@@ -78,6 +78,8 @@ class Recipe(object):
       Partition parameter whose name cannot be represented unambiguously in
       buildout syntax are ignored. They cannot be accessed from buildout syntax
       anyway, and are available through "configuration" output key.
+    instance-state
+      The instance state.
   """
 
   # XXX: used to detect if a configuration key is a valid section key. This
@@ -91,10 +93,12 @@ class Recipe(object):
           options.get('key'),
           options.get('cert'),
       )
-      parameter_dict = slap.registerComputerPartition(
+      computer_partition = slap.registerComputerPartition(
           options['computer'],
           options['partition'],
-      ).getInstanceParameterDict()
+      )
+      parameter_dict = computer_partition.getInstanceParameterDict()
+      options['instance-state'] = computer_partition.getState()
       # XXX: those are not partition parameters, strictly speaking.
       # Make them available as individual section keys.
       for his_key in (
@@ -129,9 +133,9 @@ class Recipe(object):
 
       # also export single ip values for those recipes that don't support sets.
       if ipv4_set:
-          options['ipv4-random'] = list(ipv4_set)[0]
+          options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
       if ipv6_set:
-          options['ipv6-random'] = list(ipv6_set)[0]
+          options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
 
       options['tap'] = tap_set
       parameter_dict = self._expandParameterDict(options, parameter_dict)

slapos/recipe/slaprunner/__init__.py

@@ -40,8 +40,8 @@ class Recipe(GenericBaseRecipe):
     self.partition_amount = options['partition-amount'].strip()
     self.cloud9_url = options.get('cloud9-url', '').strip()
     self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
-    # Set slaprunner access URL
-    options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
+    # Set slaprunner access URL, CLN Beware ipv6 access is made throught nginx
+    options['access-url'] = 'https://[%s]:%s' % (self.ipv6, self.runner_port)
 
   def install(self):
     path_list = []
@@ -49,7 +49,7 @@ class Recipe(GenericBaseRecipe):
     configuration = dict(
         software_root=self.software_directory,
         instance_root=self.instance_directory,
-        master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
+        master_url='http://%s:%s' % (self.ipv4, self.proxy_port),
         computer_id='slaprunner',
         partition_amount=self.partition_amount,
         slapgrid_sr=self.options['slapgrid_sr'],
@@ -62,7 +62,7 @@ class Recipe(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['run_dir'],
         log_dir=self.options['log_dir'],
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,
@@ -132,7 +132,7 @@ class Test(GenericBaseRecipe):
         etc_dir=self.options['etc_dir'],
         run_dir=self.options['etc_dir'],
         log_dir=self.workdir,
-        runner_host=self.ipv6,
+        runner_host=self.ipv4,
         runner_port=self.runner_port,
         ipv4_address=self.ipv4,
         ipv6_address=self.ipv6,

slapos/recipe/slaprunner/backup.py

@@ -71,7 +71,7 @@ class ExportRecipe(GenericBaseRecipe):
                   done
                 }
                 sync_element %(srv-directory)s/runner  %(backup-directory)s/runner/ instance project  proxy.db softwareLink
-                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users ssh
+                sync_element %(etc-directory)s  %(backup-directory)s/etc/ .rcode .project .users .htpasswd ssh
                 if [ -d %(backup-directory)s/runner/software ]; then
                   rm %(backup-directory)s/runner/software/*
                 fi
@@ -120,12 +120,12 @@ class ImportRecipe(GenericBaseRecipe):
                   done
                 }
                 restore_element %(backup-directory)s/runner/ %(srv-directory)s/runner  instance project  proxy.db softwareLink
-                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users ssh
+                restore_element  %(backup-directory)s/etc/ %(etc-directory)s .rcode .project .users .htpasswd ssh
                 ifs=$IFS IFS=';'
                 read user pass remaining < %(etc-directory)s/.users
                 IFS=$ifs
-                %(curl-binary)s -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
-                %(curl-binary)s -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
+                %(curl-binary)s --insecure -vg6L -F clogin="$user" -F cpwd="$pass" --dump-header login_cookie  %(backend-url)s/doLogin;
+                %(curl-binary)s --insecure -vg6LX POST --cookie login_cookie --max-time 5  %(backend-url)s/runSoftwareProfile;
                 rm -f login_cookie
                 """ % self.options)
         self.createExecutable(wrapper, content=content)

slapos/recipe/softwaretype.py

@@ -87,8 +87,6 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
-    self.logger.info('Deploying instance with software type %s' % \
-        software_type)
 
     # Raise if request software_type does not exist ...
     if software_type not in self.options:

slapos/recipe/squid/__init__.py

+##############################################################################
+#
+# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+
+class Recipe(GenericBaseRecipe):
+  """
+  squid instance configuration.
+
+  wrapper-path -- location of the init script to generate
+
+  prepare-path -- location of the directory creation script to generate
+
+  binary-path -- location of the squid command
+
+  conf-path -- location of the configuration file
+
+  cache-path -- location of the cache directory
+
+  XXXX No good, specific...
+  open_port -- entrance port to the host and allowed to use cache
+
+  ip -- ip of the squid server
+
+  port -- port of the squid server
+
+  backend-ip -- ip of the service to cache
+
+  backend-port -- port of the service to cache
+
+  access-log-path -- location of the access log
+
+  cache-log-path -- location of the cache log
+
+  pid-filename-path -- location of the pid filename
+
+  """
+
+  def install(self):
+    config = dict(
+      ip=self.options['ip'], 
+      port=self.options['port'],
+      backend_ip=self.options['backend-ip'], 
+      backend_port=self.options['backend-port'],
+      cache_path=self.options['cache-path'],
+      access_log_path=self.options['access-log-path'],
+      cache_log_path=self.options['cache-log-path'],
+      pid_filename_path=self.options['pid-filename-path'],
+      open_port=self.options['open-port'],
+      )
+
+    template_filename = self.getTemplateFilename('squid.conf.in')
+    configuration_path = self.createFile(
+      self.options['conf-path'],
+      self.substituteTemplate(template_filename, config))
+
+    # Prepare directories
+    prepare_path = self.createPythonScript(
+      self.options['prepare-path'],
+      'slapos.recipe.librecipe.execute.execute',
+      arguments=[self.options['binary-path'].strip(), 
+                 '-z',
+                 '-f', configuration_path,
+                 ],)
+
+    # Create running wrapper
+    wrapper_path = self.createPythonScript(
+      self.options['wrapper-path'],
+      'slapos.recipe.librecipe.execute.execute',
+      arguments=[self.options['binary-path'].strip(), 
+                 '-N',
+                 '-f', configuration_path,
+                 ],)
+
+    return [configuration_path, wrapper_path, prepare_path]

slapos/recipe/squid/template/squid.conf.in

+refresh_pattern .   0 20%% 4320 max-stale=604800
+
+# Dissallow cachemgr access
+http_access deny manager
+
+# Squid service configuration
+http_port %(ip)s:%(port)s accel defaultsite=%(ip)s
+
+cache_peer %(backend_ip)s parent %(backend_port)s 0 no-query originserver name=backend
+
+acl our_sites port %(open_port)s
+http_access allow our_sites
+cache_peer_access backend allow our_sites
+cache_peer_access backend deny all
+
+# Drop squid headers
+# via off
+# reply_header_access X-Cache-Lookup deny all
+# reply_header_access X-Squid-Error deny all
+# reply_header_access X-Cache deny all
+
+header_replace X-Forwarded-For
+follow_x_forwarded_for allow all
+forwarded_for on
+
+# Use 1Go of RAM
+cache_mem 1024 MB
+# But do not keep big object in RAM
+maximum_object_size_in_memory 2048 KB
+
+# Log
+access_log %(access_log_path)s
+cache_log %(cache_log_path)s
+pid_filename %(pid_filename_path)s

slapos/recipe/sshkeys_authority.py

@@ -113,11 +113,11 @@ class Request(GenericBaseRecipe):
       hashlib.sha256(options['name']).hexdigest())
     self.public_key = self.private_key + '.pub'
 
-    if os.path.exists(self.public_key):
-      with open(self.public_key) as key:
-        options['public-key-value'] = key.read()
-    else:
     options['public-key-value'] = ''
+    if os.path.exists(self.public_key):
+      key_content = open(self.public_key).read()
+      if key_content:
+        options['public-key-value'] = key_content
 
   def install(self):
     requests_directory = self.options['request-directory']

software/apache-frontend/TODO.txt

+Apache:
+=======
+- set a redirection of / in option
+- Implement support of multiple ip (if possible) for multiple ssl (other solution is to ask master instance but quid to much apache process?)
+
+Squid:
+======
+- Only cache in ram. Problems with to much squid on the computer or too many slave?
+
+SlapOS:
+=======
+- Implement intelligent apache graceful -> Should check slave configuration and return error to slave if there is a problem (important but difficult)
+- useless srv/squid_cache directory so far

software/apache-frontend/common.cfg

 [buildout]
 extends =
+# dev Stuff
+  ../../component/git/buildout.cfg
+
+  ../../stack/slapos.cfg
   ../../component/binutils/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/stunnel/buildout.cfg
-  ../../component/varnish/buildout.cfg
   ../../component/dcron/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
-  ../../stack/slapos.cfg
+  ../../component/squid/buildout.cfg
 
-parts =
+parts +=
+  slapos-cookbook
+  slapos-toolbox
   template
+  template-apache-frontend
+  template-apache-replicate
   binutils
   apache-2.2
   apache-antiloris-apache-2.2
 
   stunnel
-  varnish-2.1
 
   dcron
   logrotate
   rdiff-backup
+  squid
 
-# Buildoutish
-  eggs
-  instance-recipe-egg
-
-[instance-recipe]
-# Note: In case if specific instantiation recipe is used this is the place to
-# put its name
-egg = slapos.cookbook
-module = apache.frontend
-
-[instance-recipe-egg]
+[slapos-toolbox]
 recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[eggs]
-recipe = z3c.recipe.scripts
 eggs =
   ${lxml-python:egg}
   slapos.toolbox
 
+scripts =
+  killpidfromfile
+  onetimedownload
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+
 [template]
-# Default template for apache instance.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = e7b9f57da7eb1450fc15789e239388d4
+md5sum = 9c6346c8eaf484748e6be0b62b65cf2e
 output = ${buildout:directory}/template.cfg
 mode = 0644
+
+[template-apache-frontend]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-apache-frontend.cfg
+md5sum = 7e7e7599ec41cf1eb6e8e725d855c345
+output = ${buildout:directory}/template-apache-frontend.cfg
+mode = 0644
+
+[template-apache-replicate]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
+md5sum = 2c96799f1429d0541c04c0875d864777
+mode = 0644
+
+[template-slave-list]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
+md5sum = f5eef006211809669b12422240c6f436
+mode = 640
+
+[template-slave-configuration]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/slave-virtualhost.conf.in
+md5sum = a7ad2e83b7f919fc45a7ef1e64344dcb
+mode = 640
+
+[template-replicate-publish-slave-information]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/replicate-publish-slave-information.cfg.in
+mode = 640
+
+[template-apache-frontend-configuration]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/apache.conf.in
+md5sum = c141b9e78c7e80d75bb40493910294e5
+mode = 640
+
+[template-apache-cached-configuration]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/apache_cached.conf.in
+md5sum = 0c4393db80670daf18b432b7f07383e9
+mode = 640
+
+[template-rewrite-cached]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/apache_cached_rewrite.txt.in
+md5sum = 2f30af4f9da340c2b0618599da03ed4b
+mode = 640
+
+[template-custom-slave-list]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/apache-default-slave-list.cfg.in
+md5sum = 9362384cd80727987b34c7746a6de196
+mode = 640
+
+[template-not-found-html]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/notfound.html
+filename = notfound.html
+md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
+mode = 640
+
+[template-default-virtualhost]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/000.conf.in
+md5sum = c2bbf029e6adc432de0884fb5cf5d2ab
+mode = 640
+
+[template-default-slave-virtualhost]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
+md5sum = ac845c0fa3835832307a0e7323cb339d
+mode = 640
+
+[template-empty]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/empty.in
+md5sum = c2314c3a9c3412a38d14b312d3df83c1
+mode = 640
\ No newline at end of file

software/apache-frontend/instance-apache-replicate.cfg.in

+{% if slap_software_type.startswith(software_type) -%}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/${:filename}
+extra-context =
+context =
+    import json_module json
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key slap_software_type slap-parameter:slap_software_type
+    key slave_instance_list slap-parameter:slave_instance_list
+    ${:extra-context}
+
+{% set part_list = [] -%}
+{% set type_key = 'replicate-' %}
+{% set type_key_length = type_key | length %}
+{% if slap_software_type.startswith(type_key) %}
+{%   set frontend_type = slap_software_type[type_key_length:] -%}
+{% else -%}
+{%   set frontend_type = slapparameter_dict.pop('-frontend-type', 'default') -%}
+{% endif -%}
+{% set frontend_quantity = slapparameter_dict.pop('-frontend-quantity', '2') | int -%}
+{% set slave_list_name = 'extra_slave_instance_list' -%}
+{% set frontend_list = [] %}
+{% set frontend_section_list = [] %}
+{% set namebase = 'apache-frontend' -%}
+
+# Here we request individualy each frontend.
+# The presence of sla parameters is checked and added if found
+{% for i in range(1, frontend_quantity + 1) -%}
+{%   set frontend_name = "%s-%s" % (namebase, i) -%}
+{%   set request_section_title = 'request-%s' % frontend_name -%}
+{%   set sla_key = "-sla-%s-" % i -%}
+{%   set sla_key_length = sla_key | length %}
+{%   set sla_parameters = [] %}
+{%   for key in slapparameter_dict.keys() %}
+{%     if key.startswith(sla_key) %}
+{%       do sla_parameters.append(key[sla_key_length:]) %}
+{%     endif -%}
+{%   endfor -%}
+{%   do frontend_list.append(frontend_name) -%}
+{%   do frontend_section_list.append(request_section_title) -%}
+{%   do part_list.append(request_section_title) -%}
+[{{request_section_title}}]
+<= replicate
+name = {{frontend_name}}
+{%   if sla_parameters %}
+sla = {{ ' '.join(sla_parameters) }}
+{%     for parameter in sla_parameters -%}
+sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
+{%     endfor -%}
+{%   endif -%}
+{% endfor -%}
+
+[replicate]
+<= slap-connection
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+software-type = {{frontend_type}}
+return = private-ipv4 public-ipv4 slave-instance-information-list
+config = {{ ' '.join(slapparameter_dict.keys()) + ' ' + slave_list_name }}
+{% for parameter, value in slapparameter_dict.iteritems() -%}
+config-{{parameter}} = {{ value }}
+{% endfor -%}
+config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
+
+[publish-information]
+recipe = slapos.cookbook:publish
+domain = {{ slapparameter_dict.get('domain') }}
+slave-amount = {{ slave_instance_list | length }}
+{% for frontend in frontend_list -%}
+#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
+{% endfor -%}
+
+
+#----------------------------
+#--
+#-- Publish slave information
+[publish-slave-information]
+recipe = slapos.cookbook:softwaretype
+default = ${dynamic-publish-slave-information:rendered}
+replicate = ${dynamic-publish-slave-information:rendered}
+
+[slave-information]
+{% for frontend_section in frontend_section_list -%}
+{{ frontend_section }} = {{ "${%s:connection-slave-instance-information-list}" % frontend_section }}
+{% endfor -%}
+
+[dynamic-publish-slave-information]
+< = jinja2-template-base
+template = {{ template_publish_slave_information }}
+filename = dynamic-publish-slave-information.cfg
+extensions = jinja2.ext.do
+extra-context =
+    section slave_information slave-information
+
+[buildout]
+parts =
+  publish-slave-information
+  publish-information
+{% for part in part_list -%}
+{{ '  %s' % part }}
+{% endfor -%}
+#      publish-information
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+
+[slap_connection]
+# Kept for backward compatiblity
+computer_id = ${slap-connection:computer-id}
+partition_id = ${slap-connection:partition-id}
+server_url = ${slap-connection:server-url}
+software_release_url = ${slap-connection:software-release-url}
+key_file = ${slap-connection:key-file}
+cert_file = ${slap-connection:cert-file}
+
+[slap-parameter]
+slave_instance_list =
+-frontend-quantity = 2
+-frontend-type = default
+
+{%- endif %}

software/apache-frontend/instance.cfg

 [buildout]
 parts =
-  directory
-  apache
-  configtest
-  logrotate
-  logrotate-entry-apache
+  dynamic-template-apache-replicate
+  switch-softwaretype
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-# Create all needed directories
-[directory]
-recipe = slapos.cookbook:mkdirectory
-
-bin = $${buildout:directory}/bin/
-etc = $${buildout:directory}/etc/
-srv = $${buildout:directory}/srv/
-var = $${buildout:directory}/var/
-
-backup = $${:srv}/backup
-log = $${:var}/log
-run = $${:var}/run
-service = $${:etc}/service
-
-logrotate-backup = $${:backup}/logrotate
-logrotate-entries = $${:etc}/logrotate.d
-
-
-# Deploy Apache (old way, with monolithic recipe)
-[apache]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-httpd_home = ${apache-2.2:location}
-httpd_binary = ${apache-2.2:location}/bin/httpd
-logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
-openssl_binary = ${openssl:location}/bin/openssl
-dcrond_binary = ${dcron:location}/sbin/crond
-varnishd_binary = ${varnish-2.1:location}/sbin/varnishd
-stunnel_binary = ${stunnel:location}/bin/stunnel
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-gcc_binary = gcc
-binutils_directory = ${binutils:location}/bin/
-
-access-log = $${directory:log}/frontend-apache-access.log
-error-log = $${directory:log}/frontend-apache-error.log
-pid-file = $${directory:run}/httpd.pid
-
-
-# Create wrapper for "apachectl conftest" in bin
-[configtest]
-recipe = slapos.cookbook:wrapper
-command-line = $${apache:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
-wrapper-path = $${directory:bin}/apache-configtest
-
-
-# Deploy Logrotate
-[logrotate]
-recipe = slapos.cookbook:logrotate
-# Binaries
-logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
-gzip-binary = ${gzip:location}/bin/gzip
-gunzip-binary = ${gzip:location}/bin/gunzip
-# Directories
-wrapper = $${directory:bin}/logrotate
-conf = $${directory:etc}/logrotate.conf
-logrotate-entries = $${directory:logrotate-entries}
-backup = $${directory:logrotate-backup}
-state-file = $${directory:srv}/logrotate.status
-
-[logrotate-entry-apache]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = apache
-log = $${apache:error-log} $${apache:access-log}
-frequency = daily
-rotate-num = 30
-post = ${buildout:bin-directory}/killpidfromfile $${apache:pid-file} SIGUSR1
-sharedscripts = true
-notifempty = true
-create = true
+offline = true
+
+[slap-parameters]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/$${:filename}
+extra-context =
+context =
+    import json_module json
+    key eggs_directory buildout:eggs-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key slap_software_type slap-parameters:slap-software-type
+    key slapparameter_dict slap-parameters:configuration
+    key slave_instance_list slap-parameters:slave-instance-list
+    $${:extra-context}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = ${template-apache-frontend:output}
+custom-personal = ${template-apache-frontend:output}
+custom-group = ${template-apache-frontend:output}
+replicate-default = $${dynamic-template-apache-replicate:rendered}
+replicate-custom-personal = $${dynamic-template-apache-replicate:rendered}
+replicate-custom-group = $${dynamic-template-apache-replicate:rendered}
+replicate = $${dynamic-template-apache-replicate:rendered}
+
+[dynamic-template-apache-replicate]
+< = jinja2-template-base
+template = ${template-apache-replicate:target}
+filename = instance-apache-replicate.cfg
+extensions = jinja2.ext.do
+extra-context =
+    raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type replicate

software/apache-frontend/templates/000.conf.in

+<VirtualHost *:{{ https_port }}>
+  ServerName www.example.org
+  SSLEngine on
+  SSLProxyEngine on
+  SSLProtocol -ALL +SSLv3 +TLSv1
+  SSLHonorCipherOrder On
+  SSLCipherSuite RC4-SHA:HIGH:!ADH
+
+  # Rewrite part
+  ProxyVia On
+  ProxyPreserveHost On
+  ProxyTimeout 600
+  RewriteEngine On
+
+
+  ErrorDocument 404 /notfound.html
+
+</VirtualHost>
+
+<VirtualHost *:{{ http_port }}>
+
+  ServerName www.example.org
+  ErrorDocument 404 /notfound.html
+</VirtualHost>
\ No newline at end of file

software/apache-frontend/templates/apache_cached_rewrite.txt.in

+{% for server_tuple in server_dict.items() -%}
+{{ "%s %s" % server_tuple }}
+{% endfor -%}

software/apache-frontend/templates/default-virtualhost.conf.in

+{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
+ 
+<VirtualHost *:{{ https_port }}>
+  ServerName {{ slave_parameter.get('domain') }}
+  ServerAlias {{ slave_parameter.get('domain') }}
+
+  SSLEngine on
+  SSLProxyEngine on
+  SSLProtocol -ALL +SSLv3 +TLSv1
+  SSLHonorCipherOrder On
+  SSLCipherSuite RC4-SHA:HIGH:!ADH
+
+{% set ssl_configuration_list = [('SSLCertificateFile', 'path_to_ssl_crt'),
+       			      	 ('SSLCertificateKeyFile', 'path_to_ssl_key'),
+                                 ('SSLCACertificateFile', 'path_to_ssl_ca_crt'),
+                                 ('SSLCertificateChainFile', 'path_to_ssl_ca_crt')] -%}
+
+{% for key, value in ssl_configuration_list -%}
+{%   if value in slave_parameter -%}
+{{ '  %s' % key }} {{ slave_parameter.get(value) }}
+{% endif -%}
+{% endfor -%}
+
+
+  # One Slave two logs
+  ErrorLog "{{ error_log }}"
+  LogLevel info
+  LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+  CustomLog "{{ access_log }}" combined
+
+  # Rewrite part
+  ProxyVia On
+  ProxyPreserveHost On
+  ProxyTimeout 600
+  RewriteEngine On
+
+{% if slave_parameter.get('type', '') ==  'zope' -%}
+  # First, we check if we have a zope backend server
+  # If so, let's use Virtual Host Daemon rewrite
+  # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/https/{{ slave_parameter.get('domain', '') }}:443/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
+{% else -%}
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/$1 [L,P]
+{% endif -%}
+</VirtualHost>
+
+<VirtualHost *:{{ http_port }}>
+  ServerName {{ slave_parameter.get('domain') }}
+  ServerAlias {{ slave_parameter.get('domain') }}
+  SSLProxyEngine on
+  # Rewrite part
+  ProxyVia On
+  ProxyPreserveHost On
+  ProxyTimeout 600
+  RewriteEngine On
+
+  # One Slave two logs
+  ErrorLog "{{ error_log }}"
+  LogLevel info
+  LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+  CustomLog "{{ access_log }}" combined
+
+  # Remove "Secure" from cookies, as backend may be https
+  Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
+
+# Next line is forbidden and people who copy it will be hanged short
+{% set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES -%}
+{% if https_only -%}
+  # Not using HTTPS? Ask that guy over there.
+  # Dummy redirection to https. Note: will work only if https listens
+  # on standard port (443).
+  RewriteCond     %{SERVER_PORT}  !^{{ https_port }}$
+  RewriteRule     ^/(.*)          https://%{SERVER_NAME}/$1 [NC,R,L]
+{% elif slave_parameter.get('type', '') ==  'zope' -%}
+  # First, we check if we have a zope backend server
+  # If so, let's use Virtual Host Daemon rewrite
+  # We suppose that Apache listens to 80 (even indirectly thanks to things like iptables)
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/http/{{ slave_parameter.get('domain', '') }}:80/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
+{% else -%}
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/$1 [L,P]
+{% endif -%}
+  # If nothing exist : put a nice error
+#  ErrorDocument 404 /notfound.html
+# Dadiboom
+
+</VirtualHost>

software/apache-frontend/templates/empty.in

+{{ content }}
\ No newline at end of file

software/apache-frontend/templates/slave-virtualhost.conf.in

+<VirtualHost *:{{ https_port }}>
+{{ apache_custom_https }}
+</VirtualHost>
+
+<VirtualHost *:{{ http_port }}>
+{{ apache_custom_http }}
+</VirtualHost>

software/backupserver/template-nginx-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+if [ ! -e $${nginx-configuration:ssl_crt} ]
+then
+  ${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
+  ${openssl-output:openssl} req -new \
+    -subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
+    -key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
+  ${openssl-output:openssl} x509 -req -days 365 \
+    -in $${nginx-configuration:ssl_csr} \
+    -signkey $${nginx-configuration:ssl_key} \
+    -out $${nginx-configuration:ssl_crt}
+fi
+
+exec ${nginx-output:nginx} \
+  -c $${nginx-configuration:output}