Merge branch 'master' into clearroad

component/dcron/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../coreutils/buildout.cfg
   ../patch/buildout.cfg
+  ../randomsleep/buildout.cfg
 
 parts = dcron-output
 
@@ -18,6 +19,7 @@ make-options =
 post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
 environment =
   PATH=${patch:location}/bin:%(PATH)s
+randomsleep_install = ${randomsleep:location}
 
 [dcron-output]
 # Shared binary location to ease migration

component/fontconfig/buildout.cfg

@@ -7,14 +7,15 @@ extends =
   ../bzip2/buildout.cfg
   ../zlib/buildout.cfg
   ../bzip2/buildout.cfg
+  ../gperf/buildout.cfg
 
 parts =
   fontconfig
 
 [fontconfig]
 recipe = slapos.recipe.cmmi
-url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
-md5sum = b5af5a423ee3b5cfc34846838963c058
+url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
+md5sum = 733f5e2371ca77b69707bd7b30cc2163
 pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
 # XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
 configure-options =
@@ -23,7 +24,7 @@ configure-options =
   --enable-libxml2
   --with-default-fonts=${fonts:location}
 environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib

component/gcc/buildout.cfg

@@ -2,6 +2,7 @@
 # Mostly required to support languages different than C or C++
 [buildout]
 extends =
+  ../gettext/buildout.cfg
   ../gmp/buildout.cfg
   ../perl/buildout.cfg
   ../tar/buildout.cfg
@@ -32,7 +33,7 @@ configure-options =
   --with-as=${binutils:location}/bin/as
 environment =
   LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
-  PATH=${binutils:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
+  PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
 
 [gcc-minimal]
 <= gcc-common

component/golang/buildout.cfg

@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
 
 [golang18]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
-md5sum = 64e9380e07bba907e26a00cf5fcbe77e
+url = https://dl.google.com/go/go1.8.7.src.tar.gz
+md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
 
 # go1.8 needs go1.4 to bootstrap
 environment-extra =
@@ -43,8 +43,8 @@ environment-extra =
 
 [golang19]
 <= golang-common
-url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
-md5sum = 44105c865a1a810464df79233a05a568
+url = https://dl.google.com/go/go1.9.4.src.tar.gz
+md5sum = 6816441fd6680c63865cdd5cb8bc1960
 
 # go1.9 needs go1.4 to bootstrap
 environment-extra =
@@ -63,6 +63,11 @@ environment-extra =
 #       lab.nexedi.com/kirr/neo/go/...  \
 #       github.com/pkg/profile          \
 #       golang.org/x/perf/cmd/benchstat
+#
+# it is possible to specify Go build flags used for compilation e.g. this way:
+#
+#   [gowork]
+#   buildflags = -race
 [gowork]
 directory = ${buildout:directory}/go.work
 src	= ${:directory}/src
@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
 # go version used for the workspace (possible to override in applications)
 golang  = ${golang19:location}
 
+# no special build flags by default
+buildflags =
+
 # everything is done by dependent parts
 recipe  = plone.recipe.command
 command = :
@@ -100,7 +108,7 @@ stop-on-error = true
 # clients should put package list to install to gowork:install ("..." requests installing everything)
 [gowork.goinstall]
 recipe  = plone.recipe.command
-command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
+command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
 update-command = ${:command}
 stop-on-error = true
 
@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
 # 3) provide repository (which is not the same as importpath in general case)
 #
 # the list of go packages for a go workspace state can be automatically
-# generated with the help of go-pkg-snapshot tool.
+# generated with the help of gowork-snapshot tool.
 [go-git-package]
 <= git-repository
 location = ${gowork:src}/${:go.importpath}

component/golang/gowork-snapshot

@@ -25,9 +25,16 @@ gogit_list() {
 # git_upstream_url <repo>   - show current branch upstream URL
 git_upstream_url() {
     repo=$1
-    head="`git -C $repo symbolic-ref --short HEAD`"             # current branch - e.g. "t"
-    remote="`git -C $repo config --get branch.$head.remote`"    # upstream name, e.g. "kirr"
-    url="`git -C $repo config --get remote.$remote.url`"        # upstream URL
+    head="`git -C $repo symbolic-ref -q --short HEAD`"            # current branch - e.g. "master"
+    if [ -z $head ] ; then
+        remote="origin" # If we are in detached HEAD, assume the upstream name is origin
+    else
+        remote="`git -C $repo config --get branch.$head.remote`"  # upstream name, e.g. "origin"
+    fi
+    url="`git -C $repo config --get remote.$remote.url`"          # upstream URL
+    if [ -z $url ] ; then
+        echo "ERROR: Failed to guess upstream url from $repo" 1>&2
+    fi
     echo "$url"
 }
 
@@ -59,5 +66,5 @@ while read pkg url rev; do
     echo "<= go-git-package"
     echo "go.importpath = $pkg"
     echo "repository    = $url"
-    echo "revision  = $rev"
+    echo "revision      = $rev"
 done

component/h5py/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../hdf5/buildout.cfg
   ../cython/buildout.cfg
+  ../numpy/buildout.cfg
 parts =
   h5py
 
@@ -12,7 +13,9 @@ CPPFLAGS = -I${hdf5:location}/include
 [h5py]
 recipe = zc.recipe.egg:custom
 egg = h5py
-setup-eggs = ${cython:egg}
+setup-eggs = 
+  ${cython:egg}
+  ${numpy:egg}
 include-dirs =
   ${hdf5:location}/include
 library-dirs =

component/libxml2/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [libxml2]
 recipe = slapos.recipe.cmmi
-url = http://xmlsoft.org/sources/libxml2-2.9.5.tar.gz
-md5sum = 5ce0da9bdaa267b40c4ca36d35363b8b
+url = http://xmlsoft.org/sources/libxml2-2.9.7.tar.gz
+md5sum = 896608641a08b465098a40ddf51cefba
 configure-options =
   --disable-static
   --without-python

component/libxslt/buildout.cfg

@@ -7,8 +7,8 @@ parts =
   libxslt
 
 [libxslt]
-url = ftp://xmlsoft.org/libxslt/libxslt-1.1.30.tar.gz
-md5sum = 70becbbcb1dad55f14de0b84171b91d5
+url = ftp://xmlsoft.org/libxslt/libxslt-1.1.32.tar.gz
+md5sum = 1fc72f98e98bf4443f1651165f3aa146
 recipe = slapos.recipe.cmmi
 configure-options =
   --disable-static
@@ -20,5 +20,3 @@ configure-options =
 environment =
   CPPFLAGS=-I${zlib:location}/include
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  LIBXML_CFLAGS=-I${libxml2:location}/include
-  LIBXML_LIBS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib

component/mariadb/buildout.cfg

@@ -27,8 +27,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.1.29
-md5sum = 6d2cc6b0f8c5c7525e08aaddad8dca96
+version = 10.1.31
+md5sum = 14ab0398c019eb531bc29f2c437ccb51
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3

component/mariadb/mariarocks.cfg

 # Do not extend any file that touch buildout:parts.
 
 [mariadb]
-version = 10.2.11
-md5sum = 954088299fe5f11b4fda3b540558adbd
+version = 10.2.13
+md5sum = 20c61bd4059ba287e54cfb2862bae81d
 stable-patches =
 configure-options +=
-# force build of TokuDB due to a regression in 10.2.11
-  -DTOKUDB_OK=1
   -DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
   -DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
 extra_cflags = -I${zstd:location}/include

component/nginx/buildout.cfg

@@ -18,6 +18,7 @@ patch-options = -p0
 patch-binary = ${patch:location}/bin/patch
 patches =
   http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
+  ${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
 
 [nginx]
 <= nginx-common

component/nginx/fix-gcc7-implicit-fallthrough-errors.patch

+commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
+Author: Maxim Dounin <mdounin@mdounin.ru>
+Date:   Thu Apr 27 16:57:18 2017 +0300
+
+    Added missing "fall through" comments (ticket #1259).
+    
+    Found by gcc7 (-Wimplicit-fallthrough).
+
+diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
+index c31e0e03..5ade658d 100644
+--- src/core/ngx_murmurhash.c
++++ src/core/ngx_murmurhash.c
+@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
+     switch (len) {
+     case 3:
+         h ^= data[2] << 16;
++        /* fall through */
+     case 2:
+         h ^= data[1] << 8;
++        /* fall through */
+     case 1:
+         h ^= data[0];
+         h *= 0x5bd1e995;
+diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
+index 36220fdc..e8e51563 100644
+--- src/http/ngx_http_parse.c
++++ src/http/ngx_http_parse.c
+@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
+                 goto done;
+             case '+':
+                 r->plus_in_uri = 1;
++                /* fall through */
+             default:
+                 state = sw_usual;
+                 *u++ = ch;
+diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
+index dd50b5ca..993c032a 100644
+--- src/os/unix/ngx_process.c
++++ src/os/unix/ngx_process.c
+@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
+                 break;
+             }
+             ngx_debug_quit = 1;
++            /* fall through */
+         case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
+             ngx_quit = 1;
+             action = ", shutting down";

component/ocropy/buildout.cfg

@@ -32,8 +32,8 @@ setup-eggs =
   ${matplotlib:egg}
   ${pillow-python:egg}
 patches =
-  ${:_profile_base_location_}/ocropy.patch
-patch-options = -p0
+  ${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
+patch-options = -p1
 patch-binary = ${patch:location}/bin/patch
 environment = ocropy-env
 find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz

component/ocropy/ocropy.patch

-diff --git ocrolib/__init__.py ocrolib/__init__.py
+From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
+From: Francois Le Corre <francois.lecorre@nexedi.com>
+Date: Thu, 6 Apr 2017 11:32:27 +0200
+Subject: [PATCH] WIP
+
+- New OCROPY_MODEL_PATH environment variable to specify the position of models
+- Import for the first time a module that compiles C code from python,
+  allowing us to have the static library built before runtime and
+  preventing us from using gcc through zope.
+---
+ ocrolib/__init__.py |  3 ++-
+ ocrolib/common.py   |  7 ++++---
+ ocrolib/native.py   |  1 +
+ setup.py            | 15 ++++++++++++---
+ 4 files changed, 19 insertions(+), 7 deletions(-)
+
+diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
 index 1e0d627..81e85fb 100644
---- ocrolib/__init__.py
-+++ ocrolib/__init__.py
+--- a/ocrolib/__init__.py
++++ b/ocrolib/__init__.py
 @@ -1,7 +1,7 @@
  __all__ = [
      "binnednn","cairoextras","common","components","dbtables",
@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
  ]
  
  ################################################################
-@@ -9,5 +9,6 @@ __all__ = [
+@@ -9,5 +9,6 @@
  ################################################################
  
  import default
 +from psegutils import *
  from common import *
  from default import traceback as trace
-diff --git ocrolib/common.py ocrolib/common.py
+diff --git a/ocrolib/common.py b/ocrolib/common.py
 index 27c0f26..14f088f 100644
---- ocrolib/common.py
-+++ ocrolib/common.py
-@@ -14,6 +14,7 @@ import unicodedata
+--- a/ocrolib/common.py
++++ b/ocrolib/common.py
+@@ -14,6 +14,7 @@
  import inspect
  import glob
  import cPickle
@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
  def load_object(fname,zip=0,nofind=0,verbose=0):
      """Loads an object from disk. By default, this handles zipped files
      and searches in the usual places for OCRopus. It also handles some
-@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
+@@ -439,8 +441,7 @@ class names that have changed."""
      if zip==0 and fname.endswith(".gz"):
          zip = 1
      if zip>0:
@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
  
      possible_prefixes.append("/usr/local/share/ocropus")
  
-diff --git ocrolib/native.py ocrolib/native.py
+diff --git a/ocrolib/native.py b/ocrolib/native.py
 index b7a207f..240450b 100644
---- ocrolib/native.py
-+++ ocrolib/native.py
+--- a/ocrolib/native.py
++++ b/ocrolib/native.py
 @@ -44,6 +44,7 @@ class CompileError(Exception):
  
  def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
      if not os.path.exists(prefix):
          os.mkdir(prefix)
      m = hashlib.md5()
-diff --git setup.py setup.py
-index 2ec5832..6697b12 100644
---- setup.py
-+++ setup.py
-@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
+diff --git a/setup.py b/setup.py
+index 2ec5832..0ad4d85 100644
+--- a/setup.py
++++ b/setup.py
+@@ -10,7 +10,9 @@
  from distutils.core import setup #, Extension, Command
  #from distutils.command.install_data import install_data
  
 -if not os.path.exists("models/en-default.pyrnn.gz"):
-+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
-+    [c for c in glob.glob("models/*pyrnn.gz")]
++models = os.environ.get('OCROPY_MODEL_PATH')
++models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
 +if not models:
      print()
      print("You should download the default model 'en-default.pyrnn.gz'")
      print("and put it into ./models.")
-@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
+@@ -18,16 +20,23 @@
      print("Check https://github.com/tmbdev/ocropy for the location")
      print("of model files.")
      print()
@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
 +    data_files= [('share/ocropus', models), ("", ["LICENSE"])],
      scripts = scripts,
      )
+-- 
+2.14.1
+

component/python-2.7/asyncore_poll_insteadof_select.patch

+From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
+From: Kirill Smelkov <kirr@nexedi.com>
+Date: Thu, 06 Jul 2017 21:15:31 +0200
+Subject: [PATCH] asyncore: switch default to use poll instead of select
+
+Recently we started to get the following errors on a Wendelin production
+instance:
+
+      File ".../bin/runzope", line 211, in <module>
+        sys.exit(Zope2.Startup.run.run())
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
+        starter.run()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
+        Lifetime.loop()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
+        lifetime_loop()
+      File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
+        asyncore.poll(timeout, map)
+      File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
+        r, w, e = select.select(r, w, e, timeout)
+    ValueError: filedescriptor out of range in select()
+
+Initially we thought the reason here is that number of file descriptors this
+process uses goes beyond allowed limit (65K open files on that instance)
+because wendelin.core uses 1 fd per an array view (opening
+/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
+
+The reason here is that select() has limit for how many #fd it can
+monitor at all. The limit is system-specific but on Linux it is usually
+1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
+
+    $ cat s.c
+    #include <sys/select.h>
+    #include <stdio.h>
+
+    int main() {
+            printf("%d\n", FD_SETSIZE);
+            return 0;
+    }
+
+    $ tcc -run s.c
+    1024
+
+Also select() can monitor only file descriptors which are by itself
+"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
+opened file descriptors because 1025 is not < 1024.
+
+As was said above in wendelin.core every array view uses 1 file
+descriptor, so if we are using not so small amount of arrays, even
+though #fd does not go beyond process-specific ulimit, because of
+select() usage the total number of allowed opened file descriptors
+becomes essentially 1024.
+
+So let's switch from select() to poll() which does not have this 1024
+ #fd limit. Asyncore already support using poll() out of the box -
+either via passing use_pull=True to asyncore.loop()
+
+https://docs.python.org/2/library/asyncore.html#asyncore.loop
+
+or by using asyncore.poll2() instead of asyncore.poll()
+
+https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
+https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
+
+--------
+
+One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
+place in Zope - inside lifetime_loop(). There are however many such similar
+places in Zope (e.g. in medusa) and in other software.
+
+For this reason, for me, what makes sense to do is not to patch all such
+places, but instead change via runtime-patching asyncore.poll to be
+asyncore.poll2 - this way all software will automatically benefit from
+poll() usage instead of select.
+
+P.S.
+
+What might also make sense to do in the future is to actually let
+asyncore.poll to use epoll(), as both select() and poll() are doing all
+fd registration on _every_ call, so when #fd grows this preparation
+time grows too. For epoll() file descriptors are registered only once.
+For this to work asyncore.socket_map has to be patched also, since there
+are places in code which modify this fd registry directly (e.g. remove
+fd from there etc)
+
+Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
+
+---
+ Lib/asyncore.py | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/Lib/asyncore.py b/Lib/asyncore.py
+index 105982f..a3025a4 100644
+--- a/Lib/asyncore.py
++++ b/Lib/asyncore.py
+@@ -661,3 +661,12 @@ if os.name == 'posix':
+             self.socket = file_wrapper(fd)
+             self._fileno = self.socket.fileno()
+             self.add_channel()
++
++# monkeypatch to switch everyone's default to use poll instead of select
++# (select has 1024 fd limit)
++# TODO it is better to use epoll
++_poll_select = poll
++_poll_poll   = poll2
++poll = _poll_poll
++
++#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
+-- 
+2.16.1.101.gde0f0111ea

component/python-2.7/buildout.cfg

@@ -43,6 +43,7 @@ patches =
   ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
   ${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
   ${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
+  ${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
 url =
   http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/qemu-kvm/buildout.cfg

@@ -19,12 +19,11 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2
-md5sum = 17940dce063b6ce450a12e719a6c9c43
+url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
+md5sum = 335994a755bc655e88a87aeb36bfc0b9
 configure-options =
   --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
   --enable-system
-  --with-system-pixman
   --disable-sdl
   --disable-xen
   --disable-vnc-sasl
@@ -58,25 +57,26 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 <= debian-netinst-base
 arch = amd64
 
-[debian-amd64-squeeze-netinst.iso]
-<= debian-amd64-netinst-base
-version = 6.0.10
-md5sum = 7f82d341561035f65933da43f94d5b52
-
 [debian-amd64-wheezy-netinst.iso]
 <= debian-amd64-netinst-base
 version = 7.11.0
 md5sum = 096c1c18b44c269808bd815d58c53c8f
 
+[debian-amd64-jessie-netinst.iso]
+<= debian-amd64-netinst-base
+release = archive
+version = 8.10.0
+md5sum = 19dcfc381bd3e609c6056216d203f5bc
+
 [debian-amd64-netinst.iso]
-# Download the installer of Debian 8 (Jessie)
+# Download the installer of Debian 9 (Stretch)
 <= debian-amd64-netinst-base
 release = release/current
-version = 8.6.0
-md5sum = e9f61bf327db6d8f7cee05a99f2353cc
+version = 9.3.0
+md5sum = db8ab7871bc2b7d456c4746e706fb5d3
 
 [debian-amd64-testing-netinst.iso]
-# Download the installer of Debian Stretch
+# Download the installer of Debian Buster
 <= debian-amd64-netinst-base
 release = stretch_di_alpha7
 version = stretch-DI-alpha7

component/randomsleep/buildout.cfg

+[buildout]
+extends = 
+  ../coreutils/buildout.cfg
+  ../bash/buildout.cfg
+
+[randomsleep]
+recipe = slapos.recipe.build
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+bin_dir = ${buildout:bin-directory}
+bash_script_code = 
+  if [ "$#" -ne 1 ]; then
+    echo "usage: randomsleep maxseconds"
+    exit
+  fi
+  ${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
+wrapper_script_code = 
+  #!${bash:location}/bin/bash
+  ${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
+script = 
+  os.makedirs(self.options['location'])
+
+  bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
+  with open(bash_script_path, 'w') as f:
+    f.write(self.options['bash_script_code'])
+
+  wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
+  with open(wrapper_script_path, 'w') as f:
+    f.write(self.options['wrapper_script_code'])
+  os.chmod(wrapper_script_path, 0750)

component/rsync/buildout.cfg

@@ -7,8 +7,8 @@ parts =
 
 [rsync]
 recipe = slapos.recipe.cmmi
-url = http://rsync.samba.org/ftp/rsync/src/rsync-3.1.1.tar.gz
-md5sum = 43bd6676f0b404326eee2d63be3cdcfe
+url = https://download.samba.org/pub/rsync/rsync-3.1.2.tar.gz
+md5sum = 0f758d7e000c0f7f7d3792610fad70cb
 make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
 environment =

component/tensorboard/0.4.patch → component/tensorboard/0.4.0.patch

 diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
-index b386d59..f03b056 100755
+index 88fb5a2..782dbc5 100755
 --- a/tensorboard/pip_package/build_pip_package.sh
 +++ b/tensorboard/pip_package/build_pip_package.sh
 @@ -26,6 +26,7 @@ function main() {
@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
  
    echo $(date) : "=== Using tmpdir: ${TMPDIR}"
  
-@@ -45,8 +46,8 @@ function main() {
+@@ -44,9 +45,7 @@ function main() {
+   pushd ${TMPDIR} >/dev/null
    rm -f MANIFEST
-   echo $(date) : "=== Building wheel"
-   echo $(pwd)
--  python setup.py bdist_wheel >/dev/null
--  python3 setup.py bdist_wheel >/dev/null
+   echo $(date) : "=== Building python2 wheel in $PWD"
+-  python setup.py bdist_wheel --python-tag py2 >/dev/null
+-  echo $(date) : "=== Building python3 wheel in $PWD"
+-  python setup.py bdist_wheel --python-tag py3 >/dev/null
 +  PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
-+  #python3 setup.py bdist_wheel >/dev/null
    mkdir -p ${DEST}
    cp dist/* ${DEST}
-   popd
+   popd >/dev/null

component/tensorboard/buildout.cfg

@@ -11,11 +11,11 @@ parts =
 recipe = plone.recipe.command
 stop-on-error = true
 repository = https://github.com/tensorflow/tensorboard
-tag = 0.4
+tag = 0.4.0
 git-binary = ${git:location}/bin/git
 patch-binary = ${patch:location}/bin/patch
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
+command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
 
 [tensorboard-build]
 recipe = slapos.recipe.build
@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
 script =
   os.makedirs(location)
   workdir = self.options['tensorboard-repository-path']
-  egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
+  egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
   dist_dir = os.path.join(workdir, 'dist')
   dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
   env = {'PATH':':'.join([self.options['bazel-bin'],

component/util-linux/buildout.cfg

@@ -5,8 +5,8 @@ extends =
 
 [util-linux]
 recipe = slapos.recipe.cmmi
-url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz
-md5sum = 07b6845f48a421ad5844aa9d58edb837
+url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
+md5sum = 7733b583dcb51518944d42aa62ef19ea
 configure-options =
   --disable-static
   --enable-libuuid
@@ -29,7 +29,7 @@ configure-options =
   --disable-rename
   --disable-schedutils
   --disable-setterm
-  --disable-libsmartcols
+  --enable-libsmartcols
   --disable-switch_root
   --disable-tinfo
   --disable-tls

component/zbar/buildout.cfg

+[buildout]
+parts=
+  zbar
+
+[zbar]
+recipe = slapos.recipe.cmmi
+url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
+md5sum = 0fd61eb590ac1bab62a77913c8b086a5
+configure-options = 
+  --disable-video
+  --without-imagemagick
+  --without-gtk
+  --without-xv
+  --without-qt
+  --without-python 
+  --without-x
+  --without-jpg
+environment =
+  CFLAGS=

component/zbarlight/buildout.cfg

+[buildout]
+extends =
+  ../pillow/buildout.cfg
+  ../zbar/buildout.cfg
+parts =
+  zbarlight
+
+[zbarlight]
+recipe = zc.recipe.egg:custom
+egg = zbarlight
+include-dirs =
+  ${zbar:location}/include
+library-dirs =
+  ${zbar:location}/lib
+rpath =
+  ${zbar:location}/lib
+depend-eggs = ${pillow-python:egg}

component/zstd/buildout.cfg

@@ -5,7 +5,7 @@ parts =
 [zstd]
 recipe = slapos.recipe.cmmi
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-url = https://github.com/facebook/zstd/archive/v1.3.1.tar.gz
-md5sum = e849ceef2f090240f690c13fba6ca70b
+url = https://github.com/facebook/zstd/archive/v1.3.3.tar.gz
+md5sum = 187f8df17a75a74f78a23ea4806ac65f
 configure-command = :
 make-options = PREFIX=${:location}

setup.py

@@ -72,7 +72,6 @@ setup(name=name,
         'zc.buildout': [
           'addresiliency = slapos.recipe.addresiliency:Recipe',
           'accords = slapos.recipe.accords:Recipe',
-          'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
           'apacheperl = slapos.recipe.apacheperl:Recipe',
           'apachephp = slapos.recipe.apachephp:Recipe',
           'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
@@ -118,14 +117,11 @@ setup(name=name,
           'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
           'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
           'generic.varnish = slapos.recipe.generic_varnish:Recipe',
-          'generic.zope = slapos.recipe.generic_zope:Recipe',
-          'generic.zope.zeo.client = slapos.recipe.generic_zope_zeo_client:Recipe',
           'gitinit = slapos.recipe.gitinit:Recipe',
           'haproxy = slapos.recipe.haproxy:Recipe',
           'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
           'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
           'jsondump = slapos.recipe.jsondump:Recipe',
-          'kumofs = slapos.recipe.kumofs:Recipe',
           'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
           'lamp = slapos.recipe.lamp:Request',
           'lamp.generic = slapos.recipe.lampgeneric:Recipe',
@@ -136,7 +132,6 @@ setup(name=name,
           'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
           'logrotate = slapos.recipe.logrotate:Recipe',
           'logrotate.d = slapos.recipe.logrotate:Part',
-          'memcached = slapos.recipe.memcached:Recipe',
           'mkdirectory = slapos.recipe.mkdirectory:Recipe',
           'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
           'mydumper = slapos.recipe.mydumper:Recipe',
@@ -196,7 +191,6 @@ setup(name=name,
           'urlparse = slapos.recipe._urlparse:Recipe',
           'uuid = slapos.recipe._uuid:Recipe',
           'userinfo = slapos.recipe.userinfo:Recipe',
-          'waitfor = slapos.recipe.waitfor:Recipe',
           'webchecker = slapos.recipe.web_checker:Recipe',
           'wrapper = slapos.recipe.wrapper:Recipe',
           'xvfb = slapos.recipe.xvfb:Recipe',

slapos/recipe/accords/__init__.py

@@ -91,8 +91,8 @@ class Recipe(GenericSlapRecipe):
 
     # Generate wrapper
     wrapper_location = self.createPythonScript(self.options['accords-wrapper'],
-        '%s.accords.runAccords' % __name__,
-        parameter_dict)
+        __name__ + '.accords.runAccords',
+        (parameter_dict,))
     path_list.append(wrapper_location)
 
     # Generate helper for debug

slapos/recipe/addresiliency/__init__.py

@@ -38,26 +38,18 @@ class Recipe(GenericSlapRecipe):
     """
 
     def _install(self):
-        path_list = []
-
         slap_connection = self.buildout['slap-connection']
 
-        takeover_wrapper = self.createPythonScript(
-            name=self.options['wrapper-takeover'],
-            absolute_function='slapos.recipe.addresiliency.takeover.run',
-            arguments={
+        return self.createPythonScript(
+            self.options['wrapper-takeover'],
+            __name__ + '.takeover.takeover',
+            kw={
                 'server_url': slap_connection['server-url'],
                 'key_file': slap_connection.get('key-file'),
                 'cert_file': slap_connection.get('cert-file'),
-                'computer_id': slap_connection['computer-id'],
+                'computer_guid': slap_connection['computer-id'],
                 'partition_id': slap_connection['partition-id'],
-                'software': slap_connection['software-release-url'],
+                'software_release': slap_connection['software-release-url'],
                 'namebase': self.parameter_dict['namebase'],
                 'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
             })
-
-        path_list.append(takeover_wrapper)
-
-        return path_list
-
-

slapos/recipe/addresiliency/takeover.py

@@ -78,14 +78,3 @@ def takeover(server_url, key_file, cert_file, computer_guid,
   # Create "lock" file preventing equeue to run import scripts
   # XXX hardcoded
   open(takeover_triggered_file_path, 'w').write('')
-
-def run(args):
-  slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
-                                                key_file = args.pop('key_file'),
-                                                cert_file = args.pop('cert_file'),
-                                                computer_guid = args.pop('computer_id'),
-                                                partition_id = args.pop('partition_id'),
-                                                software_release = args.pop('software'),
-                                                namebase = args.pop('namebase'),
-                                                takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
-

slapos/recipe/apache_zope_backend/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import pkg_resources
-
-class Recipe(GenericBaseRecipe):
-  def install(self):
-    try:
-      backend_list = self.options['backend-list']
-    except KeyError:
-      backend_list = [(self.options['port'], self.options['backend'])]
-
-    scheme = self.options['scheme']
-    if scheme == 'http':
-      required_path_list = []
-      ssl_enable = ssl_snippet = ''
-    elif scheme == 'https':
-      key = self.options['key-file']
-      certificate = self.options['cert-file']
-      required_path_list = [key, certificate]
-      ssl_snippet = self.substituteTemplate(self.getTemplateFilename('snippet.ssl.in'), {
-        'key': key,
-        'certificate': certificate,
-        'ssl_session_cache': self.options['ssl-session-cache'],
-      })
-      if 'ssl-authentication' in self.options and self.optionIsTrue(
-          'ssl-authentication'):
-        ssl_snippet += self.substituteTemplate(self.getTemplateFilename('snippet.ssl.ca.in'), {
-          'ca_certificate': self.options['ssl-authentication-certificate'],
-          'ca_crl': self.options['ssl-authentication-crl'],
-        })
-      ssl_enable = 'SSLEngine on'
-    else:
-      raise ValueError('Unsupported scheme %s' % scheme)
-
-    ip_list = self.options['ip']
-    if isinstance(ip_list, basestring):
-      ip_list = [ip_list]
-    backend_path = self.options.get('backend-path', '/')
-    vhost_template_name = self.getTemplateFilename('vhost.in')
-    apache_config_file = self.createFile(
-      self.options['configuration-file'],
-      self.substituteTemplate(
-        self.getTemplateFilename('apache.zope.conf.in'),
-        {
-          'path': '/',
-          'server_admin': 'admin@',
-          'pid_file': self.options['pid-file'],
-          'lock_file': self.options['lock-file'],
-          'error_log': self.options['error-log'],
-          'access_log': self.options['access-log'],
-          'access_control_string': self.options['access-control-string'],
-          'ssl_snippet': ssl_snippet,
-          'vhosts': ''.join(self.substituteTemplate(vhost_template_name, {
-            'ip': ip,
-            'port': port,
-            'backend': ('%s/%s' % (backend.rstrip('/'), backend_path.strip('/'))).rstrip('/'),
-            'ssl_enable': ssl_enable,
-          }) for (port, backend) in backend_list for ip in ip_list),
-        },
-      )
-    )
-    return [
-      apache_config_file,
-      self.createPythonScript(
-        self.options['wrapper'],
-        __name__ + '.apache.runApache',
-        [
-          {
-            'required_path_list': required_path_list,
-            'binary': self.options['apache-binary'],
-            'config': apache_config_file,
-          },
-        ],
-      ),
-    ]

slapos/recipe/apache_zope_backend/apache.py

-import os
-import sys
-import time
-
-
-def runApache(args):
-  sleep = 60
-  conf = args[0]
-  while True:
-    ready = True
-    for f in conf.get('required_path_list', []):
-      if not os.path.exists(f):
-        print 'File %r does not exists, sleeping for %s' % (f, sleep)
-        ready = False
-    if ready:
-      break
-    time.sleep(sleep)
-  apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND']
-  apache_wrapper_list.extend(sys.argv[1:])
-  sys.stdout.flush()
-  sys.stderr.flush()
-  os.execl(apache_wrapper_list[0], *apache_wrapper_list)

slapos/recipe/apache_zope_backend/template/apache.zope.conf.in

-# Apache configuration file for Zope
-# Automatically generated
-
-# List of modules
-LoadModule unixd_module modules/mod_unixd.so
-LoadModule access_compat_module modules/mod_access_compat.so
-LoadModule authz_core_module modules/mod_authz_core.so
-LoadModule authz_host_module modules/mod_authz_host.so
-LoadModule log_config_module modules/mod_log_config.so
-LoadModule setenvif_module modules/mod_setenvif.so
-LoadModule version_module modules/mod_version.so
-LoadModule proxy_module modules/mod_proxy.so
-LoadModule proxy_http_module modules/mod_proxy_http.so
-LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
-LoadModule ssl_module modules/mod_ssl.so
-LoadModule mime_module modules/mod_mime.so
-LoadModule dav_module modules/mod_dav.so
-LoadModule dav_fs_module modules/mod_dav_fs.so
-LoadModule negotiation_module modules/mod_negotiation.so
-LoadModule rewrite_module modules/mod_rewrite.so
-LoadModule headers_module modules/mod_headers.so
-
-# Basic server configuration
-PidFile "%(pid_file)s"
-ServerAdmin %(server_admin)s
-TypesConfig conf/mime.types
-AddType application/x-compress .Z
-AddType application/x-gzip .gz .tgz
-
-ServerTokens Prod
-ServerSignature Off
-TraceEnable Off
-
-# Apache 2.4's default value (60 seconds) can be a bit too short
-TimeOut 300
-
-# As backend is trusting REMOTE_USER header unset it always
-RequestHeader unset REMOTE_USER
-
-%(ssl_snippet)s
-
-# Log configuration
-ErrorLog "%(error_log)s"
-# Default apache log format with request time in microsecond at the end
-LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
-CustomLog "%(access_log)s" combined
-
-# Directory protection
-<Directory />
-    Options FollowSymLinks
-    AllowOverride None
-    Order deny,allow
-    Deny from all
-</Directory>
-
-# Path protected
-<Location %(path)s>
-  Order Deny,Allow
-  Deny from all
-  Allow from %(access_control_string)s
-</Location>
-
-# Magic of Zope related rewrite
-RewriteEngine On
-%(vhosts)s

slapos/recipe/apache_zope_backend/template/snippet.ssl.ca.in

-SSLVerifyClient require
-RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s
-SSLCACertificateFile %(ca_certificate)s
-SSLCARevocationPath %(ca_crl)s

slapos/recipe/apache_zope_backend/template/snippet.ssl.in

-SSLCertificateFile %(certificate)s
-SSLCertificateKeyFile %(key)s
-SSLRandomSeed startup builtin
-SSLRandomSeed connect builtin
-SSLProtocol all -SSLv2 -SSLv3
-SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
-SSLHonorCipherOrder on
-SSLSessionCache shmcb:%(ssl_session_cache)s(512000)
-SSLProxyEngine On

slapos/recipe/apache_zope_backend/template/vhost.in

-Listen %(ip)s:%(port)s
-<VirtualHost *:%(port)s>
-  %(ssl_enable)s
-  RewriteRule ^/(.*) %(backend)s/$1 [L,P]
-</VirtualHost>

slapos/recipe/apacheperl/__init__.py

@@ -57,10 +57,9 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        [self.options['httpd-binary'], '-f', self.options['httpd-conf'],
-         '-DFOREGROUND']
+    wrapper = self.createWrapper(self.options['wrapper'],
+        (self.options['httpd-binary'], '-f', self.options['httpd-conf'],
+          '-DFOREGROUND'),
     )
     path_list.append(wrapper)
 

slapos/recipe/apachephp/__init__.py

@@ -92,14 +92,13 @@ class Recipe(GenericBaseRecipe):
       )
       path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND'
-                                     ],
-                                 environment=self.environ)
+                                     ),
+                                 self.environ)
     path_list.append(wrapper)
 
     secret_key_filename = os.path.join(self.buildout['buildout']['directory'],

slapos/recipe/apachephpconfigure/__init__.py

@@ -118,7 +118,7 @@ class Recipe(GenericBaseRecipe):
     configureinstall_wrapper_path = self.createPythonScript(
         self.options['configureinstall-location'],
         __name__ + '.runner.executeRunner',
-        [argument, delete, rename, chmod, data]
+        (argument, delete, rename, chmod, data)
     )
     
     #TODO finish to port this and remove upper one

slapos/recipe/apachephpconfigure/runner.py

 import subprocess
 
-def executeRunner(args):
+def executeRunner(arguments, delete, rename, chmod, data):
   """Start the instance configure. this may run a python script, move or/and rename
   file or directory when dondition is filled. the condition may be when file exist or when an entry
   exist into database.
   """
-  arguments, delete, rename, chmod, data = args
-  if delete != []:
+  if delete:
     print "Calling lampconfigure with 'delete' arguments"
-    result = subprocess.Popen(arguments + delete)
-    result.wait()
-  if rename != []:
+    subprocess.call(arguments + delete)
+  if rename:
     for parameters in rename:
       print "Calling lampconfigure with 'rename' arguments"
-      result = subprocess.Popen(arguments + parameters)
-      result.wait()
-  if chmod != []:
+      subprocess.call(arguments + parameters)
+  if chmod:
     print "Calling lampconfigure with 'chmod' arguments"
-    result = subprocess.Popen(arguments + chmod)
-    result.wait()
-  if data != []:
+    subprocess.call(arguments + chmod)
+  if data:
     print "Calling lampconfigure with 'run' arguments"
     print arguments + data
-    result = subprocess.Popen(arguments + data)
-    result.wait()
-    return
+    subprocess.call(arguments + data)
 

slapos/recipe/apacheproxy/__init__.py

@@ -49,13 +49,12 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND',
-                                     ])
+                                     ))
 
     path_list.append(wrapper)
 

slapos/recipe/boinc/__init__.py

@@ -124,12 +124,15 @@ class Recipe(GenericBaseRecipe):
 
     #Generate wrapper for php
     wrapperphp = os.path.join(self.home, 'bin/php')
-    php_wrapper = self.createPythonScript(wrapperphp,
-        'slapos.recipe.librecipe.execute.executee',
-        ([self.phpbin, '-c', self.phpini], os.environ)
+    php_wrapper = self.createWrapper(wrapperphp,
+        (self.phpbin, '-c', self.phpini),
     )
     path_list.append(php_wrapper)
 
+    mysql_dict = dict(db=self.database,
+        host=self.mysqlhost, port=self.mysqlport,
+        user=self.username, passwd=self.password)
+
     #Generate python script for MySQL database test (starting)
     file_status = os.path.join(self.home, '.boinc_config')
     if os.path.exists(file_status):
@@ -137,11 +140,7 @@ class Recipe(GenericBaseRecipe):
     mysql_wrapper = self.createPythonScript(
       os.path.join(self.wrapperdir, 'start_config'),
       '%s.configure.checkMysql' % __name__,
-      dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
-          mysql_user=self.username, mysql_password=self.password,
-          database=self.database,
-          file_status=file_status, environment=environment
-      )
+      (environment, mysql_dict, file_status)
     )
 
     # Generate make project wrapper file
@@ -164,8 +163,7 @@ class Recipe(GenericBaseRecipe):
     install_wrapper = self.createPythonScript(
       os.path.join(self.wrapperdir, 'make_project'),
       '%s.configure.makeProject' % __name__,
-      dict(launch_args=launch_args, request_file=request_make_boinc,
-      make_sig=file_status, env=environment)
+      (file_status, launch_args, request_make_boinc, environment)
     )
     path_list.append(install_wrapper)
 
@@ -197,7 +195,7 @@ class Recipe(GenericBaseRecipe):
     )
     start_service = self.createPythonScript(
       os.path.join(self.wrapperdir, 'config_project'),
-      '%s.configure.services' % __name__, parameter
+      '%s.configure.services' % __name__, (parameter,)
     )
     path_list.append(start_service)
 
@@ -208,14 +206,12 @@ class Recipe(GenericBaseRecipe):
       os.unlink(start_boinc)
     boinc_parameter = dict(service_status=service_status,
         installroot=self.installroot, drop_install=drop_install,
-        mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
-        mysql_user=self.username, mysql_password=self.password,
-        database=self.database, environment=environment,
+        mysql_dict=mysql_dict, environment=environment,
         start_boinc=start_boinc)
     start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
         'start_boinc'),
         '%s.configure.restart_boinc' % __name__,
-        boinc_parameter
+        (boinc_parameter,)
     )
     path_list.append(start_wrapper)
 
@@ -362,7 +358,7 @@ class App(GenericBaseRecipe):
         )
         deploy_app = self.createPythonScript(
           os.path.join(wrapperdir, 'boinc_%s' % appname),
-          '%s.configure.deployApp' % __name__, parameter
+          '%s.configure.deployApp' % __name__, (parameter,)
         )
         path_list.append(deploy_app)
 
@@ -404,17 +400,15 @@ class Client(GenericBaseRecipe):
       cc_cmd = '--read_cc_config'
     cmd = self.createPythonScript(cmd_wrapper,
         '%s.configure.runCmd' % __name__,
-        dict(base_cmd=base_cmd, cc_cmd=cc_cmd, installdir=installdir,
-        project_url=url, key=key)
+        (base_cmd, cc_cmd, installdir, url, key)
     )
     path_list.append(cmd)
 
     #Generate BOINC client wrapper
-    boinc = self.createPythonScript(boinc_wrapper,
-            'slapos.recipe.librecipe.execute.execute',
-            [boincbin, '--allow_multiple_clients', '--gui_rpc_port',
+    boinc = self.createWrapper(boinc_wrapper,
+            (boincbin, '--allow_multiple_clients', '--gui_rpc_port',
               str(self.options['rpc-port']), '--allow_remote_gui_rpc',
-              '--dir', installdir, '--redirectio', '--check_all_logins']
+              '--dir', installdir, '--redirectio', '--check_all_logins'),
     )
     path_list.append(boinc)
 

slapos/recipe/boinc/configure.py

@@ -35,27 +35,21 @@ import filecmp
 
 from lock_file import LockFile
 
-def checkMysql(args):
-  sys.path += args['environment']['PYTHONPATH'].split(':')
+def checkMysql(environment, connect_kw, file_status=None):
+  sys.path += environment['PYTHONPATH'].split(':')
   import MySQLdb
   #Sleep until mysql server becomes available
   while True:
     try:
-      conn = MySQLdb.connect(host = args['mysql_host'],
-                          user = args['mysql_user'],
-                          port = int(args['mysql_port']),
-                          passwd = args['mysql_password'],
-                          db = args['database'])
-      conn.close()
-      print "Successfully connect to MySQL database... "
-      if args.has_key('file_status'):
-        writeFile(args['file_status'], "starting")
+      MySQLdb.connect(**connect_kw).close()
       break
     except Exception, ex:
       print "The result is: \n" + ex.message
       print "Could not connect to MySQL database... sleep for 2 secondes"
       time.sleep(2)
-
+  print "Successfully connect to MySQL database... "
+  if file_status:
+    writeFile(file_status, "starting")
 
 def checkFile(file, stime):
   """Loop until 'file' is created (exist)"""
@@ -70,18 +64,16 @@ def checkFile(file, stime):
 
 def restart_boinc(args):
   """Stop (if currently is running state) and start all Boinc service"""
+  environment = args['environment']
   if args['drop_install']:
     checkFile(args['service_status'], 3)
   else:
-    checkMysql(args)
+    checkMysql(environment, args['mysql_dict'], args.get('file_status'))
   print "Restart Boinc..."
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
-  binstart = os.path.join(args['installroot'], 'bin/start')
-  binstop = os.path.join(args['installroot'], 'bin/stop')
-  os.system(binstop)
-  os.system(binstart)
+  env = os.environ.copy()
+  env.update(environment)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
   writeFile(args['start_boinc'], "started")
   print "Done."
 
@@ -122,17 +114,16 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
     return False
   return True
 
-def makeProject(args):
+def makeProject(make_sig, launch_args, request_file, extra_environ):
   """Run BOINC make_project script but once only"""
   #Wait for DateBase initialization...
-  checkFile(args['make_sig'], 3)
+  checkFile(make_sig, 3)
   print "Cheking if needed to run BOINC make_project..."
-  if os.path.exists(args['request_file']):
-    env = os.environ
-    env['PATH'] = args['env']['PATH']
-    env['PYTHONPATH'] = args['env']['PYTHONPATH']
-    if startProcess(args['launch_args'], env=env):
-      os.unlink(args['request_file'])
+  if os.path.exists(request_file):
+    env = os.environ.copy()
+    env.update(extra_environ)
+    if startProcess(launch_args, env=env):
+      os.unlink(request_file)
     print "Finished running BOINC make_projet...Ending"
   else:
     print "No new request for make_project. Exiting..."
@@ -155,9 +146,8 @@ def services(args):
     return
 
   print "execute script xadd..."
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  env = os.environ.copy()
+  env.update(args['environment'])
   if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
     return
   print "Update files and directories permissions..."
@@ -212,9 +202,8 @@ def deployManagement(args):
     newInstall = True
   #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
   checkFile(args['start_boinc'], 3)
-  env = os.environ
-  env['PATH'] = args['environment']['PATH']
-  env['PYTHONPATH'] = args['environment']['PYTHONPATH']
+  env = os.environ.copy()
+  env.update(args['environment'])
 
   print "setup directories..."
   numversion = args['version'].replace('.', '')
@@ -263,7 +252,7 @@ def deployManagement(args):
     privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
     output = open(binary + '.sig', 'w')
     p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
-            stderr=subprocess.STDOUT)
+            stderr=subprocess.STDOUT, env=env)
     result = p_sign.communicate()[0]
     if p_sign.returncode is None or p_sign.returncode != 0:
       print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
@@ -290,10 +279,8 @@ def deployManagement(args):
   create_wu(args, env)
 
   print "Restart Boinc..."
-  binstart = os.path.join(args['installroot'], 'bin/start')
-  binstop = os.path.join(args['installroot'], 'bin/stop')
-  os.system(binstop)
-  os.system(binstart)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
+  subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
 
   print "Boinc Application deployment is done... writing end signal file..."
   writeFile(token, str(args['wu_number']))
@@ -315,22 +302,21 @@ def create_wu(args, env):
     startProcess(launch_args, env, args['installroot'])
 
 
-def runCmd(args):
+def runCmd(base_cmd, cc_cmd, installdir, url, key):
   """Wait for Boinc Client started and run boinc cmd"""
-  client_config = os.path.join(args['installdir'], 'client_state.xml')
+  client_config = os.path.join(installdir, 'client_state.xml')
   checkFile(client_config, 5)
   time.sleep(10)
   #Scan client state xml to find client ipv4 adress
   host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
                 open(client_config, 'r').read()).group(1)
-  args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
-  print "Run boinccmd with host at %s " % args['base_cmd'][2]
-  project_args = args['base_cmd'] + ['--project_attach', args['project_url'],
-                      args['key']]
-  startProcess(project_args, cwd=args['installdir'])
-  if args['cc_cmd'] != '':
+  base_cmd[2] = host + ':' + base_cmd[2]
+  print "Run boinccmd with host at %s " % base_cmd[2]
+  project_args = base_cmd + ['--project_attach', url, key]
+  startProcess(project_args, cwd=installdir)
+  if cc_cmd:
     #Load or reload cc_config file
-    startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
+    startProcess(base_cmd + [cc_cmd], cwd=installdir)
 
 
 def writeFile(file, content):

slapos/recipe/bonjourgrid/__init__.py

@@ -62,8 +62,8 @@ class Recipe(GenericBaseRecipe):
                       condor_wrapper_list=condor_wrapper_list,
                       boinc_wrapper_list=boinc_wrapper_list)
     bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
-        '%s.configure.launchScript' % __name__,
-        parameters
+        __name__ + '.configure.launchScript',
+        (parameters,)
     )
     path_list.append(bonjourGrid_wrapper)
 
@@ -73,16 +73,15 @@ class Recipe(GenericBaseRecipe):
     bg_wrapper = self.options['wrapper'].strip()
     log = self.options['log_file'].strip()
     pid_file = self.options['pid_file'].strip()
-    wrapper = self.createPythonScript(bg_wrapper,
-        'slapos.recipe.librecipe.execute.execute',
-        ([python, bonjourgrid_master, '--log_file', log,
+    wrapper = self.createWrapper(bg_wrapper,
+        (python, bonjourgrid_master, '--log_file', log,
           '--pid_file', pid_file,
           '--master_wrapper', grid_wrapper,
           '--directory', self.options['work_dir'].strip(),
           '--server', self.options['redis-url'].strip(),
           '--port', self.options['redis-port'].strip(),
           '--num_workers', self.options['nworkers'].strip(),
-        ])
+         ),
     )
     path_list.append(wrapper)
 
@@ -113,9 +112,8 @@ class Client(GenericBaseRecipe):
     bg_wrapper = self.options['wrapper'].strip()
     log = self.options['log_file'].strip()
     pid_file = self.options['pid_file'].strip()
-    wrapper = self.createPythonScript(bg_wrapper,
-        'slapos.recipe.librecipe.execute.execute',
-        ([python, bonjourgrid_client, '--log_file', log,
+    wrapper = self.createWrapper(bg_wrapper,
+        (python, bonjourgrid_client, '--log_file', log,
           '--pid_file', pid_file,
           '--boinc_wrapper', boinc_script,
           '--condor_wrapper', condor_script,
@@ -123,7 +121,7 @@ class Client(GenericBaseRecipe):
           '--install_directory', self.options['install_dir'].strip(),
           '--server', self.options['redis-url'].strip(),
           '--port', self.options['redis-port'].strip(),
-        ])
+         ),
     )
     path_list.append(wrapper)
 

slapos/recipe/certificate_authority/__init__.py

@@ -40,13 +40,10 @@ class Recipe(GenericBaseRecipe):
     self.ca_private = self.options['ca-private']
     self.ca_certs = self.options['ca-certs']
     self.ca_newcerts = self.options['ca-newcerts']
-    self.ca_crl = self.options['ca-crl']
     self.ca_key_ext = '.key'
     self.ca_crt_ext = '.crt'
 
   def install(self):
-    path_list = []
-
     ca_country_code = self.options.get('country-code', 'XX')
     ca_email = self.options.get('email', 'xx@example.com')
     # XXX-BBB: State by mistake has been configured as string "('State',)"
@@ -77,21 +74,15 @@ class Recipe(GenericBaseRecipe):
     self.createFile(openssl_configuration, self.substituteTemplate(
       self.getTemplateFilename('openssl.cnf.ca.in'), config))
 
-    ca_wrapper = self.createPythonScript(
+    return self.createPythonScript(
       self.options['wrapper'],
-      '%s.certificate_authority.runCertificateAuthority' % __name__,
-      dict(
-        openssl_configuration=openssl_configuration,
-        openssl_binary=self.options['openssl-binary'],
-        certificate=os.path.join(self.ca_dir, 'cacert.pem'),
-        key=os.path.join(self.ca_private, 'cakey.pem'),
-        crl=self.ca_crl,
-        request_dir=self.request_directory
-      )
+      __name__ + '.certificate_authority.runCertificateAuthority',
+      (os.path.join(self.ca_private, 'cakey.pem'),
+       os.path.join(self.ca_dir, 'cacert.pem'),
+       self.options['openssl-binary'],
+       openssl_configuration,
+       self.request_directory)
     )
-    path_list.append(ca_wrapper)
-
-    return path_list
 
 class Request(Recipe):
 
@@ -146,11 +137,10 @@ class Request(Recipe):
 
     path_list = [key_file, cert_file]
     if request_needed:
-      wrapper = self.createPythonScript(
+      wrapper = self.createWrapper(
         self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute_wait',
-        [ [self.options['executable']],
-          [certificate, key] ],
+        (self.options['executable'],),
+        wait_list=(certificate, key),
       )
       path_list.append(wrapper)
 

slapos/recipe/certificate_authority/certificate_authority.py

@@ -102,10 +102,8 @@ class CertificateAuthority:
             'certificate_file')):
         print 'Created certificate %r' % parser.get('certificate', 'name')
 
-def runCertificateAuthority(ca_conf):
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
+def runCertificateAuthority(*args):
+  ca = CertificateAuthority(*args)
   while True:
     ca.checkAuthority()
     ca.checkRequestDir()

slapos/recipe/cloud9/__init__.py

@@ -42,8 +42,6 @@ class Recipe(GenericBaseRecipe):
     options['access-url'] = 'http://[%s]:%s' % (self.ip, self.port)
 
   def install(self):
-    path_list = []
-
     environment = {
         'PATH': os.path.dirname(self.git) + ':' + os.environ['PATH'],
     }
@@ -51,10 +49,4 @@ class Recipe(GenericBaseRecipe):
     cloud9_args = [self.node_executable, self.cloud9, '-l', self.ip, '-p',
         self.port, '-w', self.workdir]
 
-    wrapper = self.createPythonScript(self.wrapper,
-        'slapos.recipe.librecipe.execute.executee',
-        (cloud9_args, environment)
-    )
-    path_list.append(wrapper)
-
-    return path_list
+    return self.createWrapper(self.wrapper, cloud9_args, environment)

slapos/recipe/condor/__init__.py

@@ -178,12 +178,11 @@ class Recipe(GenericBaseRecipe):
       os.chmod(wrapper_location, 0744)
 
     #generate script for start condor
-    start_condor = os.path.join(self.wrapperdir, 'start_condor')
-    start_bin = os.path.join(self.wrapper_sbin, 'condor_master')
-    condor_reconfig = os.path.join(self.wrapper_sbin, 'condor_reconfig')
-    wrapper = self.createPythonScript(start_condor,
-        '%s.configure.condorStart' % __name__,
-        dict(start_bin=start_bin, condor_reconfig=condor_reconfig)
+    wrapper = self.createPythonScript(
+        os.path.join(self.wrapperdir, 'start_condor'),
+        __name__ + '.configure.condorStart',
+        (os.path.join(self.wrapper_sbin, 'condor_reconfig'),
+         os.path.join(self.wrapper_sbin, 'condor_master'))
     )
     path_list.append(wrapper)
     return path_list
@@ -276,13 +275,11 @@ class AppSubmit(GenericBaseRecipe):
           os.unlink(destination)
         os.symlink(app_list[appname]['files'][file], destination)
       #generate wrapper for submitting job
-      condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
-      parameter = dict(submit=condor_submit, sig_install=sig_install,
-                      submit_file='submit',
-                      appname=appname, appdir=appdir)
       submit_job = self.createPythonScript(
         os.path.join(self.options['wrapper-dir'].strip(), appname),
-        '%s.configure.submitJob' % __name__, parameter
+        __name__ + '.configure.submitJob',
+        (os.path.join(self.options['bin'].strip(), 'condor_submit'),
+         'submit', appdir, appname, sig_install)
       )
       path_list.append(submit_job)
     return path_list
\ No newline at end of file

slapos/recipe/condor/configure.py

@@ -29,27 +29,25 @@ import os
 import subprocess
 import time
 
-def submitJob(args):
+def submitJob(submit, submit_file, appdir, appname, sig_install):
   """Run condor_submit (if needed) for job deployment"""
   time.sleep(10)
-  print "Check if needed to submit %s job's" % args['appname']
-  if not os.path.exists(args['sig_install']):
+  print "Check if needed to submit %s job's" % appname
+  if not os.path.exists(sig_install):
     print "Nothing for install or update...Exited"
     return
   # '-a', "log = out.log", '-a', "error = error.log",
-  launch_args = [args['submit'], '-verbose', args['submit_file']]
+  launch_args = submit, '-verbose', submit_file
   process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
-              stderr=subprocess.STDOUT, cwd=args['appdir'])
+              stderr=subprocess.STDOUT, cwd=appdir)
   result = process.communicate()[0]
   if process.returncode is None or process.returncode != 0:
     print "Failed to execute condor_submit.\nThe error was: %s" % result
   else:
-    os.unlink(args['sig_install'])
+    os.unlink(sig_install)
 
-def condorStart(args):
+def condorStart(condor_reconfig, start_bin):
   """Start Condor if deamons is currently stopped"""
-  result = os.system(args['condor_reconfig'])
-  if result != 0:
+  if subprocess.call(condor_reconfig):
     #process failled to reconfig condor that mean that condor deamons is not curently started
-    os.system(args['start_bin'])
-  
+    subprocess.call(start_bin)
\ No newline at end of file

slapos/recipe/davstorage/__init__.py

@@ -98,26 +98,19 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(config_file)
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
+    wrapper = self.createWrapper(self.options['wrapper'],
+      (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
     path_list.append(wrapper)
 
     promise = self.createPythonScript(self.options['promise'],
       __name__ + '.promise',
-      dict(host=self.options['ip'], port=int(self.options['port_webdav']),
-           user=self.options['user'], password=self.options['password'])
-                                     )
+      (self.options['ip'], int(self.options['port_webdav']),
+       self.options['user'], self.options['password']))
     path_list.append(promise)
 
     return path_list
 
-def promise(args):
-  host = args['host']
-  port = args['port']
-  user = args['user']
-  password = args['password']
-
+def promise(host, port, user, password):
   connection = httplib.HTTPSConnection(host, port)
   auth = base64.b64encode('%s:%s' % (user, password))
   connection.request('OPTIONS', '/',

slapos/recipe/dcron.py

@@ -35,15 +35,14 @@ class Recipe(GenericBaseRecipe):
     self.logger.info("Installing dcron...")
 
     options = self.options
-    script = self.createWrapper(name=options['binary'],
-                                command=options['dcrond-binary'].strip(),
-                                parameters=[
+    script = self.createWrapper(options['binary'],
+                                (options['dcrond-binary'].strip(),
                                     '-s', options['cron-entries'],
                                     '-c', options['crontabs'],
                                     '-t', options['cronstamps'],
                                     '-f', '-l', '5',
                                     '-M', options['catcher']
-                                    ])
+                                    ))
 
     self.logger.debug('Main cron executable created at : %r', script)
 

slapos/recipe/dropbear.py

@@ -57,8 +57,6 @@ class KnownHostsFile(dict):
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    path_list = []
-
     dropbear_cmd = [self.options['dropbear-binary']]
     # Don't fork into background
     dropbear_cmd.append('-F')
@@ -95,19 +93,12 @@ class Recipe(GenericBaseRecipe):
     if 'shell' in self.options:
       env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell']
 
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.executee',
-      (dropbear_cmd, env, )
-    )
-    path_list.append(wrapper)
-
-    return path_list
+    return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
 
 class Client(GenericBaseRecipe):
 
   def install(self):
-    env = dict()
+    env = {}
 
     if 'home' in self.options:
       env['HOME'] = self.options['home']
@@ -120,13 +111,7 @@ class Client(GenericBaseRecipe):
     if 'identity-file' in self.options:
       dropbear_cmd.extend(['-i', self.options['identity-file']])
 
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.executee',
-      (dropbear_cmd, env, )
-    )
-
-    return [wrapper]
+    return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
 
 
 class AddAuthorizedKey(GenericBaseRecipe):

slapos/recipe/duplicity.py

@@ -46,7 +46,4 @@ class Recipe(GenericBaseRecipe):
       cmd.extend(options)
       cmd.extend([backup_directory, remote_url])
 
-    wrapper = self.createPythonScript(self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute', cmd)
-
-    return [wrapper]
+    return self.createWrapper(self.options['wrapper'], cmd)

slapos/recipe/equeue.py

@@ -30,23 +30,20 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
 
-    parameters = [
+    args = [
+      self.options['equeue-binary'],
       '--database', self.options['database'],
       '--logfile', self.options['log'],
       '--lockfile', self.options['lockfile']
     ]
 
     if 'takeover-triggered-file-path' in self.options:
-      parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']])
+      args += ('--takeover-triggered-file-path',
+               self.options['takeover-triggered-file-path'])
 
     if 'loglevel' in self.options:
-      parameters.extend(['--loglevel', self.options['loglevel']])
+      args += '--loglevel', self.options['loglevel']
 
-    parameters.append(self.options['socket'])
-
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['equeue-binary'],
-                                 parameters=parameters)
-
-    return [wrapper]
+    args.append(self.options['socket'])
 
+    return self.createWrapper(self.options['wrapper'], args)

slapos/recipe/erp5_test/__init__.py

@@ -67,7 +67,7 @@ class Recipe(GenericBaseRecipe):
         openssl_binary=self.options['openssl-binary'],
         test_ca_path=self.options['certificate-authority-path'],
     )
-    common_list = [
+    common_list = (
       '--conversion_server_url=' + cloudooo_url,
       # BBB: We still have test suites that only accept the following 2 options.
       '--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
@@ -76,19 +76,19 @@ class Recipe(GenericBaseRecipe):
       '--volatile_memcached_server_port=%s' % memcached_parsed.port,
       '--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname,
       '--persistent_memcached_server_port=%s' % kumofs_parsed.port,
-    ]
+    )
     path_list.append(self.createPythonScript(self.options['run-unit-test'],
-        __name__ + '.test.runUnitTest', [dict(
-        call_list=[self.options['run-unit-test-binary'],
+        __name__ + '.test.runUnitTest',
+        ((self.options['run-unit-test-binary'],
           '--erp5_sql_connection_string', mysql_connection_string,
           '--extra_sql_connection_string_list', ','.join(
             mysql_connection_string_list),
-          ] + common_list, **common_dict)]))
+          ) + common_list, common_dict)))
     path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runUnitTest', [dict(
-        call_list=[self.options['run-test-suite-binary'],
+        __name__ + '.test.runTestSuite',
+        ((self.options['run-test-suite-binary'],
           '--db_list', ','.join(mysql_connection_string_list),
-          ] + common_list, **common_dict)]))
+          ) + common_list, common_dict)))
 
     return path_list
 
@@ -98,20 +98,18 @@ class CloudoooRecipe(GenericBaseRecipe):
     common_dict = dict(
         prepend_path=self.options['prepend-path'],
     )
-    common_list = [
+    common_list = (
            "--paster_path", self.options['ooo-paster'],
            self.options['configuration-file']
-          ]
-    run_unit_test_path = self.createPythonScript(self.options['run-unit-test'],
-        __name__ + '.test.runUnitTest', [dict(
-        call_list=[self.options['run-unit-test-binary'],
-          ] + common_list, **common_dict)])
-
-    path_list.append(run_unit_test_path)
+          )
+    path_list.append(self.createPythonScript(self.options['run-unit-test'],
+        __name__ + '.test.runUnitTest',
+        ((self.options['run-unit-test-binary'],
+          ) + common_list, common_dict)))
     path_list.append(self.createPythonScript(self.options['run-test-suite'],
-        __name__ + '.test.runTestSuite', [dict(
-        call_list=[self.options['run-test-suite-binary'],
-          ], **common_dict)]))
+        __name__ + '.test.runTestSuite',
+        ((self.options['run-test-suite-binary'],
+          ), common_dict)))
 
     return path_list
 
@@ -121,32 +119,20 @@ class EggTestRecipe(GenericBaseRecipe):
   off a list of Python eggs.
   """
   def install(self):
-    path_list = []
     test_list = self.options['test-list'].strip().replace('\n', ',')
-    common_dict = {}
 
-    environment_dict = {}
+    common_dict = {}
     if self.options.get('environment'):
       environment_part = self.buildout.get(self.options['environment'])
       if environment_part:
-        for key, value in environment_part.iteritems():
-          environment_dict[key] = value
+        common_dict['environment'] = dict(environment_part)
 
-    common_list = [ "--source_code_path_list", test_list]
-
-    argument_dict = dict(
-        call_list=[self.options['run-test-suite-binary'],] + common_list,
-        environment=environment_dict,
-        **common_dict
-    )
     if 'prepend-path' in self.options:
-      argument_dict['prepend_path'] = self.options['prepend-path']
+      common_dict['prepend_path'] = self.options['prepend-path']
 
-    run_test_suite_script = self.createPythonScript(
+    return self.createPythonScript(
         self.options['run-test-suite'], __name__ + '.test.runTestSuite',
-        [argument_dict]
+        ((self.options['run-test-suite-binary'],
+          "--source_code_path_list", test_list),
+         common_dict)
     )
-
-    path_list.append(run_test_suite_script)
-
-    return path_list

slapos/recipe/erp5_test/test.py

@@ -26,15 +26,18 @@
 ##############################################################################
 import os
 import sys
-def runTestSuite(args):
+
+def runTestSuite(args, d):
   env = os.environ.copy()
-  d = args[0]
   if 'openssl_binary' in d:
     env['OPENSSL_BINARY'] = d['openssl_binary']
   if 'test_ca_path' in d:
     env['TEST_CA_PATH'] = d['test_ca_path']
   if 'prepend_path' in d:
-    env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
+    try:
+      env['PATH'] = d['prepend_path'] + ':' + env['PATH']
+    except KeyError:
+      env['PATH'] = d['prepend_path']
   if 'instance_home' in d:
     env['INSTANCE_HOME'] = d['instance_home']
     env['REAL_INSTANCE_HOME'] = d['instance_home']
@@ -44,42 +47,15 @@ def runTestSuite(args):
     env.update(d['environment'])
 
   # Deal with Shebang size limitation
-  executable_filepath = d['call_list'][0]
-  file_object = open(executable_filepath, 'r')
-  line = file_object.readline()
-  file_object.close()
-  argument_list = []
-  if line[:2] == '#!':
-    executable_filepath = line[2:].strip()
-    argument_list.append(executable_filepath)
-  argument_list.extend(d['call_list'])
-  argument_list.extend(sys.argv[1:])
-  argument_list.append(env)
-  os.execle(executable_filepath, *argument_list)
-
-def runUnitTest(args):
-  env = os.environ.copy()
-  d = args[0]
-  if 'openssl_binary' in d:
-    env['OPENSSL_BINARY'] = d['openssl_binary']
-  if 'test_ca_path' in d:
-    env['TEST_CA_PATH'] = d['test_ca_path']
-  if 'prepend_path' in d:
-    env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
-  if 'instance_home' in d:
-    env['INSTANCE_HOME'] = d['instance_home']
-    env['REAL_INSTANCE_HOME'] = d['instance_home']
-  # Deal with Shebang size limitation
-  executable_filepath = d['call_list'][0]
-  file_object = open(executable_filepath, 'r')
-  line = file_object.readline()
-  file_object.close()
+  executable_filepath = args[0]
+  with open(executable_filepath, 'r') as f:
+    line = f.readline()
   argument_list = []
   if line[:2] == '#!':
     executable_filepath = line[2:].strip()
     argument_list.append(executable_filepath)
-  argument_list.extend(d['call_list'])
-  argument_list.extend(sys.argv[1:])
-  argument_list.append(env)
-  os.execle(executable_filepath, *argument_list)
+  argument_list += args
+  argument_list += sys.argv[1:]
+  os.execve(executable_filepath, argument_list, env)
 
+runUnitTest = runTestSuite

slapos/recipe/erp5testnode/__init__.py

@@ -68,17 +68,14 @@ class Recipe(GenericBaseRecipe):
     )
     self.path_list.append(configuration_file)
     self.path_list.append(
-      self.createPythonScript(
-        self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.executee',
-        [ # Executable
-          [ self.options['testnode'], '-l', self.options['log-file'],
-            configuration_file],
+      self.createWrapper(self.options['wrapper'],
+          # Command
+          ( self.options['testnode'], '-l', self.options['log-file'],
+            configuration_file),
           # Environment
           {
             'GIT_SSL_NO_VERIFY': '1',
           }
-        ],
       )
     )
     self.installApache()
@@ -106,9 +103,8 @@ class Recipe(GenericBaseRecipe):
                                apache_config)
     )
     self.path_list.append(config_file)
-    wrapper = self.createPythonScript(self.options['httpd-wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
+    wrapper = self.createWrapper(self.options['httpd-wrapper'],
+      (self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
     self.path_list.append(wrapper)
     # create empty html page to not allow listing of /
     page = open(os.path.join(self.options['log-directory'], "index.html"), "w")

slapos/recipe/generic_cloudooo/__init__.py

@@ -118,5 +118,5 @@ class Recipe(GenericBaseRecipe):
     path_list.append(config_file)
     path_list.append(self.createPythonScript(self.options['wrapper'],
      'slapos.recipe.librecipe.execute.execute_with_signal_translation',
-      [self.options['ooo-paster'].strip(), 'serve', config_file]))
+      ((self.options['ooo-paster'].strip(), 'serve', config_file),)))
     return path_list

slapos/recipe/generic_mysql/__init__.py

@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
        )]
     )
     path_list.append(mysqld)
-    environment = dict(PATH='%s' % self.options['bin-directory'])
+    environment = {'PATH': self.options['bin-directory']}
     # TODO: move to a separate recipe (ack'ed by Cedric)
     if 'backup-script' in self.options:
       # backup configuration
@@ -165,9 +165,13 @@ class Recipe(GenericBaseRecipe):
           '--defaults-file=%s' % mysql_conf_file,
           '--socket=%s' % socket.strip(), '--user=root',
           '--ibbackup=%s'% self.options['xtrabackup-binary']]
-      innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment])
+      innobackupex_incremental = self.createWrapper(
+        self.options['innobackupex-incremental'],
+        innobackupex_argument_list + ['--incremental'], environment)
       path_list.append(innobackupex_incremental)
-      innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment])
+      innobackupex_full = self.createWrapper(
+        self.options['innobackupex-full'],
+        innobackupex_argument_list, environment)
       path_list.append(innobackupex_full)
       backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
       path_list.append(backup_controller)
@@ -215,7 +219,9 @@ class Recipe(GenericBaseRecipe):
           '--defaults-file=%s' % mysql_conf_file,
           '--socket=%s' % socket.strip(), '--user=root',
           ]
-      pt_exe = self.createPythonScript(os.path.join(self.options['bin-directory'], pt_script_name), 'slapos.recipe.librecipe.execute.executee', [pt_argument_list, environment])
+      pt_exe = self.createWrapper(
+        os.path.join(self.options['bin-directory'], pt_script_name),
+        pt_argument_list, environment)
       path_list.append(pt_exe)
 
     return path_list

slapos/recipe/generic_mysql/innobackupex.py

 import os
 import glob
-def controller(args):
+def controller(innobackupex_incremental, innobackupex_full,
+               full_backup, incremental_backup):
   """Creates full or incremental backup
 
   If no full backup is done, it is created
@@ -9,8 +10,6 @@ def controller(args):
 
   base is the newest (according to date) full or incremental backup
   """
-  innobackupex_incremental, innobackupex_full, full_backup, incremental_backup \
-      = args
   if len(os.listdir(full_backup)) == 0:
     print 'Doing full backup in %r' % full_backup
     os.execv(innobackupex_full, [innobackupex_full, full_backup])

slapos/recipe/generic_mysql/mysql.py

@@ -5,9 +5,8 @@ import sys
 import pytz
 
 
-def runMysql(args):
+def runMysql(conf):
   sleep = 60
-  conf = args[0]
   mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
       conf['configuration_file']]
   # we trust mysql_install that if mysql directory is available mysql was
@@ -54,8 +53,7 @@ def runMysql(args):
   os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
 
 
-def updateMysql(args):
-  conf = args[0]
+def updateMysql(conf):
   sleep = 30
   is_succeed = False
   try:

slapos/recipe/generic_zope/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import binascii
-import hashlib
-import os
-import re
-import zc.buildout
-
-_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
-
-# based on Zope2.utilities.mkzopeinstance.write_inituser
-def Zope2InitUser(path, username, password):
-  # Set password only once
-  # Currently, rely on existence of a simple file:
-  # Create it the first time, then next time, detect this file and do no-op.
-  inituser_done_path = '%s_done' % path
-  if os.path.exists(inituser_done_path):
-    return
-
-  if os.path.exists(path):
-    return
-  open(path, 'w').write('')
-  os.chmod(path, 0600)
-  open(path, 'w').write('%s:{SHA}%s\n' % (
-    username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
-
-  open(inituser_done_path, 'w').write('"inituser" file already created once.')
-
-class Recipe(GenericBaseRecipe):
-  def _options(self, options):
-    options['password'] = self.generatePassword()
-    options['deadlock-password'] = self.generatePassword()
-
-  def install(self):
-    """
-    All zope have to share file created by portal_classes
-    (until everything is integrated into the ZODB).
-    So, do not request zope instance and create multiple in the same partition.
-    """
-    path_list = []
-    Zope2InitUser(self.options['inituser'], self.options['user'],
-      self.options['password'])
-
-    # Symlink to BT5 repositories defined in instance config.
-    # Those paths will eventually end up in the ZODB, and having symlinks
-    # inside the XXX makes it possible to reuse such ZODB with another software
-    # release[ version].
-    # Note: this path cannot be used for development, it's really just a
-    # read-only repository.
-    repository_path = self.options['bt5-repository']
-
-    self.bt5_repository_list = []
-    append = self.bt5_repository_list.append
-    for repository in self.options.get('bt5-repository-list', '').split():
-      repository = repository.strip()
-      if not repository:
-        continue
-
-      if _isurl(repository) and not repository.startswith("file://"):
-        # XXX: assume it's a valid URL
-        append(repository)
-        continue
-
-      if repository.startswith('file://'):
-        repository = repository.replace('file://', '', '')
-
-      if os.path.isabs(repository):
-        repo_id = hashlib.sha1(repository).hexdigest()
-        link = os.path.join(repository_path, repo_id)
-        if os.path.lexists(link):
-          if not os.path.islink(link):
-            raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-          os.unlink(link)
-        os.symlink(repository, link)
-        self.logger.debug('Created link %r -> %r' % (link, repository_path))
-        # Always provide a URL-Type
-        append("file://" + link)
-
-    # Create zope configuration file
-    zope_config = dict(
-        thread_amount=self.options['thread-amount'],
-        zodb_root_path=self.options['zodb-path'],
-        zodb_cache_size=int(self.options['zodb-cache-size']),
-    )
-    zope_environment = dict(
-      TMP=self.options['tmp-path'],
-      TMPDIR=self.options['tmp-path'],
-      HOME=self.options['tmp-path'],
-      PATH=self.options['bin-path']
-    )
-    # configure default Zope2 zcml
-    open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
-        'site.zcml')).read())
-    zope_config['instance'] = self.options['instance-path']
-    zope_config['event_log'] = self.options['event-log']
-    zope_config['z2_log'] = self.options['z2-log']
-    zope_config['pid-filename'] = self.options['pid-file']
-    zope_config['lock-filename'] = self.options['lock-file']
-    zope_config['products'] = 'products %s' % self.options['instance-products']
-    zope_config['address'] = '%s:%s' % (self.options['ip'], self.options['port'])
-    zope_config.update(dump_url=self.options['deadlock-path'],
-      secret=self.options['deadlock-password'])
-
-    zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
-    zope_conf_content = self.substituteTemplate(zope_wrapper_template_location,
-      zope_config)
-
-    if ('promise-path' in self.options) and ('site-id' in self.options):
-      zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
-          'zope.conf.promise.in'), {
-            'site-id': self.options['site-id'],
-            'promise-path': self.options['promise-path'],
-            })
-
-    zope_conf_path = self.createFile(self.options['configuration-file'], zope_conf_content)
-    path_list.append(zope_conf_path)
-    # Create init script
-    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', zope_conf_path], zope_environment]))
-    return path_list

slapos/recipe/generic_zope/template/site.zcml

-<configure
-    xmlns="http://namespaces.zope.org/zope"
-    xmlns:meta="http://namespaces.zope.org/meta"
-    xmlns:five="http://namespaces.zope.org/five">
-
-  <include package="Products.Five" />
-  <meta:redefinePermission from="zope2.Public" to="zope.Public" />
-
-
-  <!-- Load the meta -->
-  <include files="package-includes/*-meta.zcml" />
-  <five:loadProducts file="meta.zcml"/>
-
-  <!-- Load the configuration -->
-  <include files="package-includes/*-configure.zcml" />
-  <five:loadProducts />
-
-  <!-- Load the configuration overrides-->
-  <includeOverrides files="package-includes/*-overrides.zcml" />
-  <five:loadProductsOverrides />
-
-
-  <securityPolicy
-      component="Products.Five.security.FiveSecurityPolicy" />
-
-</configure>

slapos/recipe/generic_zope/template/zope.conf.in

-## Zope 2 configuration file generated by SlapOS
-
-# Some defines
-%%define INSTANCE %(instance)s
-instancehome $INSTANCE
-
-# Used products
-%(products)s
-
-# Environment is setup in running wrapper script
-# Reason: zope.conf is read too late for some componets
-
-# No need to debug
-debug-mode off
-
-# One thread is safe enough
-zserver-threads %(thread_amount)s
-
-# File location
-pid-filename %(pid-filename)s
-lock-filename %(lock-filename)s
-
-# Encoding
-rest-input-encoding utf-8
-rest-output-encoding utf-8
-default-zpublisher-encoding utf-8
-
-# Disable ownership checking to execute codes generated by alarm
-skip-ownership-checking on
-
-# Temporary storage database (for sessions)
-<zodb_db temporary>
-     <temporarystorage>
-       name temporary storage for sessioning
-     </temporarystorage>
-     mount-point /temp_folder
-     container-class Products.TemporaryFolder.TemporaryContainer
-</zodb_db>
-
-# Logging configuration
-<eventlog>
-  <logfile>
-    dateformat
-    path %(event_log)s
-  </logfile>
-</eventlog>
-<logger access>
-  <logfile>
-    dateformat
-    path %(z2_log)s
-  </logfile>
-</logger>
-
-# Serving configuration
-<http-server>
-  address %(address)s
-</http-server>
-
-# ZODB configuration
-<zodb_db root>
-    cache-size %(zodb_cache_size)d
-    <filestorage>
-      path %(zodb_root_path)s
-    </filestorage>
-    mount-point /
-</zodb_db>
-<zoperunner>
-    program $INSTANCE/bin/runzope
-</zoperunner>
-
-# DeadlockDebugger configuration
-<product-config DeadlockDebugger>
-  dump_url %(dump_url)s
-  secret %(secret)s
-</product-config>
-
-# ERP5 Timer Service
-%%import timerserver
-<timer-server>
-  interval 5
-</timer-server>

slapos/recipe/generic_zope/template/zope.conf.promise.in

-
-# ERP5 promise
-<product-config /%(site-id)s>
-  promise_path %(promise-path)s
-</product-config>

slapos/recipe/generic_zope_zeo_client/__init__.py

-##############################################################################
-#
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import binascii
-import hashlib
-import os
-import re
-import zc.buildout
-
-_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
-
-# based on Zope2.utilities.mkzopeinstance.write_inituser
-def Zope2InitUser(path, username, password):
-  # Set password only once
-  # Currently, rely on existence of a simple file:
-  # Create it the first time, then next time, detect this file and do no-op.
-  inituser_done_path = '%s_done' % path
-  if os.path.exists(inituser_done_path):
-    return
-
-  if os.path.exists(path):
-    return
-  open(path, 'w').write('')
-  os.chmod(path, 0600)
-  open(path, 'w').write('%s:{SHA}%s\n' % (
-    username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
-
-  open(inituser_done_path, 'w').write('"inituser" file already created once.')
-
-class Recipe(GenericBaseRecipe):
-  def _options(self, options):
-    if 'password' not in options:
-      options['password'] = self.generatePassword()
-
-  def install(self):
-    """
-    All zope have to share file created by portal_classes
-    (until everything is integrated into the ZODB).
-    So, do not request zope instance and create multiple in the same partition.
-    """
-    path_list = []
-    Zope2InitUser(self.options['inituser'], self.options['user'],
-      self.options['password'])
-
-    # Symlink to BT5 repositories defined in instance config.
-    # Those paths will eventually end up in the ZODB, and having symlinks
-    # inside the XXX makes it possible to reuse such ZODB with another software
-    # release[ version].
-    # Note: this path cannot be used for development, it's really just a
-    # read-only repository.
-    repository_path = self.options['bt5-repository']
-
-    self.bt5_repository_list = []
-    append = self.bt5_repository_list.append
-    for repository in self.options.get('bt5-repository-list', '').split():
-      repository = repository.strip()
-      if not repository:
-        continue
-
-      if _isurl(repository) and not repository.startswith("file://"):
-        # XXX: assume it's a valid URL
-        append(repository)
-        continue
-
-      if repository.startswith('file://'):
-        repository = repository.replace('file://', '', '')
-
-      if os.path.isabs(repository):
-        repo_id = hashlib.sha1(repository).hexdigest()
-        link = os.path.join(repository_path, repo_id)
-        if os.path.lexists(link):
-          if not os.path.islink(link):
-            raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-          os.unlink(link)
-        os.symlink(repository, link)
-        self.logger.debug('Created link %r -> %r' % (link, repository_path))
-        # Always provide a URL-Type
-        append("file://" + link)
-
-    zope_environment = {
-      'TMP': self.options['tmp-path'],
-      'TMPDIR': self.options['tmp-path'],
-      'HOME': self.options.get('home-path', self.options.get('tmp-path')),
-      'PATH': self.options['bin-path'],
-      'TZ': self.options['timezone'],
-    }
-    instance_home = self.options.get("instancehome-path", None)
-    if instance_home:
-      zope_environment["INSTANCE_HOME"] = instance_home
-
-    # configure default Zope2 zcml
-    open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
-        'site.zcml')).read())
-
-    # Create init script
-    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', self.options['configuration-file']], zope_environment]))
-    return path_list

slapos/recipe/generic_zope_zeo_client/template/site.zcml

-<configure
-    xmlns="http://namespaces.zope.org/zope"
-    xmlns:meta="http://namespaces.zope.org/meta"
-    xmlns:five="http://namespaces.zope.org/five">
-
-  <include package="Products.Five" />
-  <meta:redefinePermission from="zope2.Public" to="zope.Public" />
-
-
-  <!-- Load the meta -->
-  <include files="package-includes/*-meta.zcml" />
-  <five:loadProducts file="meta.zcml"/>
-
-  <!-- Load the configuration -->
-  <include files="package-includes/*-configure.zcml" />
-  <five:loadProducts />
-
-  <!-- Load the configuration overrides-->
-  <includeOverrides files="package-includes/*-overrides.zcml" />
-  <five:loadProductsOverrides />
-
-
-  <securityPolicy
-      component="Products.Five.security.FiveSecurityPolicy" />
-
-</configure>

slapos/recipe/haproxy/__init__.py

@@ -120,12 +120,11 @@ class Recipe(GenericBaseRecipe):
          'server_text': server_snippet},
       )
     )
-    wrapper_path = self.createPythonScript(
+    wrapper_path = self.createWrapper(
       self.options['wrapper-path'],
-      'slapos.recipe.librecipe.execute.execute',
-      arguments=[self.options['binary-path'].strip(), '-f', configuration_path],)
+      (self.options['binary-path'].strip(), '-f', configuration_path))
     ctl_path = self.createPythonScript(
       self.options['ctl-path'],
-      '%s.haproxy.haproxyctl' % __name__,
-      {'socket_path':self.options['socket-path']})
+      __name__ + '.haproxy.haproxyctl',
+      (self.options['socket-path'],))
     return [configuration_path, wrapper_path, ctl_path]

slapos/recipe/haproxy/haproxy.py

@@ -4,7 +4,7 @@ try:
 except ImportError:
   pass
 
-def haproxyctl(conf):
+def haproxyctl(socket_path):
   while True:
     try:
       l = raw_input('> ')
@@ -14,7 +14,7 @@ def haproxyctl(conf):
     if l == 'quit':
       break
     s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    s.connect(conf['socket_path'])
+    s.connect(socket_path)
     s.send('%s\n' % l)
     while True:
       r = s.recv(1024)

slapos/recipe/kumofs/__init__.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import BaseSlapRecipe
-import hashlib
-import os
-import pkg_resources
-import sys
-import zc.buildout
-import ConfigParser
-
-class Recipe(BaseSlapRecipe):
-  def getTemplateFilename(self, template_name):
-    return pkg_resources.resource_filename(__name__,
-        'template/%s' % template_name)
-
-  def _install(self):
-    self.path_list = []
-    self.requirements, self.ws = self.egg.working_set()
-    # XXX-Cedric : add logrotate?
-    self.cron_d = self.installCrond()
-    kumo_conf = self.installKumo(self.getLocalIPv4Address())
-
-    ca_conf = self.installCertificateAuthority()
-    key, certificate = self.requestCertificate('Login Based Access')
-
-    stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
-        self.getLocalIPv4Address(), 12345, kumo_conf['kumo_gateway_port'],
-        certificate, key, ca_conf['ca_crl'],
-        ca_conf['certificate_authority_path'])
-
-    self.linkBinary()
-    self.setConnectionDict(dict(
-      stunnel_ip = stunnel_conf['public_ip'],
-      stunnel_port = stunnel_conf['public_port'],
-    ))
-    return self.path_list
-
-  def linkBinary(self):
-    """Links binaries to instance's bin directory for easier exposal"""
-    for linkline in self.options.get('link_binary_list', '').splitlines():
-      if not linkline:
-        continue
-      target = linkline.split()
-      if len(target) == 1:
-        target = target[0]
-        path, linkname = os.path.split(target)
-      else:
-        linkname = target[1]
-        target = target[0]
-      link = os.path.join(self.bin_directory, linkname)
-      if os.path.lexists(link):
-        if not os.path.islink(link):
-          raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-        os.unlink(link)
-      os.symlink(target, link)
-      self.logger.debug('Created link %r -> %r' % (link, target))
-      self.path_list.append(link)
-
-
-  def installCrond(self):
-    timestamps = self.createDataDirectory('cronstamps')
-    cron_output = os.path.join(self.log_directory, 'cron-output')
-    self._createDirectory(cron_output)
-    catcher = zc.buildout.easy_install.scripts([('catchcron',
-      __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
-      self.bin_directory, arguments=[cron_output])[0]
-    self.path_list.append(catcher)
-    cron_d = os.path.join(self.etc_directory, 'cron.d')
-    crontabs = os.path.join(self.etc_directory, 'crontabs')
-    self._createDirectory(cron_d)
-    self._createDirectory(crontabs)
-    # Use execute from erp5.
-    wrapper = zc.buildout.easy_install.scripts([('crond',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
-        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
-      )[0]
-    self.path_list.append(wrapper)
-    return cron_d
-
-  def installLogrotate(self):
-    """Installs logortate main configuration file and registers its to cron"""
-    logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
-      'logrotate.d'))
-    self._createDirectory(logrotate_d)
-    logrotate_backup = self.createBackupDirectory('logrotate')
-    logrotate_conf = self.createConfigurationFile("logrotate.conf",
-        "include %s" % logrotate_d)
-    logrotate_cron = os.path.join(self.cron_d, 'logrotate')
-    state_file = os.path.join(self.data_root_directory, 'logrotate.status')
-    open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
-        (self.options['logrotate_binary'], state_file, logrotate_conf))
-    self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
-    return logrotate_d, logrotate_backup
-
-  def registerLogRotation(self, name, log_file_list, postrotate_script):
-    """Register new log rotation requirement"""
-    open(os.path.join(self.logrotate_d, name), 'w').write(
-        self.substituteTemplate(self.getTemplateFilename(
-          'logrotate_entry.in'),
-          dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
-            postrotate=postrotate_script, olddir=self.logrotate_backup)))
-
-  def installCertificateAuthority(self, ca_country_code='XX',
-      ca_email='xx@example.com', ca_state='State', ca_city='City',
-      ca_company='Company'):
-    backup_path = self.createBackupDirectory('ca')
-    self.ca_dir = os.path.join(self.data_root_directory, 'ca')
-    self._createDirectory(self.ca_dir)
-    self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
-    self._createDirectory(self.ca_request_dir)
-    config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
-    self.ca_private = os.path.join(self.ca_dir, 'private')
-    self.ca_certs = os.path.join(self.ca_dir, 'certs')
-    self.ca_crl = os.path.join(self.ca_dir, 'crl')
-    self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
-    self.ca_key_ext = '.key'
-    self.ca_crt_ext = '.crt'
-    for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
-      self._createDirectory(d)
-    for f in ['crlnumber', 'serial']:
-      if not os.path.exists(os.path.join(self.ca_dir, f)):
-        open(os.path.join(self.ca_dir, f), 'w').write('01')
-    if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
-      open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
-    openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
-    config.update(
-        working_directory=self.ca_dir,
-        country_code=ca_country_code,
-        state=ca_state,
-        city=ca_city,
-        company=ca_company,
-        email_address=ca_email,
-    )
-    self._writeFile(openssl_configuration, pkg_resources.resource_string(
-      __name__, 'template/openssl.cnf.ca.in') % config)
-    self.path_list.extend(zc.buildout.easy_install.scripts([
-      ('certificate_authority',
-        __name__ + '.certificate_authority', 'runCertificateAuthority')],
-        self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
-          openssl_configuration=openssl_configuration,
-          openssl_binary=self.options['openssl_binary'],
-          certificate=os.path.join(self.ca_dir, 'cacert.pem'),
-          key=os.path.join(self.ca_private, 'cakey.pem'),
-          crl=os.path.join(self.ca_crl),
-          request_dir=self.ca_request_dir
-          )]))
-    # configure backup
-    backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
-    open(backup_cron, 'w').write(
-        '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
-          rdiff_backup=self.options['rdiff_backup_binary'],
-          source=self.ca_dir,
-          destination=backup_path))
-    self.path_list.append(backup_cron)
-
-    return dict(
-      ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
-      ca_crl=os.path.join(config['ca_dir'], 'crl'),
-      certificate_authority_path=config['ca_dir']
-    )
-
-  def requestCertificate(self, name):
-    hash = hashlib.sha512(name).hexdigest()
-    key = os.path.join(self.ca_private, hash + self.ca_key_ext)
-    certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
-    parser = ConfigParser.RawConfigParser()
-    parser.add_section('certificate')
-    parser.set('certificate', 'name', name)
-    parser.set('certificate', 'key_file', key)
-    parser.set('certificate', 'certificate_file', certificate)
-    parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
-    return key, certificate
-
-  def installStunnel(self, public_ip, private_ip, public_port, private_port,
-      ca_certificate, key, ca_crl, ca_path):
-    """Installs stunnel"""
-    template_filename = self.getTemplateFilename('stunnel.conf.in')
-    log = os.path.join(self.log_directory, 'stunnel.log')
-    pid_file = os.path.join(self.run_directory, 'stunnel.pid')
-    stunnel_conf = dict(
-        public_ip=public_ip,
-        private_ip=private_ip,
-        public_port=public_port,
-        pid_file=pid_file,
-        log=log,
-        cert = ca_certificate,
-        key = key,
-        ca_crl = ca_crl,
-        ca_path = ca_path,
-        private_port = private_port,
-    )
-    stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
-        self.substituteTemplate(template_filename,
-          stunnel_conf))
-    wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[
-        [self.options['stunnel_binary'].strip(), stunnel_conf_path],
-        [ca_certificate, key]]
-      )[0]
-    self.path_list.append(wrapper)
-
-    return stunnel_conf
-
-  def installKumo(self, ip, kumo_manager_port=13101, kumo_server_port=13201,
-      kumo_server_listen_port=13202, kumo_gateway_port=13301):
-    # XXX: kumo is not storing pid in file, unless it is not running as daemon
-    #      but running daemons is incompatible with SlapOS, so there is currently
-    #      no way to have Kumo's pid files to rotate logs and send signals to them
-    config = dict(
-      kumo_gateway_binary=self.options['kumo_gateway_binary'],
-      kumo_gateway_ip=ip,
-      kumo_gateway_log=os.path.join(self.log_directory, "kumo-gateway.log"),
-      kumo_manager_binary=self.options['kumo_manager_binary'],
-      kumo_manager_ip=ip,
-      kumo_manager_log=os.path.join(self.log_directory, "kumo-manager.log"),
-      kumo_server_binary=self.options['kumo_server_binary'],
-      kumo_server_ip=ip,
-      kumo_server_log=os.path.join(self.log_directory, "kumo-server.log"),
-      kumo_server_storage=os.path.join(self.data_root_directory, "kumodb.tch"),
-      kumo_manager_port=kumo_manager_port,
-      kumo_server_port=kumo_server_port,
-      kumo_server_listen_port=kumo_server_listen_port,
-      kumo_gateway_port=kumo_gateway_port
-    )
-
-    self.path_list.append(self.createRunningWrapper('kumo_gateway',
-      self.substituteTemplate(self.getTemplateFilename('kumo_gateway.in'),
-        config)))
-
-    self.path_list.append(self.createRunningWrapper('kumo_manager',
-      self.substituteTemplate(self.getTemplateFilename('kumo_manager.in'),
-        config)))
-
-    self.path_list.append(self.createRunningWrapper('kumo_server',
-      self.substituteTemplate(self.getTemplateFilename('kumo_server.in'),
-        config)))
-
-    return dict(
-      kumo_address = '%s:%s' % (config['kumo_gateway_ip'],
-        config['kumo_gateway_port']),
-      kumo_gateway_ip=config['kumo_gateway_ip'],
-      kumo_gateway_port=config['kumo_gateway_port'],
-    )

slapos/recipe/kumofs/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions',
-          'v3_ca', '-keyout', self.key, '-out', self.certificate,
-          '-days', '10950'], 'Automatic Certificate Authority\n')
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/kumofs/template/kumo_gateway.in

-#!/bin/sh
-exec %(kumo_gateway_binary)s -F -E -m %(kumo_manager_ip)s:%(kumo_manager_port)s -t %(kumo_gateway_ip)s:%(kumo_gateway_port)s -o %(kumo_gateway_log)s

slapos/recipe/kumofs/template/kumo_manager.in

-#!/bin/sh
-exec %(kumo_manager_binary)s -a -l %(kumo_manager_ip)s:%(kumo_manager_port)s -o %(kumo_manager_log)s

slapos/recipe/kumofs/template/kumo_server.in

-#!/bin/sh
-exec %(kumo_server_binary)s -l %(kumo_server_ip)s:%(kumo_server_port)s -L %(kumo_server_listen_port)s -m %(kumo_manager_ip)s:%(kumo_manager_port)s -s %(kumo_server_storage)s -o %(kumo_server_log)s

slapos/recipe/kumofs/template/openssl.cnf.ca.in

-#
-# OpenSSL example configuration file.
-# This is mostly being used for generation of certificate requests.
-#
-
-# This definition stops the following lines choking if HOME isn't
-# defined.
-HOME			= .
-RANDFILE		= $ENV::HOME/.rnd
-
-# Extra OBJECT IDENTIFIER info:
-#oid_file		= $ENV::HOME/.oid
-oid_section		= new_oids
-
-# To use this configuration file with the "-extfile" option of the
-# "openssl x509" utility, name here the section containing the
-# X.509v3 extensions to use:
-# extensions		= 
-# (Alternatively, use a configuration file that has only
-# X.509v3 extensions in its main [= default] section.)
-
-[ new_oids ]
-
-# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
-# Add a simple OID like this:
-# testoid1=1.2.3.4
-# Or use config file substitution like this:
-# testoid2=${testoid1}.5.6
-
-# Policies used by the TSA examples.
-tsa_policy1 = 1.2.3.4.1
-tsa_policy2 = 1.2.3.4.5.6
-tsa_policy3 = 1.2.3.4.5.7
-
-####################################################################
-[ ca ]
-default_ca	= CA_default		# The default ca section
-
-####################################################################
-[ CA_default ]
-
-dir		= %(working_directory)s		# Where everything is kept
-certs		= $dir/certs		# Where the issued certs are kept
-crl_dir		= $dir/crl		# Where the issued crl are kept
-database	= $dir/index.txt	# database index file.
-#unique_subject	= no			# Set to 'no' to allow creation of
-					# several ctificates with same subject.
-new_certs_dir	= $dir/newcerts		# default place for new certs.
-
-certificate	= $dir/cacert.pem 	# The CA certificate
-serial		= $dir/serial 		# The current serial number
-crlnumber	= $dir/crlnumber	# the current crl number
-					# must be commented out to leave a V1 CRL
-crl		= $dir/crl.pem 		# The current CRL
-private_key	= $dir/private/cakey.pem # The private key
-RANDFILE	= $dir/private/.rand	# private random number file
-
-x509_extensions	= usr_cert		# The extentions to add to the cert
-
-# Comment out the following two lines for the "traditional"
-# (and highly broken) format.
-name_opt 	= ca_default		# Subject Name options
-cert_opt 	= ca_default		# Certificate field options
-
-# Extension copying option: use with caution.
-# copy_extensions = copy
-
-# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
-# so this is commented out by default to leave a V1 CRL.
-# crlnumber must also be commented out to leave a V1 CRL.
-# crl_extensions	= crl_ext
-
-default_days	= 3650			# how long to certify for
-default_crl_days= 30			# how long before next CRL
-default_md	= default		# use public key default MD
-preserve	= no			# keep passed DN ordering
-
-# A few difference way of specifying how similar the request should look
-# For type CA, the listed attributes must be the same, and the optional
-# and supplied fields are just that :-)
-policy		= policy_match
-
-# For the CA policy
-[ policy_match ]
-countryName		= match
-stateOrProvinceName	= match
-organizationName	= match
-organizationalUnitName	= optional
-commonName		= supplied
-emailAddress		= optional
-
-# For the 'anything' policy
-# At this point in time, you must list all acceptable 'object'
-# types.
-[ policy_anything ]
-countryName		= optional
-stateOrProvinceName	= optional
-localityName		= optional
-organizationName	= optional
-organizationalUnitName	= optional
-commonName		= supplied
-emailAddress		= optional
-
-####################################################################
-[ req ]
-default_bits		= 2048
-default_md		= sha1
-default_keyfile 	= privkey.pem
-distinguished_name	= req_distinguished_name
-#attributes		= req_attributes
-x509_extensions	= v3_ca	# The extentions to add to the self signed cert
-
-# Passwords for private keys if not present they will be prompted for
-# input_password = secret
-# output_password = secret
-
-# This sets a mask for permitted string types. There are several options. 
-# default: PrintableString, T61String, BMPString.
-# pkix	 : PrintableString, BMPString (PKIX recommendation before 2004)
-# utf8only: only UTF8Strings (PKIX recommendation after 2004).
-# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
-# MASK:XXXX a literal mask value.
-# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
-string_mask = utf8only
-
-# req_extensions = v3_req # The extensions to add to a certificate request
-
-[ req_distinguished_name ]
-countryName			= Country Name (2 letter code)
-countryName_value		= %(country_code)s
-countryName_min			= 2
-countryName_max			= 2
-
-stateOrProvinceName		= State or Province Name (full name)
-stateOrProvinceName_value	= %(state)s
-
-localityName			= Locality Name (eg, city)
-localityName_value		= %(city)s
-
-0.organizationName		= Organization Name (eg, company)
-0.organizationName_value	= %(company)s
-
-# we can do this but it is not needed normally :-)
-#1.organizationName		= Second Organization Name (eg, company)
-#1.organizationName_default	= World Wide Web Pty Ltd
-
-commonName			= Common Name (eg, your name or your server\'s hostname)
-commonName_max			= 64
-
-emailAddress			= Email Address
-emailAddress_value = %(email_address)s
-emailAddress_max		= 64
-
-# SET-ex3			= SET extension number 3
-
-#[ req_attributes ]
-#challengePassword		= A challenge password
-#challengePassword_min		= 4
-#challengePassword_max		= 20
-#
-#unstructuredName		= An optional company name
-
-[ usr_cert ]
-
-# These extensions are added when 'ca' signs a request.
-
-# This goes against PKIX guidelines but some CAs do it and some software
-# requires this to avoid interpreting an end user certificate as a CA.
-
-basicConstraints=CA:FALSE
-
-# Here are some examples of the usage of nsCertType. If it is omitted
-# the certificate can be used for anything *except* object signing.
-
-# This is OK for an SSL server.
-# nsCertType			= server
-
-# For an object signing certificate this would be used.
-# nsCertType = objsign
-
-# For normal client use this is typical
-# nsCertType = client, email
-
-# and for everything including object signing:
-# nsCertType = client, email, objsign
-
-# This is typical in keyUsage for a client certificate.
-# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-# This will be displayed in Netscape's comment listbox.
-nsComment			= "OpenSSL Generated Certificate"
-
-# PKIX recommendations harmless if included in all certificates.
-subjectKeyIdentifier=hash
-authorityKeyIdentifier=keyid,issuer
-
-# This stuff is for subjectAltName and issuerAltname.
-# Import the email address.
-# subjectAltName=email:copy
-# An alternative to produce certificates that aren't
-# deprecated according to PKIX.
-# subjectAltName=email:move
-
-# Copy subject details
-# issuerAltName=issuer:copy
-
-#nsCaRevocationUrl		= http://www.domain.dom/ca-crl.pem
-#nsBaseUrl
-#nsRevocationUrl
-#nsRenewalUrl
-#nsCaPolicyUrl
-#nsSslServerName
-
-# This is required for TSA certificates.
-# extendedKeyUsage = critical,timeStamping
-
-[ v3_req ]
-
-# Extensions to add to a certificate request
-
-basicConstraints = CA:FALSE
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-[ v3_ca ]
-
-
-# Extensions for a typical CA
-
-
-# PKIX recommendation.
-
-subjectKeyIdentifier=hash
-
-authorityKeyIdentifier=keyid:always,issuer
-
-# This is what PKIX recommends but some broken software chokes on critical
-# extensions.
-#basicConstraints = critical,CA:true
-# So we do this instead.
-basicConstraints = CA:true
-
-# Key usage: this is typical for a CA certificate. However since it will
-# prevent it being used as an test self-signed certificate it is best
-# left out by default.
-# keyUsage = cRLSign, keyCertSign
-
-# Some might want this also
-# nsCertType = sslCA, emailCA
-
-# Include email address in subject alt name: another PKIX recommendation
-# subjectAltName=email:copy
-# Copy issuer details
-# issuerAltName=issuer:copy
-
-# DER hex encoding of an extension: beware experts only!
-# obj=DER:02:03
-# Where 'obj' is a standard or added object
-# You can even override a supported extension:
-# basicConstraints= critical, DER:30:03:01:01:FF
-
-[ crl_ext ]
-
-# CRL extensions.
-# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
-
-# issuerAltName=issuer:copy
-authorityKeyIdentifier=keyid:always
-
-[ proxy_cert_ext ]
-# These extensions should be added when creating a proxy certificate
-
-# This goes against PKIX guidelines but some CAs do it and some software
-# requires this to avoid interpreting an end user certificate as a CA.
-
-basicConstraints=CA:FALSE
-
-# Here are some examples of the usage of nsCertType. If it is omitted
-# the certificate can be used for anything *except* object signing.
-
-# This is OK for an SSL server.
-# nsCertType			= server
-
-# For an object signing certificate this would be used.
-# nsCertType = objsign
-
-# For normal client use this is typical
-# nsCertType = client, email
-
-# and for everything including object signing:
-# nsCertType = client, email, objsign
-
-# This is typical in keyUsage for a client certificate.
-# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-# This will be displayed in Netscape's comment listbox.
-nsComment			= "OpenSSL Generated Certificate"
-
-# PKIX recommendations harmless if included in all certificates.
-subjectKeyIdentifier=hash
-authorityKeyIdentifier=keyid,issuer
-
-# This stuff is for subjectAltName and issuerAltname.
-# Import the email address.
-# subjectAltName=email:copy
-# An alternative to produce certificates that aren't
-# deprecated according to PKIX.
-# subjectAltName=email:move
-
-# Copy subject details
-# issuerAltName=issuer:copy
-
-#nsCaRevocationUrl		= http://www.domain.dom/ca-crl.pem
-#nsBaseUrl
-#nsRevocationUrl
-#nsRenewalUrl
-#nsCaPolicyUrl
-#nsSslServerName
-
-# This really needs to be in place for it to be a proxy certificate.
-proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
-
-####################################################################
-[ tsa ]
-
-default_tsa = tsa_config1	# the default TSA section
-
-[ tsa_config1 ]
-
-# These are used by the TSA reply generation only.
-dir		= /etc/pki/tls		# TSA root directory
-serial		= $dir/tsaserial	# The current serial number (mandatory)
-crypto_device	= builtin		# OpenSSL engine to use for signing
-signer_cert	= $dir/tsacert.pem 	# The TSA signing certificate
-					# (optional)
-certs		= $dir/cacert.pem	# Certificate chain to include in reply
-					# (optional)
-signer_key	= $dir/private/tsakey.pem # The TSA private key (optional)
-
-default_policy	= tsa_policy1		# Policy if request did not specify it
-					# (optional)
-other_policies	= tsa_policy2, tsa_policy3	# acceptable policies (optional)
-digests		= md5, sha1		# Acceptable message digests (mandatory)
-accuracy	= secs:1, millisecs:500, microsecs:100	# (optional)
-clock_precision_digits  = 0	# number of digits after dot. (optional)
-ordering		= yes	# Is ordering defined for timestamps?
-				# (optional, default: no)
-tsa_name		= yes	# Must the TSA name be included in the reply?
-				# (optional, default: no)
-ess_cert_id_chain	= no	# Must the ESS cert id chain be included?
-				# (optional, default: no)

slapos/recipe/kumofs/template/stunnel.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-CApath = %(ca_path)s
-key = %(key)s
-CRLpath = %(ca_crl)s
-cert = %(cert)s
-
-[service]
-accept = %(public_ip)s:%(public_port)s
-connect = %(private_ip)s:%(private_port)s

slapos/recipe/lamp/__init__.py

@@ -298,9 +298,9 @@ class Request(BaseRecipe):
         'local_host': local_host, 'local_port': local_port,
       }))
     wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[
-        self.options['stunnel_binary'].strip(), stunnel_conf_path]
+      'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
+      sys.executable, self.wrapper_directory, arguments='%r, %r'
+        % (self.options['stunnel_binary'].strip(), stunnel_conf_path)
       )[0]
     self.path_list.append(wrapper)
     return (local_host, local_port,)

slapos/recipe/lampgeneric/__init__.py

@@ -71,13 +71,12 @@ class Recipe(GenericBaseRecipe):
     )
     path_list.append(httpd_conf)
 
-    wrapper = self.createWrapper(name=self.options['wrapper'],
-                                 command=self.options['httpd-binary'],
-                                 parameters=[
+    wrapper = self.createWrapper(self.options['wrapper'],
+                                 (self.options['httpd-binary'],
                                      '-f',
                                      self.options['httpd-conf'],
                                      '-DFOREGROUND'
-                                     ])
+                                     ))
     path_list.append(wrapper)
 
 

slapos/recipe/librecipe/execute.py

@@ -2,7 +2,6 @@ import sys
 import os
 import signal
 import subprocess
-import time
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
@@ -29,65 +28,96 @@ def _wait_files_creation(file_list):
         if event.name in directory:
           directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
-def execute(args):
-  """Portable execution with process replacement"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args, None, None])
-
-def execute_wait(args):
-  """Execution but after all files in args[1] exists"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args[0], args[1], None])
-
+def _libc():
+  from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
+  libc = CDLL(util.find_library('c'), use_errno=True)
+  libc_mount = libc.mount
+  libc_mount.argtypes = c_char_p, c_char_p, c_char_p, c_ulong, c_char_p
+  def mount(source, target, filesystemtype, mountflags, data):
+    if libc_mount(source, target, filesystemtype, mountflags, data):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  libc_unshare = libc.unshare
+  libc_unshare.argtypes = c_int,
+  def unshare(flags):
+    if libc_unshare(flags):
+      e = get_errno()
+      raise OSError(e, os.strerror(e))
+  return mount, unshare
+
+def generic_exec(args, extra_environ=None, wait_list=None,
+                 pidfile=None, reserve_cpu=False, private_dev_shm=None,
+                 #shebang_workaround=False, # XXX: still needed ?
+                 ):
+  args = list(args)
+
+  if pidfile:
+    import psutil
+    try:
+      with open(pidfile) as f:
+        pid = int(f.read())
+      running = psutil.Process(pid).cmdline()
+    except Exception:
+      pass
+    else:
+      # With chained shebangs, several paths may be inserted at the beginning.
+      n = len(args)
+      for i in xrange(1+len(running)-n):
+        if args == running[i:n+i]:
+          sys.exit("Already running with pid %s." % pid)
+    with open(pidfile, 'w') as f:
+      f.write(str(os.getpid()))
+
+  args += sys.argv[1:]
+
+  if reserve_cpu:
+    # If the CGROUPS cpuset is available (and prepared by slap format),
+    # request an exclusive CPU core for this process.
+    with open(os.path.expanduser('~/.slapos-cpu-exclusive'), 'a') as f:
+      f.write('%s\n' % os.getpid())
+
+  if wait_list:
+    _wait_files_creation(wait_list)
+
+  if private_dev_shm:
+    mount, unshare = _libc()
+    CLONE_NEWNS   = 0x00020000
+    CLONE_NEWUSER = 0x10000000
+    uid = os.getuid()
+    gid = os.getgid()
+    unshare(CLONE_NEWUSER |CLONE_NEWNS)
+    with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
+    with open('/proc/self/uid_map',   'wb') as f: f.write('%s %s 1' % (uid, uid))
+    with open('/proc/self/gid_map',   'wb') as f: f.write('%s %s 1' % (gid, gid))
+    mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
+
+  if extra_environ:
+    env = os.environ.copy()
+    env.update(extra_environ)
+    os.execve(args[0], args, env)
+  else:
+    os.execv(args[0], args)
 
 child_pg = None
 
-
-def executee(args):
-  """Portable execution with process replacement and environment manipulation"""
-  # XXX: Kept for backward compatibility
-  generic_exec([args[0], None, args[1]])
-
-def executee_wait(args):
-  """Portable execution with process replacement and environment manipulation"""
-  # XXX: Kept for backward compatibility
-  generic_exec(args)
-
-def generic_exec(args):
-  exec_list = list(args[0])
-  file_list = args[1]
-  environment_overriding = args[2]
-
-  exec_env = os.environ.copy()
-  if environment_overriding is not None:
-      exec_env.update(environment_overriding)
-
-  if file_list is not None:
-      _wait_files_creation(file_list)
-
-  os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
-
 def sig_handler(sig, frame):
   print 'Received signal %r, killing children and exiting' % sig
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
-  sys.exit(0)
-
-signal.signal(signal.SIGINT, sig_handler)
-signal.signal(signal.SIGQUIT, sig_handler)
-signal.signal(signal.SIGTERM, sig_handler)
-
+  sys.exit()
 
 def execute_with_signal_translation(args):
   """Run process as children and translate from SIGTERM to another signal"""
   global child_pg
+  signal.signal(signal.SIGINT, sig_handler)
+  signal.signal(signal.SIGQUIT, sig_handler)
+  signal.signal(signal.SIGTERM, sig_handler)
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:
     print 'Process %r started' % args
-    while True:
-      time.sleep(10)
+    signal.pause()
   finally:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)

slapos/recipe/librecipe/generic.py

@@ -33,7 +33,6 @@ import sys
 import inspect
 import re
 import shutil
-from textwrap import dedent
 import urllib
 import urlparse
 
@@ -61,7 +60,13 @@ class GenericBaseRecipe(object):
     self._options(options) # Options Hook
     self.options = options.copy() # Updated options dict
 
-    self._ws = self.getWorkingSet()
+  @property
+  def _ws(self):
+    # getWorkingSet() is slow and it is not always needed.
+    # So _ws should be a lazy attribute.
+    if getattr(self, '_ws_internal', None) is None:
+      self._ws_internal = self.getWorkingSet()
+    return self._ws_internal
 
   def update(self):
     """By default update method does the same thing than install"""
@@ -110,92 +115,60 @@ class GenericBaseRecipe(object):
       with io.open(filepath, 'w+', encoding=encoding) as f:
         f.write(u'\n'.join(lines))
 
-  def createPythonScript(self, name, absolute_function, arguments=''):
+  def createPythonScript(self, name, absolute_function, args=(), kw={}):
     """Create a python script using zc.buildout.easy_install.scripts
 
      * function should look like 'module.function', or only 'function'
        if it is a builtin function."""
-    absolute_function = tuple(absolute_function.rsplit('.', 1))
-    if len(absolute_function) == 1:
-      absolute_function = ('__builtin__',) + absolute_function
-    if len(absolute_function) != 2:
-      raise ValueError("A non valid function was given")
-
-    module, function = absolute_function
+    function = absolute_function.rsplit('.', 1)
+    if len(function) == 1:
+      module = '__builtin__'
+      function, = function
+    else:
+      module, function = function
     path, filename = os.path.split(os.path.abspath(name))
 
-    script = zc.buildout.easy_install.scripts(
+    assert not isinstance(args, (basestring, dict)), args
+    args = map(repr, args)
+    args += map('%s=%r'.__mod__, kw.iteritems())
+
+    return zc.buildout.easy_install.scripts(
       [(filename, module, function)], self._ws, sys.executable,
-      path, arguments=arguments)[0]
-    return script
+      path, arguments=', '.join(args))[0]
 
-  def createWrapper(self, name, command, parameters, comments=[],
-      parameters_extra=False, environment=None,
-      pidfile=None, reserve_cpu=False
-  ):
-    """
-    Creates a shell script for process replacement.
-    Takes care of quoting.
-    Takes care of #! line limitation when the wrapped command is a script.
-    if pidfile parameter is specified, then it will make the wrapper a singleton,
-    accepting to run only if no other instance is running.
+  def createWrapper(self, path, args, env=None, **kw):
+    """Create a wrapper script for process replacement"""
+    assert args
+    if kw:
+      return self.createPythonScript(path,
+        'slapos.recipe.librecipe.execute.generic_exec',
+        (args, env) if env else (args,), kw)
 
-    :param reserve_cpu: bool, try to reserve one core for the `command`
-    """
+    # Simple case: creates a basic shell script for process replacement.
+    # This must be kept minimal to avoid code duplication with generic_exec.
+    # In particular, do not implement workaround for shebang size limitation
+    # here (note that this can't be done correctly with a POSIX shell, because
+    # the process can't be given a name).
+
+    lines = ['#!/bin/sh']
+
+    if env:
+      for k, v in sorted(env.iteritems()):
+        lines.append('export %s=%s' % (k, shlex.quote(v)))
+
+    lines.append('exec')
 
-    lines = [ '#!/bin/sh' ]
-
-    if comments:
-      lines += '# ', '\n# '.join(comments), '\n'
-
-    lines.append('COMMAND=' + shlex.quote(command))
-
-    for key in environment or ():
-      lines.append('export %s=%s' % (key, environment[key]))
-
-    if pidfile:
-      lines.append(dedent("""
-          # Check for other instances
-          pidfile=%s
-          if [ -s $pidfile ]; then
-            if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
-              echo "Already running with pid $pid."
-              exit 1
-            fi
-          fi
-          echo $$ > $pidfile""" % shlex.quote(pidfile)))
-
-    if reserve_cpu:
-      # if the CGROUPS cpuset is available (and prepared by slap format)
-      # request an exclusive CPU core for this process
-      lines.append(dedent("""
-        # put own PID into waiting list for exclusive CPU-core access
-        echo $$ >> ~/.slapos-cpu-exclusive
-        """))
-
-    lines.append(dedent('''
-    # If the wrapped command uses a shebang, execute the referenced
-    # executable passing the script path as first argument.
-    # This is to workaround the limitation of 127 characters in #!
-    [ ! -f "$COMMAND" ] || {
-      [ "`head -c2`" != "#!" ] || read -r EXE ARG
-    } < "$COMMAND"
-
-    exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
-
-    parameters = map(shlex.quote, parameters)
-    if parameters_extra:
-      # pass-through further parameters
-      parameters.append('"$@"')
-    for param in parameters:
+    args = map(shlex.quote, args)
+    args.append('"$@"')
+    for arg in args:
       if len(lines[-1]) < 40:
-        lines[-1] += ' ' + param
+        lines[-1] += ' ' + arg
       else:
         lines[-1] += ' \\'
-        lines.append('\t' + param)
+        lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(name, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0700)
 
   def createDirectory(self, parent, name, mode=0700):
     path = os.path.join(parent, name)

slapos/recipe/logrotate.py

@@ -46,10 +46,10 @@ class Recipe(GenericBaseRecipe):
 
     state_file = self.options['state-file']
 
-    logrotate = self.createPythonScript(
+    logrotate = self.createWrapper(
       self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['logrotate-binary'], '-s', state_file, logrotate_conf_file, ]
+      (self.options['logrotate-binary'],
+        '-s', state_file, logrotate_conf_file),
     )
 
     return [logrotate, logrotate_conf_file]

slapos/recipe/memcached/__init__.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from slapos.recipe.librecipe import BaseSlapRecipe
-import hashlib
-import os
-import pkg_resources
-import sys
-import zc.buildout
-import ConfigParser
-
-class Recipe(BaseSlapRecipe):
-  def getTemplateFilename(self, template_name):
-    return pkg_resources.resource_filename(__name__,
-        'template/%s' % template_name)
-
-  def _install(self):
-    self.path_list = []
-    self.requirements, self.ws = self.egg.working_set()
-    # XXX-Cedric : add logrotate?
-    self.cron_d = self.installCrond()
-    memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
-        port=11000)
-
-    ca_conf = self.installCertificateAuthority()
-    key, certificate = self.requestCertificate('Memcached')
-
-    stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
-        self.getLocalIPv4Address(), 12345, memcached_conf['memcached_port'],
-        certificate, key, ca_conf['ca_crl'],
-        ca_conf['certificate_authority_path'])
-
-    self.linkBinary()
-    self.setConnectionDict(dict(
-      stunnel_ip = stunnel_conf['public_ip'],
-      stunnel_port = stunnel_conf['public_port'],
-    ))
-    return self.path_list
-
-  def linkBinary(self):
-    """Links binaries to instance's bin directory for easier exposal"""
-    for linkline in self.options.get('link_binary_list', '').splitlines():
-      if not linkline:
-        continue
-      target = linkline.split()
-      if len(target) == 1:
-        target = target[0]
-        path, linkname = os.path.split(target)
-      else:
-        linkname = target[1]
-        target = target[0]
-      link = os.path.join(self.bin_directory, linkname)
-      if os.path.lexists(link):
-        if not os.path.islink(link):
-          raise zc.buildout.UserError(
-              'Target link already %r exists but it is not link' % link)
-        os.unlink(link)
-      os.symlink(target, link)
-      self.logger.debug('Created link %r -> %r' % (link, target))
-      self.path_list.append(link)
-
-  def installCrond(self):
-    timestamps = self.createDataDirectory('cronstamps')
-    cron_output = os.path.join(self.log_directory, 'cron-output')
-    self._createDirectory(cron_output)
-    catcher = zc.buildout.easy_install.scripts([('catchcron',
-      __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
-      self.bin_directory, arguments=[cron_output])[0]
-    self.path_list.append(catcher)
-    cron_d = os.path.join(self.etc_directory, 'cron.d')
-    crontabs = os.path.join(self.etc_directory, 'crontabs')
-    self._createDirectory(cron_d)
-    self._createDirectory(crontabs)
-    # Use execute from erp5.
-    wrapper = zc.buildout.easy_install.scripts([('crond',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
-        self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
-        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
-      )[0]
-    self.path_list.append(wrapper)
-    return cron_d
-
-  def installLogrotate(self):
-    """Installs logortate main configuration file and registers its to cron"""
-    logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
-      'logrotate.d'))
-    self._createDirectory(logrotate_d)
-    logrotate_backup = self.createBackupDirectory('logrotate')
-    logrotate_conf = self.createConfigurationFile("logrotate.conf",
-        "include %s" % logrotate_d)
-    logrotate_cron = os.path.join(self.cron_d, 'logrotate')
-    state_file = os.path.join(self.data_root_directory, 'logrotate.status')
-    open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
-        (self.options['logrotate_binary'], state_file, logrotate_conf))
-    self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
-    return logrotate_d, logrotate_backup
-
-  def registerLogRotation(self, name, log_file_list, postrotate_script):
-    """Register new log rotation requirement"""
-    open(os.path.join(self.logrotate_d, name), 'w').write(
-        self.substituteTemplate(self.getTemplateFilename(
-          'logrotate_entry.in'),
-          dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
-            postrotate=postrotate_script, olddir=self.logrotate_backup)))
-
-  def installCertificateAuthority(self, ca_country_code='XX',
-      ca_email='xx@example.com', ca_state='State', ca_city='City',
-      ca_company='Company'):
-    backup_path = self.createBackupDirectory('ca')
-    self.ca_dir = os.path.join(self.data_root_directory, 'ca')
-    self._createDirectory(self.ca_dir)
-    self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
-    self._createDirectory(self.ca_request_dir)
-    config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
-    self.ca_private = os.path.join(self.ca_dir, 'private')
-    self.ca_certs = os.path.join(self.ca_dir, 'certs')
-    self.ca_crl = os.path.join(self.ca_dir, 'crl')
-    self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
-    self.ca_key_ext = '.key'
-    self.ca_crt_ext = '.crt'
-    for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
-      self._createDirectory(d)
-    for f in ['crlnumber', 'serial']:
-      if not os.path.exists(os.path.join(self.ca_dir, f)):
-        open(os.path.join(self.ca_dir, f), 'w').write('01')
-    if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
-      open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
-    openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
-    config.update(
-        working_directory=self.ca_dir,
-        country_code=ca_country_code,
-        state=ca_state,
-        city=ca_city,
-        company=ca_company,
-        email_address=ca_email,
-    )
-    self._writeFile(openssl_configuration, pkg_resources.resource_string(
-      __name__, 'template/openssl.cnf.ca.in') % config)
-    self.path_list.extend(zc.buildout.easy_install.scripts([
-      ('certificate_authority',
-        __name__ + '.certificate_authority', 'runCertificateAuthority')],
-        self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
-          openssl_configuration=openssl_configuration,
-          openssl_binary=self.options['openssl_binary'],
-          certificate=os.path.join(self.ca_dir, 'cacert.pem'),
-          key=os.path.join(self.ca_private, 'cakey.pem'),
-          crl=os.path.join(self.ca_crl),
-          request_dir=self.ca_request_dir
-          )]))
-    # configure backup
-    backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
-    open(backup_cron, 'w').write(
-        '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
-          rdiff_backup=self.options['rdiff_backup_binary'],
-          source=self.ca_dir,
-          destination=backup_path))
-    self.path_list.append(backup_cron)
-
-    return dict(
-      ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
-      ca_crl=os.path.join(config['ca_dir'], 'crl'),
-      certificate_authority_path=config['ca_dir']
-    )
-
-  def requestCertificate(self, name):
-    hash = hashlib.sha512(name).hexdigest()
-    key = os.path.join(self.ca_private, hash + self.ca_key_ext)
-    certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
-    parser = ConfigParser.RawConfigParser()
-    parser.add_section('certificate')
-    parser.set('certificate', 'name', name)
-    parser.set('certificate', 'key_file', key)
-    parser.set('certificate', 'certificate_file', certificate)
-    parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
-    return key, certificate
-
-  def installStunnel(self, public_ip, private_ip, public_port, private_port,
-      ca_certificate, key, ca_crl, ca_path):
-    """Installs stunnel"""
-    template_filename = self.getTemplateFilename('stunnel.conf.in')
-    log = os.path.join(self.log_directory, 'stunnel.log')
-    pid_file = os.path.join(self.run_directory, 'stunnel.pid')
-    stunnel_conf = dict(
-        public_ip=public_ip,
-        private_ip=private_ip,
-        public_port=public_port,
-        pid_file=pid_file,
-        log=log,
-        cert = ca_certificate,
-        key = key,
-        ca_crl = ca_crl,
-        ca_path = ca_path,
-        private_port = private_port,
-    )
-    stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
-        self.substituteTemplate(template_filename,
-          stunnel_conf))
-    wrapper = zc.buildout.easy_install.scripts([('stunnel',
-      'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
-      sys.executable, self.wrapper_directory, arguments=[
-        [self.options['stunnel_binary'].strip(), stunnel_conf_path],
-        [ca_certificate, key]]
-      )[0]
-    self.path_list.append(wrapper)
-    return stunnel_conf
-
-  def installMemcached(self, ip, port):
-    config = dict(
-        memcached_binary=self.options['memcached_binary'],
-        memcached_ip=ip,
-        memcached_port=port,
-    )
-    self.path_list.append(self.createRunningWrapper('memcached',
-      self.substituteTemplate(self.getTemplateFilename('memcached.in'),
-        config)))
-    return dict(memcached_url='%s:%s' %
-        (config['memcached_ip'], config['memcached_port']),
-        memcached_ip=config['memcached_ip'],
-        memcached_port=config['memcached_port'])

slapos/recipe/memcached/certificate_authority.py

-import os
-import subprocess
-import time
-import ConfigParser
-
-
-def popenCommunicate(command_list, input=None):
-  subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-  if input is not None:
-    subprocess_kw.update(stdin=subprocess.PIPE)
-  popen = subprocess.Popen(command_list, **subprocess_kw)
-  result = popen.communicate(input)[0]
-  if popen.returncode is None:
-    popen.kill()
-  if popen.returncode != 0:
-    raise ValueError('Issue during calling %r, result was:\n%s' % (
-      command_list, result))
-  return result
-
-
-class CertificateAuthority:
-  def __init__(self, key, certificate, openssl_binary,
-      openssl_configuration, request_dir):
-    self.key = key
-    self.certificate = certificate
-    self.openssl_binary = openssl_binary
-    self.openssl_configuration = openssl_configuration
-    self.request_dir = request_dir
-
-  def checkAuthority(self):
-    file_list = [ self.key, self.certificate ]
-    ca_ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ca_ready = False
-        break
-    if ca_ready:
-      return
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    try:
-      # no CA, let us create new one
-      popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
-          self.openssl_configuration, '-new', '-x509', '-extensions',
-          'v3_ca', '-keyout', self.key, '-out', self.certificate,
-          '-days', '10950'], 'Automatic Certificate Authority\n')
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-
-  def _checkCertificate(self, common_name, key, certificate):
-    file_list = [key, certificate]
-    ready = True
-    for f in file_list:
-      if not os.path.exists(f):
-        ready = False
-        break
-    if ready:
-      return False
-    for f in file_list:
-      if os.path.exists(f):
-        os.unlink(f)
-    csr = certificate + '.csr'
-    try:
-      popenCommunicate([self.openssl_binary, 'req', '-config',
-        self.openssl_configuration, '-nodes', '-new', '-keyout',
-        key, '-out', csr, '-days', '3650'],
-        common_name + '\n')
-      try:
-        popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
-          self.openssl_configuration, '-out', certificate,
-          '-infiles', csr])
-      finally:
-        if os.path.exists(csr):
-          os.unlink(csr)
-    except:
-      try:
-        for f in file_list:
-          if os.path.exists(f):
-            os.unlink(f)
-      except:
-        # do not raise during cleanup
-        pass
-      raise
-    else:
-      return True
-
-  def checkRequestDir(self):
-    for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
-      parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
-      if self._checkCertificate(parser.get('certificate', 'name'),
-          parser.get('certificate', 'key_file'), parser.get('certificate',
-            'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
-
-def runCertificateAuthority(args):
-  ca_conf = args[0]
-  ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
-      ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
-      ca_conf['request_dir'])
-  while True:
-    ca.checkAuthority()
-    ca.checkRequestDir()
-    time.sleep(60)

slapos/recipe/memcached/template/memcached.in

-#!/bin/sh
-exec %(memcached_binary)s -p %(memcached_port)s -U %(memcached_port)s -l %(memcached_ip)s

slapos/recipe/memcached/template/openssl.cnf.ca.in

-#
-# OpenSSL example configuration file.
-# This is mostly being used for generation of certificate requests.
-#
-
-# This definition stops the following lines choking if HOME isn't
-# defined.
-HOME			= .
-RANDFILE		= $ENV::HOME/.rnd
-
-# Extra OBJECT IDENTIFIER info:
-#oid_file		= $ENV::HOME/.oid
-oid_section		= new_oids
-
-# To use this configuration file with the "-extfile" option of the
-# "openssl x509" utility, name here the section containing the
-# X.509v3 extensions to use:
-# extensions		= 
-# (Alternatively, use a configuration file that has only
-# X.509v3 extensions in its main [= default] section.)
-
-[ new_oids ]
-
-# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
-# Add a simple OID like this:
-# testoid1=1.2.3.4
-# Or use config file substitution like this:
-# testoid2=${testoid1}.5.6
-
-# Policies used by the TSA examples.
-tsa_policy1 = 1.2.3.4.1
-tsa_policy2 = 1.2.3.4.5.6
-tsa_policy3 = 1.2.3.4.5.7
-
-####################################################################
-[ ca ]
-default_ca	= CA_default		# The default ca section
-
-####################################################################
-[ CA_default ]
-
-dir		= %(working_directory)s		# Where everything is kept
-certs		= $dir/certs		# Where the issued certs are kept
-crl_dir		= $dir/crl		# Where the issued crl are kept
-database	= $dir/index.txt	# database index file.
-#unique_subject	= no			# Set to 'no' to allow creation of
-					# several ctificates with same subject.
-new_certs_dir	= $dir/newcerts		# default place for new certs.
-
-certificate	= $dir/cacert.pem 	# The CA certificate
-serial		= $dir/serial 		# The current serial number
-crlnumber	= $dir/crlnumber	# the current crl number
-					# must be commented out to leave a V1 CRL
-crl		= $dir/crl.pem 		# The current CRL
-private_key	= $dir/private/cakey.pem # The private key
-RANDFILE	= $dir/private/.rand	# private random number file
-
-x509_extensions	= usr_cert		# The extentions to add to the cert
-
-# Comment out the following two lines for the "traditional"
-# (and highly broken) format.
-name_opt 	= ca_default		# Subject Name options
-cert_opt 	= ca_default		# Certificate field options
-
-# Extension copying option: use with caution.
-# copy_extensions = copy
-
-# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
-# so this is commented out by default to leave a V1 CRL.
-# crlnumber must also be commented out to leave a V1 CRL.
-# crl_extensions	= crl_ext
-
-default_days	= 3650			# how long to certify for
-default_crl_days= 30			# how long before next CRL
-default_md	= default		# use public key default MD
-preserve	= no			# keep passed DN ordering
-
-# A few difference way of specifying how similar the request should look
-# For type CA, the listed attributes must be the same, and the optional
-# and supplied fields are just that :-)
-policy		= policy_match
-
-# For the CA policy
-[ policy_match ]
-countryName		= match
-stateOrProvinceName	= match
-organizationName	= match
-organizationalUnitName	= optional
-commonName		= supplied
-emailAddress		= optional
-
-# For the 'anything' policy
-# At this point in time, you must list all acceptable 'object'
-# types.
-[ policy_anything ]
-countryName		= optional
-stateOrProvinceName	= optional
-localityName		= optional
-organizationName	= optional
-organizationalUnitName	= optional
-commonName		= supplied
-emailAddress		= optional
-
-####################################################################
-[ req ]
-default_bits		= 2048
-default_md		= sha1
-default_keyfile 	= privkey.pem
-distinguished_name	= req_distinguished_name
-#attributes		= req_attributes
-x509_extensions	= v3_ca	# The extentions to add to the self signed cert
-
-# Passwords for private keys if not present they will be prompted for
-# input_password = secret
-# output_password = secret
-
-# This sets a mask for permitted string types. There are several options. 
-# default: PrintableString, T61String, BMPString.
-# pkix	 : PrintableString, BMPString (PKIX recommendation before 2004)
-# utf8only: only UTF8Strings (PKIX recommendation after 2004).
-# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
-# MASK:XXXX a literal mask value.
-# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
-string_mask = utf8only
-
-# req_extensions = v3_req # The extensions to add to a certificate request
-
-[ req_distinguished_name ]
-countryName			= Country Name (2 letter code)
-countryName_value		= %(country_code)s
-countryName_min			= 2
-countryName_max			= 2
-
-stateOrProvinceName		= State or Province Name (full name)
-stateOrProvinceName_value	= %(state)s
-
-localityName			= Locality Name (eg, city)
-localityName_value		= %(city)s
-
-0.organizationName		= Organization Name (eg, company)
-0.organizationName_value	= %(company)s
-
-# we can do this but it is not needed normally :-)
-#1.organizationName		= Second Organization Name (eg, company)
-#1.organizationName_default	= World Wide Web Pty Ltd
-
-commonName			= Common Name (eg, your name or your server\'s hostname)
-commonName_max			= 64
-
-emailAddress			= Email Address
-emailAddress_value = %(email_address)s
-emailAddress_max		= 64
-
-# SET-ex3			= SET extension number 3
-
-#[ req_attributes ]
-#challengePassword		= A challenge password
-#challengePassword_min		= 4
-#challengePassword_max		= 20
-#
-#unstructuredName		= An optional company name
-
-[ usr_cert ]
-
-# These extensions are added when 'ca' signs a request.
-
-# This goes against PKIX guidelines but some CAs do it and some software
-# requires this to avoid interpreting an end user certificate as a CA.
-
-basicConstraints=CA:FALSE
-
-# Here are some examples of the usage of nsCertType. If it is omitted
-# the certificate can be used for anything *except* object signing.
-
-# This is OK for an SSL server.
-# nsCertType			= server
-
-# For an object signing certificate this would be used.
-# nsCertType = objsign
-
-# For normal client use this is typical
-# nsCertType = client, email
-
-# and for everything including object signing:
-# nsCertType = client, email, objsign
-
-# This is typical in keyUsage for a client certificate.
-# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-# This will be displayed in Netscape's comment listbox.
-nsComment			= "OpenSSL Generated Certificate"
-
-# PKIX recommendations harmless if included in all certificates.
-subjectKeyIdentifier=hash
-authorityKeyIdentifier=keyid,issuer
-
-# This stuff is for subjectAltName and issuerAltname.
-# Import the email address.
-# subjectAltName=email:copy
-# An alternative to produce certificates that aren't
-# deprecated according to PKIX.
-# subjectAltName=email:move
-
-# Copy subject details
-# issuerAltName=issuer:copy
-
-#nsCaRevocationUrl		= http://www.domain.dom/ca-crl.pem
-#nsBaseUrl
-#nsRevocationUrl
-#nsRenewalUrl
-#nsCaPolicyUrl
-#nsSslServerName
-
-# This is required for TSA certificates.
-# extendedKeyUsage = critical,timeStamping
-
-[ v3_req ]
-
-# Extensions to add to a certificate request
-
-basicConstraints = CA:FALSE
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-[ v3_ca ]
-
-
-# Extensions for a typical CA
-
-
-# PKIX recommendation.
-
-subjectKeyIdentifier=hash
-
-authorityKeyIdentifier=keyid:always,issuer
-
-# This is what PKIX recommends but some broken software chokes on critical
-# extensions.
-#basicConstraints = critical,CA:true
-# So we do this instead.
-basicConstraints = CA:true
-
-# Key usage: this is typical for a CA certificate. However since it will
-# prevent it being used as an test self-signed certificate it is best
-# left out by default.
-# keyUsage = cRLSign, keyCertSign
-
-# Some might want this also
-# nsCertType = sslCA, emailCA
-
-# Include email address in subject alt name: another PKIX recommendation
-# subjectAltName=email:copy
-# Copy issuer details
-# issuerAltName=issuer:copy
-
-# DER hex encoding of an extension: beware experts only!
-# obj=DER:02:03
-# Where 'obj' is a standard or added object
-# You can even override a supported extension:
-# basicConstraints= critical, DER:30:03:01:01:FF
-
-[ crl_ext ]
-
-# CRL extensions.
-# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
-
-# issuerAltName=issuer:copy
-authorityKeyIdentifier=keyid:always
-
-[ proxy_cert_ext ]
-# These extensions should be added when creating a proxy certificate
-
-# This goes against PKIX guidelines but some CAs do it and some software
-# requires this to avoid interpreting an end user certificate as a CA.
-
-basicConstraints=CA:FALSE
-
-# Here are some examples of the usage of nsCertType. If it is omitted
-# the certificate can be used for anything *except* object signing.
-
-# This is OK for an SSL server.
-# nsCertType			= server
-
-# For an object signing certificate this would be used.
-# nsCertType = objsign
-
-# For normal client use this is typical
-# nsCertType = client, email
-
-# and for everything including object signing:
-# nsCertType = client, email, objsign
-
-# This is typical in keyUsage for a client certificate.
-# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-
-# This will be displayed in Netscape's comment listbox.
-nsComment			= "OpenSSL Generated Certificate"
-
-# PKIX recommendations harmless if included in all certificates.
-subjectKeyIdentifier=hash
-authorityKeyIdentifier=keyid,issuer
-
-# This stuff is for subjectAltName and issuerAltname.
-# Import the email address.
-# subjectAltName=email:copy
-# An alternative to produce certificates that aren't
-# deprecated according to PKIX.
-# subjectAltName=email:move
-
-# Copy subject details
-# issuerAltName=issuer:copy
-
-#nsCaRevocationUrl		= http://www.domain.dom/ca-crl.pem
-#nsBaseUrl
-#nsRevocationUrl
-#nsRenewalUrl
-#nsCaPolicyUrl
-#nsSslServerName
-
-# This really needs to be in place for it to be a proxy certificate.
-proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
-
-####################################################################
-[ tsa ]
-
-default_tsa = tsa_config1	# the default TSA section
-
-[ tsa_config1 ]
-
-# These are used by the TSA reply generation only.
-dir		= /etc/pki/tls		# TSA root directory
-serial		= $dir/tsaserial	# The current serial number (mandatory)
-crypto_device	= builtin		# OpenSSL engine to use for signing
-signer_cert	= $dir/tsacert.pem 	# The TSA signing certificate
-					# (optional)
-certs		= $dir/cacert.pem	# Certificate chain to include in reply
-					# (optional)
-signer_key	= $dir/private/tsakey.pem # The TSA private key (optional)
-
-default_policy	= tsa_policy1		# Policy if request did not specify it
-					# (optional)
-other_policies	= tsa_policy2, tsa_policy3	# acceptable policies (optional)
-digests		= md5, sha1		# Acceptable message digests (mandatory)
-accuracy	= secs:1, millisecs:500, microsecs:100	# (optional)
-clock_precision_digits  = 0	# number of digits after dot. (optional)
-ordering		= yes	# Is ordering defined for timestamps?
-				# (optional, default: no)
-tsa_name		= yes	# Must the TSA name be included in the reply?
-				# (optional, default: no)
-ess_cert_id_chain	= no	# Must the ESS cert id chain be included?
-				# (optional, default: no)

slapos/recipe/memcached/template/stunnel.conf.in

-foreground = yes
-output = %(log)s
-pid = %(pid_file)s
-syslog = no
-CApath = %(ca_path)s
-key = %(key)s
-CRLpath = %(ca_crl)s
-cert = %(cert)s
-
-[service]
-accept = %(public_ip)s:%(public_port)s
-connect = %(private_ip)s:%(private_port)s

slapos/recipe/mioga/instantiate.py

@@ -203,11 +203,10 @@ Include conf/extra/httpd-autoindex.conf
 
     services_dir = self.options['services_dir']
 
-    httpd_wrapper = self.createPythonScript(
+    httpd_wrapper = self.createWrapper(
       os.path.join(services_dir, 'httpd_wrapper'),
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['httpd_binary'], '-f', self.options['httpd_conf'],
-       '-DFOREGROUND']
+      (self.options['httpd_binary'],
+        '-f', self.options['httpd_conf'], '-DFOREGROUND'),
     )
     path_list.append(httpd_wrapper)
 
@@ -220,19 +219,17 @@ Include conf/extra/httpd-autoindex.conf
 
     site_perl_bin = os.path.join(self.options['site_perl'], 'bin')
     mioga_conf_path = os.path.join(mioga_base, 'conf', 'Mioga.conf')
-    notifier_wrapper = self.createPythonScript(
+    notifier_wrapper = self.createWrapper(
       os.path.join(services_dir, 'notifier'),
-      'slapos.recipe.librecipe.execute.execute',
-      [ os.path.join(site_perl_bin, 'notifier.pl'),
-        mioga_conf_path ]
+      (os.path.join(site_perl_bin, 'notifier.pl'),
+        mioga_conf_path),
     )
     path_list.append(notifier_wrapper)
 
-    searchengine_wrapper = self.createPythonScript(
+    searchengine_wrapper = self.createWrapper(
       os.path.join(services_dir, 'searchengine'),
-      'slapos.recipe.librecipe.execute.execute',
-      [ os.path.join(site_perl_bin, 'searchengine.pl'),
-        mioga_conf_path ]
+      (os.path.join(site_perl_bin, 'searchengine.pl'),
+        mioga_conf_path),
     )
     path_list.append(searchengine_wrapper)
 

slapos/recipe/mydumper.py

@@ -24,7 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import subprocess
+import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -58,14 +58,14 @@ def do_export(args):
 
   cmd.extend(['-o', args['directory']])
 
-  subprocess.check_call(cmd)
+  os.execv(cmd[0], cmd)
 
 
 def do_import(args):
   cmd = _mydumper_base_cmd(**args)
   cmd.append('--overwrite-tables')
   cmd.extend(['-d', args['directory']])
-  subprocess.check_call(cmd)
+  os.execv(cmd[0], cmd)
 
 
 class Recipe(GenericBaseRecipe):
@@ -95,9 +95,5 @@ class Recipe(GenericBaseRecipe):
       config['compression'] = self.optionIsTrue('compression', default=False)
       config['rows'] = self.options.get('rows')
 
-    wrapper = self.createPythonScript(name=self.options['wrapper'],
-                                      absolute_function = '%s.%s' % (__name__, function.func_name),
-                                      arguments=config)
-
-    return [wrapper]
-
+    return self.createPythonScript(self.options['wrapper'],
+      '%s.%s' % (function.__module__, function.__name__), (config,))

slapos/recipe/mysql/__init__.py

@@ -53,11 +53,10 @@ class Recipe(GenericBaseRecipe):
 
     mysql_binary = self.options['mysql-binary']
     socket = self.options['socket'],
-    post_rotate = self.createPythonScript(
+    post_rotate = self.createWrapper(
       self.options['logrotate-post'],
-      'slapos.recipe.librecipe.execute.execute',
-      [mysql_binary, '--no-defaults', '-B', '-u', 'root', '--socket=%s' % socket, '-e',
-       'FLUSH LOGS']
+      (mysql_binary, '--no-defaults', '-B', '-u', 'root',
+        '--socket=%s' % socket, '-e', 'FLUSH LOGS'),
     )
     path_list.append(post_rotate)
 
@@ -85,12 +84,12 @@ class Recipe(GenericBaseRecipe):
     mysql_update = self.createPythonScript(
       self.options['update-wrapper'],
       '%s.mysql.updateMysql' % __name__,
-      dict(
+      (dict(
         mysql_script=mysql_script,
         mysql_binary=mysql_binary,
         mysql_upgrade_binary=mysql_upgrade_binary,
         socket=socket,
-      )
+       ),)
     )
     path_list.append(mysql_update)
 
@@ -98,7 +97,7 @@ class Recipe(GenericBaseRecipe):
     mysqld = self.createPythonScript(
       self.options['wrapper'],
       '%s.mysql.runMysql' % __name__,
-      dict(
+      (dict(
         mysql_install_binary=self.options['mysql-install-binary'],
         mysqld_binary=mysqld_binary,
         data_directory=mysql_conf['data_directory'],
@@ -106,7 +105,7 @@ class Recipe(GenericBaseRecipe):
         socket=socket,
         configuration_file=mysql_conf_file,
         cwd=self.options['mysql-base-directory'],
-       )
+       ),)
     )
     path_list.append(mysqld)
 
@@ -115,12 +114,12 @@ class Recipe(GenericBaseRecipe):
       backup_script = self.createPythonScript(
         self.options['backup-script'],
         '%s.do_backup' % __name__,
-        dict(
+        (dict(
           mydumper_binary=self.options['mydumper-binary'],
           database=mysql_conf['mysql_database'],
           socket=mysql_conf['socket'],
           backup_directory=self.options['backup-directory']
-        ),
+         ),)
       )
       path_list.append(backup_script)
 
@@ -129,7 +128,7 @@ class Recipe(GenericBaseRecipe):
       recovering_script = self.createPythonScript(
         self.options['recovering-wrapper'],
         '%s.import_dump' % __name__,
-        {
+        ({
           'lock_file': os.path.join(self.work_directory,
                                     'import_done'),
           'database': mysql_conf['mysql_database'],
@@ -140,7 +139,7 @@ class Recipe(GenericBaseRecipe):
           'local_directory': self.mysql_backup_directory,
           'dump_name': dump_filename,
           'zcat_binary': self.options['zcat-binary'],
-        }
+        },)
       )
       path_list.append(recovering_script)
 

slapos/recipe/neoppod.py

@@ -42,7 +42,8 @@ class NeoBaseRecipe(GenericBaseRecipe):
       # Only then can this recipe start succeeding and actually doing anything
       # useful, as per NEO deploying constraints.
       raise UserError('"masters" parameter is mandatory')
-    option_list = [
+    args = [
+      options['binary'],
       # Keep the -l option first, as expected by logrotate snippets.
       '-l', options['logfile'],
       '-m', options['masters'],
@@ -53,17 +54,13 @@ class NeoBaseRecipe(GenericBaseRecipe):
     ]
     if options['ssl']:
       etc = os.path.join(self.buildout['buildout']['directory'], 'etc', '')
-      option_list += (
+      args += (
         '--ca', etc + 'ca.crt',
         '--cert', etc + 'neo.crt',
         '--key', etc + 'neo.key',
         )
-    option_list.extend(self._getOptionList())
-    return [self.createWrapper(
-      options['wrapper'],
-      options['binary'],
-      option_list
-    )]
+    args += self._getOptionList()
+    return self.createWrapper(options['wrapper'], args)
 
   def _getBindingAddress(self):
     options = self.options

slapos/recipe/notifier.py

@@ -32,19 +32,15 @@ class Recipe(GenericBaseRecipe):
 
   def install(self):
     options = self.options
-    script = self.createWrapper(name=options['wrapper'],
-                                command=options['server-binary'],
-                                parameters=[
+    # Script that execute the callback(s) upon receiving a notification.
+    return self.createWrapper(options['wrapper'],
+                                (options['server-binary'],
                                    '--callbacks', options['callbacks'],
                                    '--feeds', options['feeds'],
                                    '--equeue-socket', options['equeue-socket'],
                                    options['host'], options['port']
-                                   ],
-                                comments=[
-                                    '',
-                                    'Upon receiving a notification, execute the callback(s).',
-                                    ''])
-    return [script]
+                                   ),
+                                )
 
 
 class Callback(GenericBaseRecipe):
@@ -80,35 +76,32 @@ class Notify(GenericBaseRecipe):
       # Just a touch
       open(log, 'w').close()
 
-    parameters = [
+    cmd = [notifier_binary,
             '-l', log,
             '--title', title,
             '--feed', feed_url,
             '--max-run', str(max_run),
             '--notification-url',
             ]
-    parameters.extend(notification_url.split(' '))
-    parameters.extend(['--executable', executable])
+    cmd += notification_url.split(' ')
+    cmd += '--executable', executable
     # For a more verbose mode, writing feed items for any action
     instance_root_name = instance_root_name or self.options.get('instance-root-name', None)
     log_url = log_url or self.options.get('log-url', None)
     status_item_directory = status_item_directory or self.options.get('status-item-directory', None)
     if instance_root_name and log_url and status_item_directory:
-      parameters.extend([
+      cmd += (
         '--instance-root-name', instance_root_name,
         '--log-url', log_url,
         '--status-item-directory', status_item_directory,
-      ])
-
-    return self.createWrapper(name=wrapper,
-                              command=notifier_binary,
-                              parameters=parameters,
-                              pidfile=pidfile,
-                              parameters_extra=True,
-                              comments=[
-                                  '',
-                                  'Call an executable and send notification(s).',
-                                  ''])
+      )
+
+    kw = {}
+    if pidfile:
+      kw['pidfile'] = pidfile
+
+    # Script that call an executable and send notification(s).
+    return self.createWrapper(wrapper, cmd, **kw)
 
 
   def install(self):

slapos/recipe/novnc/__init__.py

@@ -35,10 +35,9 @@ class Recipe(GenericBaseRecipe):
   """
 
   def install(self):
-    runner_path = self.createPythonScript(
+    return self.createWrapper(
       self.options['path'],
-      'slapos.recipe.librecipe.execute.execute_wait',
-      [[
+      (
         self.options['websockify-path'],
         '--web',
         self.options['novnc-location'],
@@ -47,8 +46,7 @@ class Recipe(GenericBaseRecipe):
         '--ssl-only',
         '%s:%s' % (self.options['ip'], self.options['port']),
         '%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']),
-      ],
-      [self.options['ssl-key-path'], self.options['ssl-cert-path']]],
+       ),
+      wait_list=(self.options['ssl-key-path'],
+                 self.options['ssl-cert-path']),
     )
-
-    return [runner_path]

slapos/recipe/pbs.py

@@ -39,30 +39,18 @@ from slapos.recipe.notifier import Callback
 from slapos.recipe.librecipe import shlex
 
 
-def promise(args):
+def promise(ssh_client, user, host, port):
   # Redirect output to /dev/null
-  with open("/dev/null") as _dev_null:
+  with open(os.devnull) as _dev_null:
     ssh = subprocess.Popen(
-        [args['ssh_client'], '%(user)s@%(host)s' % args, '-p', '%(port)s' % args],
-        stdin=subprocess.PIPE, stdout=_dev_null, stderr=None
-    )
-
-  # Rdiff Backup protocol quit command
-  quitcommand = 'q' + chr(255) + chr(0) * 7
-
-  ssh.stdin.write(quitcommand)
-  ssh.stdin.flush()
-  ssh.stdin.close()
-  ssh.wait()
-
-  if ssh.poll() is None:
-    return 1
-  if ssh.returncode != 0:
+        (ssh_client, '%s@%s' % (user, host), '-p', str(port)),
+        stdin=subprocess.PIPE, stdout=_dev_null)
+  ssh.communicate('q' + chr(255) + chr(0) * 7)
+  if ssh.returncode:
     sys.stderr.write("SSH Connection failed\n")
   return ssh.returncode
 
 
-
 class Recipe(GenericSlapRecipe, Notify, Callback):
   def _options(self, options):
     options['rdiff-backup-data-folder'] = ""
@@ -244,15 +232,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
 
-    promise_path = os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id)
-    promise_dict = dict(ssh_client=self.options['sshclient-binary'],
-                        user=parsed_url.username,
-                        host=parsed_url.hostname,
-                        port=parsed_url.port)
-    promise = self.createPythonScript(promise_path,
-                                      __name__ + '.promise',
-                                      promise_dict)
-    path_list.append(promise)
+    path_list.append(self.createPythonScript(
+      os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),
+      __name__ + '.promise',
+      (self.options['sshclient-binary'],
+       parsed_url.username, parsed_url.hostname, parsed_url.port)))
 
     # Create known_hosts file by default.
     # In some case, we don't want to create it (case where we share IP mong partitions)
@@ -336,12 +320,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
     else:
       self.logger.info("Server mode")
 
-      wrapper = self.createWrapper(name=self.options['wrapper'],
-                                   command=self.options['rdiffbackup-binary'],
-                                   parameters=[
+      wrapper = self.createWrapper(self.options['wrapper'],
+                                   (self.options['rdiffbackup-binary'],
                                        '--restrict', self.options['path'],
                                        '--server'
-                                       ])
+                                       ))
       path_list.append(wrapper)
 
     return path_list

slapos/recipe/re6stnet/__init__.py

@@ -162,7 +162,6 @@ class Recipe(GenericBaseRecipe):
     return hash_string
 
   def install(self):
-    path_list = []
     token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
     token_list_path = self.options['token-dir']
 
@@ -190,20 +189,14 @@ class Recipe(GenericBaseRecipe):
 
     self.createFile(token_save_path, json.dumps(token_dict))
 
-    service_dict = dict(token_base_path=token_list_path,
-                        token_json=token_save_path,
-                        partition_id=self.computer_partition_id,
-                        computer_id=self.computer_id,
-                        registry_url=registry_url,
-                        server_url=self.server_url,
+    computer_dict = dict(partition_id=self.computer_partition_id,
+                        computer_guid=self.computer_id,
+                        master_url=self.server_url,
                         cert_file=self.cert_file,
                         key_file=self.key_file)
 
-    request_add = self.createPythonScript(
+    return self.createPythonScript(
         self.options['manager-wrapper'].strip(),
-        '%s.re6stnet.manage' % __name__, service_dict
+        __name__ + '.re6stnet.manage',
+        (registry_url, token_list_path, token_save_path, computer_dict)
       )
-    path_list.append(request_add)
-
-    return path_list
-

slapos/recipe/re6stnet/re6stnet.py

@@ -17,10 +17,8 @@ logging.trace = logging.debug
 def loadJsonFile(path):
   if os.path.exists(path):
     with open(path, 'r') as f:
-      content = f.read()
-      return json.loads(content)
-  else:
-    return {}
+      return json.load(f)
+  return {}
 
 def writeFile(path, data):
   with open(path, 'w') as f:
@@ -39,29 +37,25 @@ def updateFile(file_path, value):
     return True
   return False
 
-def getComputerPartition(server_url, key_file, cert_file, computer_guid, partition_id):
+def getComputerPartition(master_url, key_file, cert_file,
+                         computer_guid, partition_id):
   slap = slapos.slap.slap()
-
   # Redeploy instance to update published information
-  slap.initializeConnection(server_url,
-                            key_file,
-                            cert_file)
-
-  return slap.registerComputerPartition(computer_guid=computer_guid,
-                                             partition_id=partition_id)
+  slap.initializeConnection(master_url, key_file, cert_file)
+  return slap.registerComputerPartition(computer_guid, partition_id)
 
-def requestAddToken(client, base_token_path):
+def requestAddToken(client, token_base_path):
   time.sleep(3)
-  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
+  path_list = [x for x in os.listdir(token_base_path) if x.endswith('.add')]
 
-  log.info("Searching tokens to add at %s and found %s." % (base_token_path, path_list))
+  log.info("Searching tokens to add at %s and found %s." % (token_base_path, path_list))
 
   if not path_list:
     log.info("No new token to add. Exiting...")
     return
 
   for reference_key in path_list:
-    request_file = os.path.join(base_token_path, reference_key)
+    request_file = os.path.join(token_base_path, reference_key)
     token = readFile(request_file)
     log.info("Including token %s for %s" % (token, reference_key))
     if token :
@@ -79,21 +73,21 @@ def requestAddToken(client, base_token_path):
         # update information
         log.info("New token added for slave instance %s. Updating file status..." %
                             reference)
-        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        status_file = os.path.join(token_base_path, '%s.status' % reference)
         updateFile(status_file, 'TOKEN_ADDED')
         os.unlink(request_file)
     else:
       log.debug('Bad token. Request add token fail for %s...' % request_file)
 
-def requestRemoveToken(client, base_token_path):
-  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
+def requestRemoveToken(client, token_base_path):
+  path_list = [x for x in os.listdir(token_base_path) if x.endswith('.remove')]
 
   if not path_list:
     log.info("No token to delete. Exiting...")
     return
 
   for reference_key in path_list:
-    request_file = os.path.join(base_token_path, reference_key)
+    request_file = os.path.join(token_base_path, reference_key)
     token = readFile(request_file)
     if token :
       reference = reference_key.split('.')[0]
@@ -108,7 +102,7 @@ def requestRemoveToken(client, base_token_path):
         continue
       else:
         # certificate is invalidated, it will be revoked
-        writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
+        writeFile(os.path.join(token_base_path, '%s.revoke' % reference), '')
 
       if result in (True, 'True'):
         # update information
@@ -117,33 +111,17 @@ def requestRemoveToken(client, base_token_path):
 
       if result in ['True', 'False']:
         os.unlink(request_file)
-        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        status_file = os.path.join(token_base_path, '%s.status' % reference)
         if os.path.exists(status_file):
           os.unlink(status_file)
-        ipv6_file = os.path.join(base_token_path, '%s.ipv6' % reference)
+        ipv6_file = os.path.join(token_base_path, '%s.ipv6' % reference)
         if os.path.exists(ipv6_file):
           os.unlink(ipv6_file)
 
     else:
       log.debug('Bad token. Request add token fail for %s...' % request_file)
 
-def requestRevoqueCertificate(args):
-  base_token_path = args['token_base_path']
-  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
-
-  for reference_key in path_list:
-    reference = reference_key.split('.')[0]
-
-    if revokeByMail(args['registry_url'],
-                   '%s@slapos' % reference.lower(),
-                   args['db']):
-      os.unlink(os.path.join(base_token_path, reference_key))
-      log.info("Certificate revoked for slave instance %s." % reference)
-      return
-
-    log.info("Failed to revoke email for %s" % reference)
-
-def checkService(client, base_token_path, token_json, computer_partition):
+def checkService(client, token_base_path, token_json, computer_partition):
   token_dict = loadJsonFile(token_json)
   updated = False
   if not token_dict:
@@ -152,7 +130,7 @@ def checkService(client, base_token_path, token_json, computer_partition):
   # Check token status
   for slave_reference, token in token_dict.iteritems():
     log.info("%s %s" % (slave_reference, token))
-    status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
+    status_file = os.path.join(token_base_path, '%s.status' % slave_reference)
     if not os.path.exists(status_file):
       # This token is not added yet!
       log.info("Token %s dont exist yet." % status_file)
@@ -206,31 +184,22 @@ def checkService(client, base_token_path, token_json, computer_partition):
          slave_reference, traceback.format_exc())
 
 
-def manage(args, can_bang=True):
-
-  computer_guid = args['computer_id']
-  partition_id = args['partition_id']
-  server_url = args['server_url']
-  key_file = args['key_file']
-  cert_file = args['cert_file']
+def manage(registry_url, token_base_path, token_json,
+           computer_dict, can_bang=True):
 
-  client = registry.RegistryClient(args['registry_url'])
-  base_token_path = args['token_base_path']
-  token_json = args['token_json']
+  client = registry.RegistryClient(registry_url)
 
   log.info("ADD TOKEN")
   # Request Add new tokens
-  requestAddToken(client, base_token_path)
+  requestAddToken(client, token_base_path)
 
   log.info("Remove TOKEN")
   # Request delete removed token
-  requestRemoveToken(client, base_token_path)
+  requestRemoveToken(client, token_base_path)
 
-  computer_partition = getComputerPartition(server_url, key_file,
-                              cert_file, computer_guid, partition_id)
+  computer_partition = getComputerPartition(**computer_dict)
 
   log.info("Update Services")
   # check status of all token
-  checkService(client, base_token_path,
-            token_json, computer_partition)
+  checkService(client, token_base_path, token_json, computer_partition)
 

slapos/recipe/redis/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 import os
-
+import sys
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
@@ -56,10 +56,9 @@ class Recipe(GenericBaseRecipe):
       configuration))
     path_list.append(config)
 
-    redis = self.createPythonScript(
+    redis = self.createWrapper(
       self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['server_bin'], config_file]
+      (self.options['server_bin'], config_file),
     )
     path_list.append(redis)
 
@@ -67,11 +66,20 @@ class Recipe(GenericBaseRecipe):
     if promise_script:
       promise = self.createPythonScript(
         promise_script,
-        '%s.promise.main' % __name__,
-        dict(host=self.options['ipv6'], port=self.options['port'],
-             unixsocket = self.options.get('unixsocket') )
+        __name__ + '.promise',
+        (self.options['ipv6'], int(self.options['port']),
+         self.options.get('unixsocket'))
       )
       path_list.append(promise)
 
     return path_list
 
+
+def promise(host, port, unixsocket):
+  from .MyRedis2410 import Redis
+  try:
+    r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
+    r.publish("Promise-Service","SlapOS Promise")
+    r.connection_pool.disconnect()
+  except Exception, e:
+    sys.exit(e)

slapos/recipe/redis/promise.py

-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import slapos.recipe.redis.MyRedis2410 as redis
-import sys
-
-def main(args):
-  host = args['host']
-  port = int(args['port'])
-  unixsocket = args['unixsocket']
-  try:
-    r = redis.Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
-    r.publish("Promise-Service","SlapOS Promise")
-    r.connection_pool.disconnect()
-    sys.exit(0)
-  except Exception, e:
-    print str(e)
-    sys.exit(1)
\ No newline at end of file

slapos/recipe/reverse_proxy_nginx/__init__.py

@@ -70,14 +70,12 @@ class Recipe(GenericSlapRecipe):
     path_list.append(nginx_configuration_file)
     
     # Generate Nginx wrapper
-    wrapper = self.createWrapper(
-        name=self.options['wrapper'],
-        command=self.options['nginx-executable'],
-        parameters=[
+    path_list.append(self.createWrapper(
+        self.options['wrapper'],
+        (self.options['nginx-executable'],
             '-c', self.options['configuration-file'],
             '-p', self.options['home-directory']
-        ]
-    )
+        )))
 
     # TODO: reload configuration or have feature like apache_map
 

slapos/recipe/seleniumrunner/__init__.py

@@ -25,23 +25,15 @@
 #
 #############################################################################
 
-import os
-import sys
-import zc.buildout
-from slapos.recipe.librecipe import BaseSlapRecipe
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 class Recipe(GenericBaseRecipe):
-  def install(self):
 
-    runner = self.createPythonScript(
+  def install(self):
+    return self.createPythonScript(
       self.options['runner-path'],
       __name__+'.testrunner.run',
-      arguments=[self.options['suite-url'],
-                 self.options['report-url'],
-                 self.options['report-project'],
-                 self.options['browser'],
-                 ])
-
-    return [runner]
-
+      (self.options['suite-url'],
+       self.options['report-url'],
+       self.options['report-project'],
+       self.options['browser']))

slapos/recipe/seleniumrunner/testrunner.py

@@ -36,12 +36,7 @@ import urlparse
 from subprocess import Popen, PIPE
 import signal
 
-def run(args):
-  suite_url = args[0]
-  report_url = args[1]
-  project = args[2]
-  browser_binary = args[3]
-
+def run(suite_url, report_url, project, browser_binary):
   suite_parsed = urlparse.urlparse(suite_url)
 
   config = {

slapos/recipe/shell.py

@@ -31,27 +31,12 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    env = os.environ.copy()
-
-    path_list = self.options['path'].split('\n')
-    env.update(PATH=':'.join(path_list))
-    env.update(SHELL=self.options['shell'])
-    env.update(HOME=self.options['home'])
-
     ps1 = self.options.get('ps1')
-    if ps1 is not None:
-      env.update(PS1=str(json.loads(ps1)))
-    else:
-      env.update(PS1=env.get('PS1', '> '))
-
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.executee',
-      [ # Executable
-        [self.options['shell']],
-        # Environment
-        env
-      ]
-    )
-
-    return [wrapper]
+    shell = self.options['shell']
+    env = {
+      'HOME': self.options['home'],
+      'PATH': ':'.join(self.options['path'].split('\n')),
+      'PS1': str(json.loads(ps1)) if ps1 else os.getenv('PS1', '> '),
+      'SHELL': shell,
+    }
+    return self.createWrapper(self.options['wrapper'], (shell,), env)

slapos/recipe/shellinabox.py

@@ -33,25 +33,15 @@ import shlex
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def login_shell(args):
-  password_file = args['password-file']
+def login_shell(password_file, shell):
   if password_file:
     with open(password_file, 'r') as password_file:
       password = password_file.read()
 
-    if (password != ''):
-      entered_password = getpass()
-    else:
-      entered_password = ''
-
-    if not hmac.compare_digest(entered_password, password):
-      return 1
-    else:
-      commandline = shlex.split(args['shell'])
-      path = commandline[0]
-      os.execv(path, commandline)
-  else:
-    return 1
+    if not password or hmac.compare_digest(getpass(), password):
+      commandline = shlex.split(shell)
+      os.execv(commandline[0], commandline)
+  return 1
 
 def shellinabox(args):
   certificate_dir = args['certificate_dir']
@@ -95,22 +85,16 @@ def shellinabox(args):
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    path_list = []
-
-    login_shell = self.createPythonScript(
+    login_shell_wrapper = self.createPythonScript(
       self.options['login-shell'],
-      '%s.login_shell' % __name__,
-      {
-        'password-file': self.options['password-file'],
-        'shell': self.options['shell']
-      }
+      __name__ + '.login_shell',
+      (self.options['password-file'], self.options['shell'])
     )
-    path_list.append(login_shell)
 
-    wrapper = self.createPythonScript(
+    shellinabox_wrapper = self.createPythonScript(
       self.options['wrapper'],
-      '%s.shellinabox' % __name__,
-      dict(
+      __name__ + '.shellinabox',
+      (dict(
         certificate_dir=self.options['certificate-directory'],
         ssl_key=self.options['key-file'],
         ssl_certificate=self.options['cert-file'],
@@ -118,9 +102,8 @@ class Recipe(GenericBaseRecipe):
         directory=self.options['directory'],
         ipv6=self.options['ipv6'],
         port=self.options['port'],
-        login_shell=login_shell,
-      )
+        login_shell=login_shell_wrapper,
+       ),)
     )
-    path_list.append(wrapper)
 
-    return [wrapper]
+    return login_shell_wrapper, shellinabox_wrapper

slapos/recipe/signal_wrapper.py

@@ -32,6 +32,6 @@ class Recipe(GenericBaseRecipe):
       self.createPythonScript(
         self.options['wrapper-path'],
         'slapos.recipe.librecipe.execute.execute_with_signal_translation',
-        [self.options['wrapped-path']]
+        ((self.options['wrapped-path'],),)
       )
     ]

slapos/recipe/simplehttpserver/__init__.py

@@ -68,9 +68,8 @@ class Recipe(GenericBaseRecipe):
       'root-dir': self.options['root-dir']
     }
 
-    server = self.createPythonScript(
+    return self.createPythonScript(
         self.options['wrapper'].strip(),
-        '%s.simplehttpserver.run' % __name__, parameters
+        __name__ + '.simplehttpserver.run',
+        (parameters,)
       )
-
-    return [server]

slapos/recipe/simplelogger.py

@@ -29,9 +29,9 @@ import time
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def log(args):
+def log(filename):
   prefix = time.strftime('%Y-%m-%d.%H:%M.%s:')
-  with open(args['filename'], 'aw') as logfile:
+  with open(filename, 'a') as logfile:
     for line in sys.stdin:
       print >> logfile, prefix, line,
     print >> logfile, prefix, '------------------------'
@@ -39,10 +39,7 @@ def log(args):
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    wrapper = self.options['wrapper']
-    log = self.options['log']
-
-    script = self.createPythonScript(wrapper,
-                                     __name__ + '.log',
-                                     arguments=dict(filename=log))
-    return [script]
+    return self.createPythonScript(
+      self.options['wrapper'],
+      __name__ + '.log',
+      (self.options['log'],))

slapos/recipe/sphinx/__init__.py

@@ -48,11 +48,10 @@ class Recipe(GenericBaseRecipe):
     )
 
     # Create init script
-    wrapper = self.createPythonScript(
+    wrapper = self.createWrapper(
         self.options['wrapper'],
-        'slapos.recipe.librecipe.execute.execute',
-        [self.options['sphinx-searchd-binary'].strip(), '-c',
-         sphinx_conf_path, '--nodetach'],
+        (self.options['sphinx-searchd-binary'].strip(), '-c',
+          sphinx_conf_path, '--nodetach'),
         )
 
     return [wrapper, sphinx_conf_path]

slapos/recipe/squid/__init__.py

@@ -78,21 +78,15 @@ class Recipe(GenericBaseRecipe):
       self.substituteTemplate(template_filename, config))
 
     # Prepare directories
-    prepare_path = self.createPythonScript(
+    prepare_path = self.createWrapper(
       self.options['prepare-path'],
-      'slapos.recipe.librecipe.execute.execute',
-      arguments=[self.options['binary-path'].strip(), 
-                 '-z',
-                 '-f', configuration_path,
-                 ],)
+      (self.options['binary-path'].strip(),
+        '-z', '-f', configuration_path))
 
     # Create running wrapper
-    wrapper_path = self.createPythonScript(
+    wrapper_path = self.createWrapper(
       self.options['wrapper-path'],
-      'slapos.recipe.librecipe.execute.execute',
-      arguments=[self.options['binary-path'].strip(), 
-                 '-N',
-                 '-f', configuration_path,
-                 ],)
+      (self.options['binary-path'].strip(),
+        '-N', '-f', configuration_path))
 
     return [configuration_path, wrapper_path, prepare_path]

slapos/recipe/sshkeys_authority.py

@@ -34,17 +34,14 @@ from slapos.recipe.librecipe import GenericBaseRecipe
 from slapos.recipe.librecipe.inotify import subfiles
 
 # This authority only works with dropbear or openssh sshkey generators
-def sshkeys_authority(args):
-  requests_directory = args['requests']
-  keygen_binary = args['sshkeygen']
-
+def sshkeys_authority(request_directory, keygen_binary):
   if 'openssh' in keygen_binary:
     authority_type = 'openssh'
   else:
     # Keep dropbear for compatibility
     authority_type = 'dropbear'
 
-  for request_filename in subfiles(requests_directory):
+  for request_filename in subfiles(request_directory):
 
     with open(request_filename) as request_file:
       request = json.load(request_file)
@@ -98,18 +95,13 @@ def sshkeys_authority(args):
         public_key_file.write(public_key_value)
 
 
-
 class Recipe(GenericBaseRecipe):
 
   def install(self):
-    args = dict(
-      requests=self.options['request-directory'],
-      sshkeygen=self.options['keygen-binary'],
-    )
-
-    wrapper = self.createPythonScript(self.options['wrapper'],
-      __name__ + '.sshkeys_authority', args)
-    return [wrapper]
+    return self.createPythonScript(self.options['wrapper'],
+      __name__ + '.sshkeys_authority',
+      (self.options['request-directory'],
+       self.options['keygen-binary']))
 
 class Request(GenericBaseRecipe):
 
@@ -160,11 +152,9 @@ class Request(GenericBaseRecipe):
     os.symlink(self.private_key, private_key_link)
     # end-XXX
 
-    wrapper = self.createPythonScript(
+    wrapper = self.createWrapper(
       self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute_wait',
-      [ [self.options['executable']],
-        [self.private_key, self.public_key] ])
-
+      (self.options['executable'],),
+      wait_list=(self.private_key, self.public_key))
 
     return [request_file, wrapper, public_key_link, private_key_link]

slapos/recipe/stunnel/__init__.py

@@ -30,13 +30,16 @@ import errno
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
-def post_rotate(args):
-  pid_file = args['pid_file']
-
-  if os.path.exist(pid_file):
-    with open(pid_file, 'r') as file_:
-      pid = file_.read().strip()
-    os.kill(pid, signal.SIGUSR1)
+def kill(pid_file, sig=signal.SIGUSR1):
+  if os.path.exists(pid_file):
+    with open(pid_file) as f:
+      pid = int(f.read().strip())
+    try:
+      os.kill(pid, sig)
+    except OSError, e:
+      if e.errno != errno.ESRCH: # No such process
+        raise e
+      os.unlink(pid_file)
 
 class Recipe(GenericBaseRecipe):
 
@@ -76,28 +79,18 @@ class Recipe(GenericBaseRecipe):
       self.substituteTemplate(template, conf))
     path_list.append(conf_file)
 
-    wrapper = self.createPythonScript(
+    wrapper = self.createWrapper(
       self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['stunnel-binary'], conf_file]
+      (self.options['stunnel-binary'], conf_file),
     )
     path_list.append(wrapper)
 
-    if os.path.exists(pid_file):
-      with open(pid_file, 'r') as file_:
-        pid = file_.read().strip()
-      # Reload configuration
-      try:
-        os.kill(int(pid, 10), signal.SIGHUP)
-      except OSError, e:
-        if e.errno == errno.ESRCH: # No such process
-          os.unlink(pid_file)
-        else:
-          raise e
+    # Reload configuration
+    kill(pid_file, signal.SIGHUP)
 
     if 'post-rotate-script' in self.options:
-      self.createPythonScript(self.options['post-rotate-script'],
-                              __name__ + 'post_rotate',
-                              dict(pid_file=pid_file))
+      path_list.append(self.createPythonScript(
+        self.options['post-rotate-script'],
+        __name__ + '.kill', (pid_file,)))
 
     return path_list

slapos/recipe/tidstorage/__init__.py

@@ -36,19 +36,17 @@ class Recipe(GenericBaseRecipe):
     r = [configuration_file]
 
     wrapper = self.options.get('tidstorage-wrapper')
-    wrapper and r.append(self.createPythonScript(wrapper,
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['tidstoraged-binary'],  '--nofork', '--config',
-        configuration_file]))
+    wrapper and r.append(self.createWrapper(wrapper,
+      (self.options['tidstoraged-binary'],
+        '--nofork', '--config', configuration_file)))
 
-    r.append(self.createPythonScript(
+    r.append(self.createWrapper(
       self.options['repozo-wrapper'],
-      'slapos.recipe.librecipe.execute.execute',
-      [self.options['tidstorage-repozo-binary'],
+      (self.options['tidstorage-repozo-binary'],
         '--config', configuration_file,
         '--repozo', self.options['repozo-binary'],
         '--gzip',
         '--quick',
-      ]))
+       )))
 
     return r

slapos/recipe/waitfor.py

-##############################################################################
-#
-# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-import shlex
-
-from slapos.recipe.librecipe import GenericBaseRecipe
-
-class Recipe(GenericBaseRecipe):
-
-  def install(self):
-    files = [f for f in self.options['files'].split('\n') if f]
-    command_line = shlex.split(self.options['command-line'])
-    wrapper = self.createPythonScript(
-      self.options['wrapper'],
-      'slapos.recipe.librecipe.execute.execute_wait',
-      [ command_line,
-        files ],
-    )
-    return [wrapper]

slapos/recipe/wrapper.py

@@ -26,7 +26,6 @@
 ##############################################################################
 
 import shlex
-import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -38,56 +37,31 @@ class Recipe(GenericBaseRecipe):
 
     :param lines wait-for-files: list of files to wait for
     :param str pidfile: path to pidfile ensure exclusivity for the process
-    :param bool parameters-extra: whether wrapper parameters are passed onto command
+    :param str private-dev-shm: size of private /dev/shm, using user namespaces
     :param bool reserve-cpu: command will ask for an exclusive CPU core
     """
     def install(self):
-        command_line = shlex.split(self.options['command-line'])
+        args = shlex.split(self.options['command-line'])
         wrapper_path = self.options['wrapper-path']
         wait_files = self.options.get('wait-for-files')
-        environment = self.options.get('environment')
-        parameters_extra = self.options.get('parameters-extra')
         pidfile = self.options.get('pidfile')
-        reserve_cpu = self.options.get('reserve-cpu', False)
+        private_dev_shm = self.options.get('private-dev-shm')
 
-        if not wait_files and not environment:
-          # Create a simple wrapper as shell script
-          return [self.createWrapper(
-             name=wrapper_path,
-             command=command_line[0],
-             parameters=command_line[1:],
-             parameters_extra=parameters_extra,
-             pidfile=pidfile,
-             reserve_cpu=reserve_cpu
-          )]
+        environment = {}
+        for line in (self.options.get('environment') or '').splitlines():
+          line = line.strip()
+          if line:
+            k, v = line.split('=')
+            environment[k.rstrip()] = v.lstrip()
 
-        # More complex needs: create a Python script as wrapper
-
-        if wait_files is not None:
-            wait_files = [filename.strip() for filename in wait_files.split()
-                          if filename.strip()]
-        if environment is not None:
-            environment = dict((k.strip(), v.strip()) for k, v in [
-              line.split('=') for line in environment.splitlines() if line.strip() ])
-
-        # We create a python script and a wrapper around the python
-        # script because the python script might have a too long #! line
-        if os.path.exists(os.path.join(self.buildout['buildout']['directory'], "bin")): 
-          base_script_path = os.path.join(
-            self.buildout['buildout']['directory'], "bin/" + wrapper_path.split("/")[-1])
-        else:
-          base_script_path = os.path.join(
-            self.buildout['buildout']['directory'], wrapper_path.split("/")[-1])
-        python_script = self.createPythonScript(
-            base_script_path +'.py',
-            'slapos.recipe.librecipe.execute.generic_exec',
-            (command_line, wait_files, environment,), )
-        return [python_script, self.createWrapper(
-             name=wrapper_path,
-             command=python_script,
-             parameters=[],
-             parameters_extra=parameters_extra,
-             pidfile=pidfile,
-             reserve_cpu=reserve_cpu
-        )]
+        kw = {}
+        if wait_files:
+          kw['wait_list'] = wait_files.split()
+        if pidfile:
+          kw['pidfile'] = pidfile
+        if private_dev_shm:
+          kw['private_dev_shm'] = private_dev_shm
+        if self.isTrueValue(self.options.get('reserve-cpu')):
+          kw['reserve_cpu'] = True
 
+        return self.createWrapper(wrapper_path, args, environment, **kw)

slapos/recipe/zabbixagent/__init__.py

@@ -71,10 +71,10 @@ class Recipe(BaseSlapRecipe):
     self._createDirectory(cron_d)
     self._createDirectory(crontabs)
     wrapper = zc.buildout.easy_install.scripts([('crond',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.wrapper_directory, arguments=[
+      'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
+      sys.executable, self.wrapper_directory, arguments=repr((
         self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
-        '-t', timestamps, '-f', '-l', '5', '-M', catcher]
+        '-t', timestamps, '-f', '-l', '5', '-M', catcher))[1:-1]
       )[0]
     self.path_list.append(wrapper)
     return cron_d
@@ -101,10 +101,10 @@ class Recipe(BaseSlapRecipe):
     self.path_list.append(zabbix_agentd_path)
 
     wrapper = zc.buildout.easy_install.scripts([('zabbixagentd',
-      'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
-      self.bin_directory, arguments=[
+      'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
+      sys.executable, self.bin_directory, arguments=repr((
         self.options['zabbix_agentd_binary'].strip(), '-c',
-        zabbix_agentd_path])[0]
+        zabbix_agentd_path))[1:-1])[0]
 
     self.path_list.extend(zc.buildout.easy_install.scripts([
       ('zabbixagentd', __name__ + '.svcdaemon', 'svcdaemon')],

slapos/recipe/zeo/__init__.py

@@ -89,10 +89,9 @@ class Recipe(GenericBaseRecipe):
         self.substituteTemplate(template_filename, config))
 
     # Create running wrapper
-    wrapper_path = self.createPythonScript(
+    wrapper_path = self.createWrapper(
       self.options['wrapper-path'],
-      'slapos.recipe.librecipe.execute.execute',
-      arguments=[self.options['binary-path'].strip(), '-C',
-        self.options['conf-path']],)
+      (self.options['binary-path'].strip(),
+        '-C', self.options['conf-path']))
 
     return [configuration_path, wrapper_path]

slapos/test/recipe/test_re6stnet.py

@@ -79,15 +79,14 @@ class Re6stnetTest(unittest.TestCase):
     token_file = os.path.join(self.options['conf-dir'], 'token.json')
     with open(path, 'r') as f:
         content = f.read()
-    self.assertIn("'token_json': '%s'" % token_file, content)
+    self.assertIn("('http://%s:%s/', %r, %r," % (
+      self.options['ipv4'], self.options['port'], self.token_dir, token_file),
+      content)
     self.assertIn("'partition_id': 'slappart0'", content)
-    self.assertIn("'computer_id': 'comp-test'", content)
+    self.assertIn("'computer_guid': 'comp-test'", content)
     self.assertIn("'key_file': '/path/to/key'", content)
     self.assertIn("'cert_file': '/path/to/cert'", content)
-    self.assertIn("'server_url': 'http://server.com'", content)
-    self.assertIn("'token_base_path': '%s'" % self.token_dir, content)
-    self.assertIn("'registry_url': 'http://%s:%s/'" % (self.options['ipv4'],
-                                                self.options['port']), content)
+    self.assertIn("'master_url': 'http://server.com'", content)
   
   def fake_generateCertificates(self):
     return

software/agent/software.cfg

@@ -50,8 +50,8 @@ gitdb = 0.6.4
 pycrypto = 2.6.1
 pycurl = 7.43.0
 slapos.recipe.download = 1.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/apache-frontend/software.cfg

@@ -10,8 +10,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 numpy = 1.11.2
 pyasn1 = 0.2.3

software/cdn-me/software.cfg

@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 erp5.util = 0.4.49
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/cloudooo/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[template-cloudooo]
+filename = instance.cfg.in 
+md5sum = 6e4431cf4b0a0d034402604b1e2844c0
+
+[template-cloudooo-instance]
+filename = instance-cloudooo.cfg.in
+md5sum = afbfed1d762e5cdf7c6fd1292e7b28e7

software/cloudooo/instance-cloudooo.cfg.in

@@ -10,6 +10,7 @@
 {%   set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%}
 {% endif -%}
 {% do assert(next_port > 0) -%}
+{% set next_port = itertools.count(next_port).next -%}
 
 {% set backend_count = instance_parameter_dict['backend-count'] | int -%}
 {% if instance_parameter_dict.get('backend-count-parameter-name') -%}
@@ -33,9 +34,8 @@
 {%   set mimetype_entry_addition = mimetype_entry_addition ~ "\n" ~ slapparameter_dict.get(instance_parameter_dict['mimetype-entry-addition-parameter-name'], '') -%}
 {% endif -%}
 
-{% set apache_port = next_port -%}
-{% set haproxy_port = next_port + 1 -%}
-{% set next_port = next_port + 2 -%}
+{% set apache_port = next_port() -%}
+{% set haproxy_port = next_port() -%}
 
 {% set apache_ip_list = [ipv4] -%}
 {% if ipv6_set -%}
@@ -184,13 +184,12 @@ ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
 {%   set name = 'cloudooo-' ~ index -%}
 [{{ cloudooo(name) }}]
 < = cloudooo-base
-port = {{ next_port }}
-openoffice-port = {{ next_port + 1 }}
+port = {{ next_port() }}
+openoffice-port = {{ next_port() }}
 configuration-file = ${directory:etc}/{{ name }}.cfg
 data-directory = ${directory:srv}/{{ name }}
 wrapper = ${directory:services}/{{ name }}
 
-{%   set next_port = next_port + 2 -%}
 {% endfor -%}
 
 [haproxy]

software/cloudooo/instance.cfg.in

@@ -39,6 +39,7 @@ extensions = jinja2.ext.do
 extra-context =
     section parameter_dict cloudooo-dynamic-template-parameter-dict
     key ipv4_set slap-parameters:ipv4
+    import itertools itertools
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype

software/cloudooo/software-common.cfg

 [buildout]
 extends =
+  buildout.hash.cfg
   ../../stack/cloudooo.cfg
   ../../stack/logrotate/buildout.cfg
 parts =
@@ -25,8 +26,7 @@ cert = ${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 rendered = ${buildout:directory}/template.cfg
-template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 4543e62c41726ef074454252c4951ac8
+template = ${:_profile_base_location_}/${:filename}
 mode = 640
 
 [template-cloudooo-base]
@@ -81,6 +81,5 @@ template-logrotate-base = ${template-logrotate-base:rendered}
 
 [template-cloudooo-instance]
 recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/instance-cloudooo.cfg.in
-md5sum = 045830db85e1200f765788aa8b62289e
+url = ${:_profile_base_location_}/${:filename}
 mode = 640

software/dream/software.cfg

@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
 # indirect dependancies
 cp.recipe.cmd = 0.5
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 zope.exceptions = 4.0.7
 zope.testing = 4.1.3
 zc.recipe.testrunner = 2.0.0

software/erp5/README.rst

@@ -3,12 +3,8 @@ Available ``software-type`` values
 
 - ``default``
 
-  Recommended for production use.
-
-- ``create-erp5-site``
-
-  Automated creation of ERP5Site instance, for easy deployment.
-  Usage in production discouraged due to the increased risk of data loss.
+  Recommended for developemnt and production use. Automatic creation of
+  erp5-site.
 
 Notes
 =====

software/erp5/instance-erp5-input-schema.json

@@ -117,7 +117,11 @@
             "timerserver-interval": {
               "description": "Timerserver tick perdiod, in seconds, or 0 to disable",
               "default": 5,
-              "type": "integer"
+              "type": "number"
+            },
+            "private-dev-shm": {
+              "description": "Size of private /dev/shm for wendelin.core. If sysctl kernel.unprivileged_userns_clone exists, it must be set to 1.",
+              "type": "string"
             },
             "ssl-authentication": {
               "title": "Enable SSL Client authentication on this zope instance.",

software/erp5/software.cfg.json

@@ -9,13 +9,6 @@
       "request": "instance-erp5-input-schema.json",
       "response": "instance-erp5-output-schema.json",
       "index": 0
-    },
-    "create-erp5-site": {
-      "title": "Create ERP5 Site", 
-      "description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
-      "request": "instance-erp5-input-schema.json",
-      "response": "instance-erp5-output-schema.json",
-      "index": 1
     }
   }
 }

software/erp5testnode/software.cfg

@@ -59,7 +59,7 @@ mode = 0644
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.50
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 ipython = 5.3.0
 apache-libcloud = 2.1.0
 gitdb2 = 2.0.2

software/erp5testnode/testsuite/deploy-test/runTestSuite.py

@@ -106,6 +106,7 @@ def waitForSite(partition_path):
       date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
       duration=end - start,
   )
+  print(try_info + 'status_dict %r' % (status_dict,))
   return status_dict
 
 

software/erp5testnode/testsuite/dummy/software.cfg

@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/gitlab/software.cfg

@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
 docutils = 0.12
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2+slapos001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 z3c.recipe.scripts = 1.0.1

software/html5as/software.cfg

@@ -72,7 +72,7 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 plone.recipe.command = 1.1

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/jstestnode/software.cfg

@@ -113,5 +113,5 @@ mode = 0644
 
 [versions]
 erp5.util = 0.4.50
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 selenium = 3.8.0

software/jupyter/software.cfg

@@ -85,7 +85,7 @@ pyzmq = 16.0.2
 scikit-learn = 0.18.1
 seaborn = 0.7.1
 simplegeneric = 0.8.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 statsmodels = 0.8.0
 terminado = 0.6
 tornado = 4.4.2

software/kumofs/instance.cfg

-[buildout]
-parts =
-  instance
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-[instance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-kumo_gateway_binary = ${kumo:location}/bin/kumo-gateway
-kumo_manager_binary = ${kumo:location}/bin/kumo-manager
-kumo_server_binary = ${kumo:location}/bin/kumo-server
-dcrond_binary = ${dcron:location}/sbin/crond
-openssl_binary = ${openssl:location}/bin/openssl
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-stunnel_binary = ${stunnel:location}/bin/stunnel

software/kumofs/software.cfg

-[buildout]
-
-extends =
-  ../../component/kumo/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/stunnel/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../stack/slapos.cfg
-
-parts =
-# Create instance template
-  template
-  rdiff-backup
-  dcron
-  kumo
-  stunnel
-  eggs
-  instance-recipe-egg
-
-[instance-recipe]
-egg = slapos.cookbook
-module = kumofs
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 056a4af7128fd9e31da42c85cc039420
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[versions]
-rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 2.4.2

software/kvm/common.cfg

@@ -41,6 +41,7 @@ parts = ${:common-parts}
 
 [eggs]
 recipe = zc.recipe.egg
+interpreter = python.eggs
 eggs =
   ${python-cffi:egg}
   ${python-cryptography:egg}
@@ -89,7 +90,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bf5ef731c0d8da0267a4939882b4eeee 
+md5sum = 5a17fc127190bbc19361c5ffb10711b3 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
+md5sum = 68b66fb3e9020642e57f4a4ee266f2b3
 download-only = true
 on-update = true
 
@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
 ignore-existing = true
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 05b6004e8c7a94de14f247affcef4971
+md5sum = ba3337b3678ed9d3578cc88749c5cd13
 download-only = true
 on-update = true
 
@@ -185,7 +186,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/template-kvm-run.in
 mode = 644
 filename = template-kvm-run.in
-md5sum = 178a24cdad77cb6c2e519ac629dd0e74
+md5sum = bd238397af6236b6b24b693012feeece
 download-only = true
 on-update = true
 
@@ -195,7 +196,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/kvm-controller-run.in
 mode = 644
 filename = kvm-controller-run.in
-md5sum = 71afd2d13f6e56993ae413a168e012d7
+md5sum = c86cd67bbdd26b7b14b7449a1bbd959b
 download-only = true
 on-update = true
 
@@ -225,7 +226,7 @@ ignore-existing = true
 url = ${:_profile_base_location_}/template/qemu-is-ready.in
 mode = 644
 filename = qemu-is-ready.in
-md5sum = 0066fa0f5f3dd47bded5e5924df2550d
+md5sum = b304eec8e2cb71f10ea83cac22f6db12
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster-input-schema.json

@@ -189,13 +189,35 @@
               "default": "started",
               "enum": ["started", "stopped"]
             },
+            "enable-device-hotplug": {
+              "title": "Enable device hotplug mode",
+              "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+              "type": "boolean",
+              "default": false
+            },
             "ram-size": {
               "title": "RAM size",
               "description": "RAM size, in MB.",
               "type": "integer",
               "default": 1024,
-              "minimum": 128,
-              "multipleOf": 128
+              "minimum": 1024,
+              "multipleOf": 512
+            },
+            "ram-max-size": {
+              "title": "Maximum RAM size, in MB",
+              "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
+              "type": "integer",
+              "default": 51200,
+              "minimum": 1024,
+              "multipleOf": 512
+            },
+            "ram-hotplug-slot-size": {
+              "title": "Size of Hotpluggable RAM slot, in MB",
+              "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+              "type": "integer",
+              "default": 512,
+              "minimum": 512,
+              "multipleOf": 512
             },
             "auto-ballooning": {
               "title": "Enable qemu auto ballooning.",
@@ -245,10 +267,13 @@
               "type": "integer",
               "minimum": 1
             },
-            "cpu-options": {
-              "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-              "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-              "type": "string"
+            "cpu-max-count": {
+              "title": "Maximum CPU amount",
+              "description": "Specifies the maximum number of CPUs.",
+              "type": "integer",
+              "default": 24,
+              "minimum": 1,
+              "maximum": 64
             },
             "numa": {
               "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -8,7 +8,7 @@
 {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
 {% set kvm_instance_dict = {} -%}
 {% set kvm_hostname_list = [] -%}
-{% set monitor_url_list = [] -%}
+{% set monitor_base_url_dict = {} -%}
 
 {% macro setconfig(name, value) -%}
 {# will set a config-name = value if value is not empty -#}
@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
 config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
 config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
 config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
+config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '51200')) }}
+config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
+config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
 config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
 config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
 config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
 {{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
 {{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
-{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
 {{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
 {{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
 
@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
 {{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
 
 {% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
-{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }}
+{{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
 config-publish-nat-url = True
 config-use-nat = {{ use_nat }}
 config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
@@ -124,10 +127,10 @@ return =
 {% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
 {{ '  ' }}tap-ipv4
 
-{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% endif -%}
+{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
 {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
 username = admin
 password = ${monitor-htpasswd:passwd}
 
-[monitor-conf-parameters]
-monitor-url-list +=
-{% for url in monitor_url_list -%}
-{{ '  ' ~ url }}
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
 {% endfor %}
+
+[monitor-conf-parameters]
 private-path-list += 
   ${directory:webroot}/
 

software/kvm/instance-kvm-input-schema.json

@@ -4,13 +4,35 @@
 
   "title": "Input Parameters",
   "properties": {
+    "enable-device-hotplug": {
+      "title": "Enable device hotplug mode",
+      "description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
+      "type": "boolean",
+      "default": false
+    },
     "ram-size": {
       "title": "RAM size",
       "description": "RAM size, in MB.",
       "type": "integer",
       "default": 1024,
-      "minimum": 128,
-      "multipleOf": 128
+      "minimum": 1024,
+      "multipleOf": 512
+    },
+    "ram-max-size": {
+      "title": "Maximum RAM size, in MB",
+      "description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
+      "type": "integer",
+      "default": 51200,
+      "minimum": 1024,
+      "multipleOf": 512
+    },
+    "ram-hotplug-slot-size": {
+      "title": "Size of Hotpluggable RAM slot, in MB",
+      "description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
+      "type": "integer",
+      "default": 512,
+      "minimum": 512,
+      "multipleOf": 512
     },
     "auto-ballooning": {
       "title": "Enable qemu auto ballooning.",
@@ -54,10 +76,25 @@
       "type": "integer",
       "minimum": 1
     },
-    "cpu-options": {
-      "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
-      "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
-      "type": "string"
+    "cpu-max-count": {
+      "title": "Maximum CPU amount",
+      "description": "Specifies the maximum number of CPUs.",
+      "type": "integer",
+      "default": 24,
+      "minimum": 1,
+      "maximum": 64
+    },
+    "max-cpu-hotplug-count": {
+      "title": "Maximum hotpluggable CPU amount",
+      "description": "Specifies the maximum number of hotpluggable CPUs.",
+      "type": "integer",
+      "default": 1
+    },
+    "cpu-hotplug-amount": {
+      "title": "CPU hotplug amount",
+      "description": "Specifies the number of CPUs to hotplug",
+      "type": "integer",
+      "default": 0
     },
     "numa": {
       "title": "Simulate a multi node NUMA system.",

software/kvm/instance-kvm.cfg.jinja2

@@ -5,6 +5,7 @@
 {% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
 {% set name = slapparameter_dict.get('name', 'localhost') -%}
 {% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
+{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
 {% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set frontend_software_type = 'default' -%}
@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
 bytes = 8
 
 [kvm-controller-parameter-dict]
-python-path = {{ python_executable }}
+python-path = {{ python_eggs_executable }}
 vnc-passwd = ${gen-passwd:passwd}
 socket-path = ${directory:var}/qmp_socket
+pid-file = ${directory:run}/pid_file
 kvm-status-path = ${directory:var}/qemu-vm-is-ready
+cpu-count = ${slap-parameter:cpu-count}
+ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
+ram-size = ${slap-parameter:ram-size}
+enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
 
 [kvm-parameter-dict]
 python-path = {{ python_executable }}
@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
 disk-type = ${slap-parameter:disk-type}
 
 
-pid-file-path = ${directory:run}/pid_file
+pid-file-path = ${kvm-controller-parameter-dict:pid-file}
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 
-smp-count = ${slap-parameter:cpu-count}
-smp-options = ${slap-parameter:cpu-options}
-ram-size = ${slap-parameter:ram-size}
-numa = ${slap-parameter:numa}
+enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
+smp-count = ${kvm-controller-parameter-dict:cpu-count}
+smp-max-count = ${slap-parameter:cpu-max-count}
+
+ram-size = ${kvm-controller-parameter-dict:ram-size}
+ram-max-size = ${slap-parameter:ram-max-size}
+init-ram-size = 1024
 mac-address = ${create-mac:mac-address}
 tap-mac-address = ${create-tap-mac:mac-address}
 
@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
 disk-aio = ${slap-parameter:disk-aio}
 auto-ballooning = ${slap-parameter:auto-ballooning}
 machine-options = ${slap-parameter:machine-options}
-cpu-options = ${slap-parameter:cpu-model}
+cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
 
 log-file = ${directory:log}/qemu.log
 
@@ -550,15 +559,15 @@ nbd-host =
 nbd2-port = 1024
 nbd2-host =
 
+enable-device-hotplug = False
 ram-size = 1024
+ram-max-size = 51200
+ram-hotplug-slot-size = 512
 disk-size = 10
 disk-type = virtio
 
 cpu-count = 1
-# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
-cpu-options = 
-# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
-numa = 
+cpu-max-count = 24
 disk-cache = writeback
 disk-aio = native
 auto-ballooning = True

software/kvm/instance.cfg.in

@@ -93,6 +93,7 @@ context =
     raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
+    raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
     raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
     raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
     raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}

software/kvm/software.cfg

@@ -5,13 +5,13 @@ extends = common.cfg
 # XXX - use websockify = 0.5.1 for compatibility with kvm frontend
 websockify = 0.5.1
 
-slapos.toolbox = 0.73
+slapos.toolbox = 0.74
 erp5.util = 0.4.49
 apache-libcloud = 1.1.0
 collective.recipe.environment = 0.2.0
 gitdb = 0.6.4
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 smmap = 0.9.0
 # websockify = 0.8.0
 
@@ -45,4 +45,4 @@ paramiko = 2.0.2
 
 # Required by:
 # slapos.toolbox==0.71
-passlib = 1.6.5
+passlib = 1.6.5
\ No newline at end of file

software/kvm/template/kvm-controller-run.in

@@ -6,40 +6,62 @@
 import socket
 import time
 import os
+from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
 
 # XXX: to be factored with slapos.toolbox qemu qmp wrapper.
 
 socket_path = '{{ parameter_dict.get("socket-path") }}'
+pid_file = '{{ parameter_dict.get("pid-file") }}'
 vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
 status_path = '{{ parameter_dict.get("kvm-status-path") }}'
+cpu_amount = {{ parameter_dict.get("cpu-count") }}
+cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
+slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
+ram_size = {{ parameter_dict.get("ram-size") }}
+enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
 
-if os.path.exists(status_path):
-  os.unlink(status_path)
+def write(message):
+  with open(status_path, 'w') as status_file:
+    status_file.write(message)
+
+def update():
 
-# Connect to KVM qmp socket
-so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-connected = False
-while not connected:
   try:
-    so.connect(socket_path)
-  except socket.error:
-    time.sleep(1)
-  else:
-    connected = True
-data = so.recv(1024)
-
-# Enable qmp
-so.send('{ "execute": "qmp_capabilities" }')
-data = so.recv(1024)
-
-# Set VNC password
-so.send('{ "execute": "change", ' \
-        '"arguments": { "device": "vnc", "target": "password", ' \
-        '               "arg": "' + vnc_password + '" } }')
-data = so.recv(1024)
-
-# Finish
-so.close()
-
-with open(status_path, 'w') as status_file:
-  status_file.write("OK")
+    init_dict = getInitialQemuResourceDict(pid_file)
+    if os.path.exists(status_path):
+      os.unlink(status_path)
+
+    if init_dict is None:
+      # qemu process is not OK
+      return
+    init_ram_size = int(init_dict['ram'].split('M')[0])
+    if cpu_amount < 1:
+      raise ValueError("CPU should be at least equal to 1.")
+    hotplug_ram = ram_size - init_ram_size
+    if hotplug_ram < 0:
+      raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
+
+    qemu_wrapper = QemuQMPWrapper(socket_path)
+    qemu_wrapper.setVNCPassword(vnc_password)
+    if enable_device_hotplug == 'true':
+      write("Qemu Controller is still running...")
+      qemu_wrapper.updateDevice({
+        'device': 'cpu',
+        'amount': cpu_amount,
+        'model': cpu_model
+      })
+      qemu_wrapper.updateDevice({
+        'device': 'memory',
+        'mem': hotplug_ram,
+        'slot': slot_hotplug_size,
+        'nslot': 128,
+        'canreboot': 1
+      })
+  except Exception, e:
+    write(str(e))
+    raise
+
+  write("")
+
+if __name__ == "__main__":
+  update()

software/kvm/template/qemu-is-ready.in

 #!{{ dash }}
 
-if [ -f "{{ qemu_ready_path }}" ]; then
-  echo "VM correctly started."
+FILE="{{ qemu_ready_path }}"
+# don't start checks too fast
+sleep 2
+if [ -f "$FILE" ]; then
+  if [ "$(cat $FILE)" = "" ]; then
+    echo "VM correctly started."
+  else
+    >&2 echo "Qemu Controller failed"
+    >&2 cat $FILE
+    exit 1
+  fi
 else
   log_file="{{ qemu_service_log_file }}"
   >&2 echo "Qemu process is not correctly started."

software/kvm/template/template-kvm-run.in

@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
 listen_ip = '{{ parameter_dict.get("ipv4") }}'
 mac_address = '{{ parameter_dict.get("mac-address") }}'
 tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
-smp_count = '{{ parameter_dict.get("smp-count") }}'
-smp_options = '{{ parameter_dict.get("smp-options") }}'.strip()
-numa_list = '{{ parameter_dict.get("numa") }}'.split()
-ram_size = '{{ parameter_dict.get("ram-size") }}'
+numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
+ram_size = {{ parameter_dict.get("ram-size") }}
+ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
+init_ram_size = {{ parameter_dict.get("init-ram-size") }}
 pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
 external_disk_number = {{ parameter_dict.get("external-disk-number") }}
-external_disk_size = '{{ parameter_dict.get("external-disk-size") }}'
+external_disk_size = {{ parameter_dict.get("external-disk-size") }}
 external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
 disk_storage_dict = {}
 disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
 disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
 disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
               disk_cache == "directsync" else "threads"
+smp_count = {{ parameter_dict.get("smp-count") }}
+smp_max_count = {{ parameter_dict.get("smp-max-count") }}
 machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
 cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
 
+enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
+
 logfile = '{{ parameter_dict.get("log-file") }}'
 
 if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
@@ -255,15 +259,14 @@ if use_tap == 'true':
             tap_interface, vhost),
           '-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
 
-smp = smp_count
-if smp_options:
-  for option in smp_options.split(','):
-    key, val = option.split('=')
-    if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
-      smp += ',%s=%s' % (key, val)
+if enable_device_hotplug != 'true':
+  smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
+  ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
+else:
+  smp = '1,maxcpus=%s' % smp_max_count
+  ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
 kvm_argument_list = [qemu_path,
-  '-enable-kvm', '-smp', smp, '-name', vm_name,
-  '-m', ram_size, '-vga', 'std',
+  '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
   '-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
   '-vnc', '%s:1,ipv4,password' % listen_ip,
   '-boot', 'order=cd,menu=on',

software/memcached/instance.cfg

-[buildout]
-parts =
-  instance
-
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-
-[instance]
-recipe = ${instance-recipe:egg}:${instance-recipe:module}
-dcrond_binary = ${dcron:location}/sbin/crond
-memcached_binary = ${memcached:location}/bin/memcached
-openssl_binary = ${openssl:location}/bin/openssl
-rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
-stunnel_binary = ${stunnel:location}/bin/stunnel

software/memcached/software.cfg

-[buildout]
-extends =
-  ../../component/memcached/buildout.cfg
-  ../../component/dcron/buildout.cfg
-  ../../component/stunnel/buildout.cfg
-  ../../component/rdiff-backup/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
-  ../../stack/slapos.cfg
-
-parts =
-# Create instance template
-  template
-  eggs
-  instance-recipe-egg
-
-[instance-recipe]
-egg = slapos.cookbook
-module = memcached
-
-[instance-recipe-egg]
-recipe = zc.recipe.egg
-eggs = ${instance-recipe:egg}
-
-[eggs]
-recipe = zc.recipe.egg
-eggs =
-  ${lxml-python:egg}
-
-[template]
-# Default template for the instance.
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg
-md5sum = 837caf9897332a5f70c72438f1dc5bae
-output = ${buildout:directory}/template.cfg
-mode = 0644
-
-[versions]
-plone.recipe.command = 1.1
-slapos.recipe.template = 2.3

software/nayuos/software.cfg

@@ -17,7 +17,7 @@ parts +=
 versions = versions
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 
 [template-instance]

software/neoppod/software-common.cfg

@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
 persistent = 4.2.3
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap2 = 2.0.1
 transaction = 1.7.0
 zodbpickle = 0.6.0

software/neotest/instance.cfg.in

-# TODO instance which runs this test periodically automatically and ingests results to ERP5
+# NEO test instance: run neotest under Nexedi testing infrastructure
+[buildout]
+parts = runTestSuite
+
+# std stuff for slapos instance
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+offline = true
+
+# software release we instantiate was supplied here
+[software]
+dir     = {{ buildout['directory'] }}
+bin     = {{ buildout['bin-directory'] }}
+
+# instance directories
+[directory]
+recipe  = slapos.cookbook:mkdirectory
+home    = ${buildout:directory}
+bin     = ${:home}/bin
+neotest = ${:home}/neotest
+
+# script to run the testsuite from inside <instance>/neotest/
+# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
+[runTestSuite]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${:_buildout_section_name_}
+command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
+# vvv appends "$@" to argv ^^^ without shell-escaping
+parameters-extra = true

software/neotest/neotest-env.sh.in

@@ -2,6 +2,8 @@
 
 . ${gowork:env.sh}
 
+PATH="${coreutils:location}/bin:$PATH"
+PATH="${util-linux:location}/bin:$PATH"
 PATH="${ethtool:location}/sbin:$PATH"
 PATH="${ioping:location}/bin:$PATH"
 PATH="${lmbench:location}/bin:$PATH"

software/neotest/neotest-runTestSuite.in

+#!/bin/bash -e
+# neotest's runTestSuite wraper so it could be run without any environment preset
+
+. ${buildout:directory}/neotest-env.sh
+exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"

software/neotest/software.cfg

@@ -12,6 +12,8 @@ extends =
     ../../component/ethtool/buildout.cfg
     ../../component/ioping/buildout.cfg
     ../../component/lmbench/buildout.cfg
+    ../../component/coreutils/buildout.cfg
+    ../../component/util-linux/buildout.cfg
 
 parts =
     gowork
@@ -19,6 +21,11 @@ parts =
     ioping
     ethtool
 
+#   neotest uses realpath & friends
+    coreutils
+#   ----//---- lsblk
+    util-linux
+
     neoppod-develop
     neoppod
     wendelin.core-dev
@@ -26,11 +33,12 @@ parts =
 
     neotest-env.sh
     neotest
+    neotest-runTestSuite
 
 #   for instance
     slapos-deps-eggs
     slapos-cookbook
-#   instance.cfg
+    instance.cfg
 
 
 # go packages to install (+ automatically their dependencies)
@@ -53,7 +61,7 @@ output  = ${buildout:directory}/${:_buildout_section_name_}
 
 [neotest-env.sh]
 <= buildout-template
-md5sum  = 6e2203c7a86e8a3e8e56b8086115f5d4
+md5sum  = 595eebbbcb56b6a8464d48833cfae57b
 
 [neotest]
 <= buildout-template
@@ -61,11 +69,25 @@ output  = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode    = 0755
 md5sum  = fb3b4109128c1db1739ef5bb6abd1d94
 
+[neotest-runTestSuite]
+<= buildout-template
+output  = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+md5sum  = 6a4281730b68cdba5c873817a6754428
+
+
+# instance
+[jinja2-template]
+recipe  = slapos.recipe.template:jinja2
+template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
+rendered= ${buildout:directory}/${:_buildout_section_name_}
+mode    = 0644
+context =
+  section buildout  buildout
 
-# instance (TODO)
 [instance.cfg]
-<= buildout-template
-md5sum  = 410e1b2d72829824b28cc0299adb472e
+<= jinja2-template
+md5sum  = 7bd68ed8842cd25301bf04bcdcef88f6
 
 
 # eggs:
@@ -80,6 +102,8 @@ eggs    =
     wendelin.core
 # for ZEO scripts (runzeo)
     ZEO
+# for nxd/runTestSuite
+    erp5.util
 
 
 # wendelin.core: latest not yet released
@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
 ZODB3 = 3.11.0
 numpy = 1.13.3
 zope.testing = 4.6.2
+erp5.util = 0.4.50
 
 # Required by:
 # ZEO==4.3.1
@@ -103,7 +128,3 @@ ZConfig = 3.2.0
 # ZEO==4.3.1
 # ZODB==4.4.5
 zc.lockfile = 1.2.1
-
-# Required by:
-# slapos.toolbox==0.73
-erp5.util = 0.4.49

software/nginx-push-stream/software.cfg

@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
 mode = 0644
 
 [versions]
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
+dnspython = 1.15.0
+PyRSS2Gen = 1.1
+erp5.util = 0.4.50
+passlib = 1.7.1
+GitPython = 2.1.8
+lockfile = 0.12.2
+apache-libcloud = 2.2.1
+feedparser = 5.2.1
+atomize = 0.2.0
+inotifyx = 0.2.2
+gitdb2 = 2.0.3
+pyasn1 = 0.4.2
+smmap2 = 2.0.3

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -81,7 +81,7 @@ username = admin
 [monitor-instance-parameter]
 monitor-httpd-port = 8099
 cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
-username = ${monitor-instance-parameter:username}
+username = admin
 password = ${monitor-htpasswd:passwd}
 
 [monitor-conf-parameters]
@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }}
 {% endfor -%}
 config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
 config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }}
-config-monitor-username = ${monitor-instance-parameter:username}
+config-monitor-username = admin
 config-monitor-password = ${monitor-htpasswd:passwd}
 
 [publish-information]

software/powerdns/software.cfg

@@ -38,7 +38,7 @@ mode = 640
 [template-dns-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
-md5sum = 3699d0751632e0c11b466342789e27a3
+md5sum = 46acd4ed071df8d7139dcd0434be42eb 
 mode = 0644
 
 [iso-list]
@@ -65,4 +65,4 @@ mode = 0644
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/re6stnet/software.cfg

@@ -110,8 +110,8 @@ gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slapos-in-partition/software.cfg

@@ -11,4 +11,4 @@ extends = common.cfg
 Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slapos-master/buildout.hash.cfg

@@ -15,7 +15,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 4a77ee4a6367fee27552f8bfe9d87aab
+md5sum = 7a1d2fe310764f5c06f636e8850c7d63
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -206,6 +206,7 @@ name = {{ partition_name }}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
+config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
 config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
@@ -327,7 +328,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
+'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

software/slapos-testing/software.cfg

@@ -97,4 +97,4 @@ mode = 640
 Pygments = 2.1.3
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/slaprunner/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 7df86928aeef0fbada832b0283b9a454
+md5sum = 8b1caca52ab1307343ebada59f5a6c66
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
 
 [instance-runner-export]
 filename = instance-runner-export.cfg.in
-md5sum = 00ddcaa6e571505d1930612d12de8fde
+md5sum = d1122988652127af5e252f1503eeacab
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2

software/slaprunner/instance-runner-export.cfg.in

@@ -7,11 +7,13 @@ parts +=
   nginx-launcher
   certificate-authority
   ca-nginx
+  logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
   publish-connection-information
   slaprunner-promise
   apache-httpd-promise
+  logrotate-entry-apache-httpd
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
   runner-sshd-graceful
@@ -30,6 +32,7 @@ parts +=
   bash-profile
   supervisord-wrapper
   supervisord-promise
+  logrotate-entry-supervisord
   httpd-graceful-wrapper
 ## Monitoring part
 ## Monitor for runner

software/slaprunner/instance-runner.cfg

@@ -4,10 +4,12 @@ parts =
   nginx-launcher
   certificate-authority
   ca-nginx
+  logrotate-entry-nginx
   gunicorn-launcher
   gunicorn-graceful
   publish-connection-information
   slaprunner-promise
+  logrotate-entry-apache-httpd
   apache-httpd-promise
   slaprunner-supervisord-wrapper
   runner-sshd-add-authorized-key
@@ -27,6 +29,7 @@ parts =
   bash-profile
   supervisord-wrapper
   supervisord-promise
+  logrotate-entry-supervisord
   httpd-graceful-wrapper
 {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
   slaprunner-frontend-promise
@@ -347,6 +350,12 @@ mode = 700
 context =
     section param_nginx_frontend nginx-frontend
 
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
+post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
+
 [httpd-parameters]
 path_pid = $${directory:run}/httpd.pid
 path_error_log = $${directory:log}/httpd-error.log
@@ -387,6 +396,12 @@ wait-for-files =
   $${ca-nginx:cert-file}
   $${ca-nginx:key-file}
 
+[logrotate-entry-apache-httpd]
+<= logrotate-entry-base
+name = apache
+log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
+post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
+
 [httpd-graceful-wrapper]
 recipe = collective.recipe.template
 input = inline:
@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
 command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
 
+[logrotate-entry-supervisord]
+<= logrotate-entry-base
+name = supervisord
+log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
+post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
+
 [supervisord-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/supervisord

software/slaprunner/software.cfg

@@ -15,8 +15,8 @@ gitdb = 0.6.4
 gunicorn = 19.7.1
 prettytable = 0.7.2
 pycurl = 7.43.0
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap = 0.9.0
 
 # Required by:

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.28
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/varnish/software.cfg

@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
 mode = 640
 
 [versions]
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3

software/wendelin-scalability/test-neo.cfg

@@ -10,12 +10,15 @@
 #   $x 8.787 0
 #   0.100036621094
 #   6556
+#
+# The 'start_process' command is similar but by growing a ZBigArray object.
+# The random data has a compression ratio of 10%.
 
 [buildout]
 extends = test-common.cfg
-parts += start_ingest
+parts += start_ingest start_process
 
-[start_ingest]
+[start-script-common]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode = 0755
@@ -27,20 +30,35 @@ template =
   _('--site-id', default='erp5')
   _('hostport', metavar='host[:port]', help='Zope address')
   _('password', help="'zope' user password")
-  _('reference', help='Data Stream reference')
-  _('mu', type=float)
-  _('sigma', type=float)
-  _('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+  options = []
+  def option(name, **kw):
+    _(name, **kw)
+    options.append(name)
+  ${:options}
   args = parser.parse_args()
   qs = []
-  for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
+  for k in options:
     v = getattr(args, k)
     if v is not None:
       t = type(v)
       qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
   c = httplib.HTTPConnection(args.hostport)
-  c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
-                      % (args.site_id, '&'.join(qs)))
+  c.putrequest('GET', '/%s/${:script}?%s' % (args.site_id, '&'.join(qs)))
   c.putheader('Authorization',
     'Basic ' + base64.b64encode('zope:'+args.password))
   c.endheaders()
+
+[start_ingest]
+<= start-script-common
+options =
+  option('id', help='Data Stream id')
+  option('mu', type=float)
+  option('sigma', type=float)
+  option('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+script = ERP5Site_simulateFluentdIngestion
+
+[start_process]
+<= start-script-common
+options =
+  option('id', help='Data Array id')
+script = ERP5Site_dummyZBigArrayProcessing

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py

@@ -5,7 +5,7 @@ from random import lognormvariate
 
 bigfile_chunk_size = 65536
 
-def simulateFluentdIngestion(self, reference, mu, sigma,
+def simulateFluentdIngestion(self, id, mu, sigma,
                              chunks_per_transaction=128):
   from time import time
   import transaction
@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma,
           + '/ingest')
   module = self['data_stream_module']
   try:
-    data_stream = module[reference]
+    data_stream = module[id]
   except KeyError:
-    data_stream = module.newContent(reference, 'Data Stream')
+    data_stream = module.newContent(id, 'Data Stream')
     transaction.commit()
 
   pack = struct.Struct('!d').pack

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.py

+def dummyZBigArrayProcessing(self, id):
+  import numpy as np
+  from random import randrange, sample
+  import transaction
+  module = self['data_array_module']
+  try:
+    array = module[id]
+  except KeyError:
+    array = module.newContent(id, 'Data Array')
+    array.initArray(shape=(0, 64), dtype=np.int32)
+    transaction.commit()
+  note = array.getPath() + '/new_data'
+  array = array.getArray()
+  rows, cols = array.shape
+
+  y = xrange(cols)
+  n = 10 * (2<<20) // (cols*4)
+  z = np.ndarray(shape=(n, cols), dtype=array.dtype)
+  for row in z:
+    for i in sample(y, 8):
+      row[i] = randrange(0, 1000)
+
+  while 1:
+    txn = transaction.begin()
+    np.random.shuffle(z)
+    rows += n
+    array.resize((rows, cols))
+    array[-n:] = z
+    txn.note(note)
+    txn.commit()

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_dummyZBigArrayProcessing.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>dummyZBigArrayProcessing</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_dummyZBigArrayProcessing</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml → software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/Module_getTotalSize.xml

@@ -54,7 +54,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>DataStreamModule_getTotalSize</string> </value>
+            <value> <string>Module_getTotalSize</string> </value>
         </item>
       </dictionary>
     </pickle>

software/wendelin-scalability/test_scalability_fluentd/bt/template_extension_id_list

-extension.erp5.ScalabilityFluentd
+extension.erp5.ScalabilityFluentd
\ No newline at end of file
+extension.erp5.ScalabilityZBigArray
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list

 portal_ingestion_policies/scalability_test_*
-portal_skins/custom/DataStreamModule_getTotalSize
+portal_skins/custom/ERP5Site_dummyZBigArrayProcessing
 portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
-portal_skins/custom/ERP5Site_simulateFluentdIngestion
+portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
+portal_skins/custom/Module_getTotalSize
\ No newline at end of file

stack/caucase/buildout.cfg

@@ -94,8 +94,8 @@ caucase = 0.1.4
 futures = 3.1.1
 gitdb2 = 2.0.2
 gunicorn = 19.7.1
-slapos.recipe.template = 4.2
-slapos.toolbox = 0.73
+slapos.recipe.template = 4.3
+slapos.toolbox = 0.74
 smmap2 = 2.0.3
 
 # Required by:

stack/cloudooo.cfg

@@ -89,7 +89,7 @@ PasteScript = 2.0.2
 WSGIUtils = 0.7
 python-magic = 0.4.6
 rdiff-backup = 1.0.5+SlapOSPatched001
-slapos.recipe.template = 4.2
+slapos.recipe.template = 4.3
 
 # Required by:
 # PasteScript==2.0

stack/erp5/buildout.cfg

@@ -63,6 +63,7 @@ extends =
   ../../component/findutils/buildout.cfg
   ../../component/userhosts/buildout.cfg
   ../../component/postfix/buildout.cfg
+  ../../component/zbarlight/buildout.cfg
   ../monitor/buildout.cfg
   ../../software/caucase/software.cfg
   ../../software/jupyter/software.cfg
@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
   ${h5py:egg}
   openpyxl
   ${statsmodels:egg}
+  ${zbarlight:egg}
   lock_file
   astor
   APacheDEX
@@ -730,7 +732,7 @@ spyne = 2.12.14
 suds = 0.4
 facebook-sdk = 2.0.0
 threadframe = 0.2
-timerserver = 2.0.2
+timerserver = 2.0.4
 urlnorm = 1.1.4
 uuid = 1.30
 validictory = 1.1.0

stack/erp5/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
+md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
 
 [mariadb-start-clone-from-backup]
 filename = instance-mariadb-start-clone-from-backup.sh.in
@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 244e185ed26831fe0b5abf49c67753d6
+md5sum = da7c36ecb490b67360d2afda94b41bff
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 2465af81147af322056cee9f6c7de14f
+md5sum = 1d6735a803c9d28930bf2ad00706c06b
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -87,11 +87,11 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = fd7e8c507cef1950e6c0347ce2a01021
+md5sum = e08c00d5973916d796bf08aa78dba34a
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = f64c568f1365eb1164f12f48fede9a99
+md5sum = a71ad387eab681b9020e271cba2c7a79
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer.cfg.in

@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
 [{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
 command = apachedex_every_23_hour
 
 [apachedex-parameters]

stack/erp5/instance-erp5.cfg.in

@@ -205,6 +205,7 @@ name = {{ partition_name }}
 {{ root_common.sla(partition_name) }}
 config-name = {{ dumps(custom_name) }}
 config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
+config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
 config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
 config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
 config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
@@ -284,7 +285,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
 config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
 config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
 config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
-'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix  --error-detail --js-embed --quiet')) }}
+'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
 
 [request-frontend-base]
 {% if has_frontend -%}

stack/erp5/instance-mariadb.cfg.in

@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
 var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
-slowquery =  ${monitor-directory:private}/slowquerydex
+slowquery = ${monitor-directory:private}/slowquery_digest
 
 [{{ section('resiliency-exclude-file') }}]
 # Generate rdiff exclude file in case of resiliency

stack/erp5/instance-zope.cfg.in

@@ -206,6 +206,7 @@ environment =
 {% endif %}
 parameters-extra = true
 command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}'
+private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
 
 [{{ section('zcml') }}]
 recipe = slapos.cookbook:copyfilelist

stack/erp5/mysql-querydigest.sh.in

@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
 
 if [ ! -d "$OUTPUT_FOLDER" ]; then
   echo "ERROR: output_folder don't exists"
-  exit 0
+  exit 1
 fi
 
-OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
-
-TODAY=`date +%Y%m%d`
+dashed_today=$(date +%Y-%m-%d)
+today=$(date -d "$dashed_today" +%Y%m%d)
 
-SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
+SLOW_LOG="$SLOW_QUERY_PATH-$today"
+OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
 
 if [ ! -f "$SLOW_LOG" ]; then
   echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
   exit 1
 fi
 
-$PT_QUERY_EXEC $SLOW_LOG >  $OUTPUT_FILE
-echo "ok"
+"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
+echo "Report generated successfully." || \
+echo "Report failed with code $?"

stack/monitor/buildout.cfg

@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
 [monitor-conf]
 <= monitor-template-base
 filename = monitor.conf.in
-md5sum = fc65084dd1c2fe2487b58a003b576f61
+md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
 
 [monitor-instance-info]
 <= monitor-template-base
@@ -131,6 +131,6 @@ depends =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 pycurl = 7.43.0
-slapos.toolbox = 0.73
+slapos.toolbox = 0.74
 pyasn1 = 0.3.7
 

stack/monitor/buildout.hash.cfg

@@ -15,4 +15,4 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = 5027f0b1abcc28ce3817cd70fb667a3b
+md5sum = 03254b14a2ff242f7588a307d8c27f23

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -42,6 +42,7 @@ services = ${directory:services}
 services-conf = ${directory:etc}/monitor.conf.d
 log = ${directory:log}/monitor
 monitor-var = ${directory:var}/monitor
+monitor-log = ${directory:monitor}/private/monitor-log
 
 [ca-directory]
 recipe = slapos.cookbook:mkdirectory
@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
 monitor-promise-folder = ${monitor-directory:promises}
 promises-timeout-file = ${monitor-promise-timeout-file:file}
 pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
+randomsleep = {{ bin_directory }}/randomsleep
 
 public-path-list = 
-private-path-list = 
-    ${directory:log}
-# 
-monitor-url-list = 
-  ${monitor-instance-parameter:monitor-url-list}
-
+private-path-list = ${directory:log}
+monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
 parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
 
 parameter-list = 
@@ -234,6 +232,12 @@ context =
     raw dash_binary {{ dash_executable_location }}
 command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 
+[logrotate-entry-monitor-httpd]
+<= logrotate-entry-base
+name = monitor-apache
+log = $${basedirectory:log}/monitor/monitor-httpd-error.log
+post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
+
 [xnice-bin]
 recipe = collective.recipe.template
 input = inline:#!/bin/sh
@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-globalstate
 frequency = * * * * *
-command = ${monitor-globalstate-wrapper:wrapper-path}
+command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
 
 [monitor-configurator-cron-entry]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
 name = monitor-configurator
 frequency = * * * * *
-command = ${monitor-configurator-wrapper:wrapper-path}
+command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
 
 [monitor-httpd-promise]
 recipe = slapos.cookbook:check_url_available
@@ -407,6 +411,7 @@ depends =
   ${monitor-bootstrap-promise:file}
   ${promise-check-slapgrid:output}
   ${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
+  ${logrotate-entry-monitor-httpd:name}
 
 [monitor-publish]
 monitor-base-url = ${monitor-publish-parameters:monitor-base-url}

stack/monitor/templates/monitor.conf.in

 [monitor]
+{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
 {% for key, value in parameter_dict.items() -%}
-{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
-{{ key }} =
+{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
+{% endfor -%}
+
+monitor-url-list =
+{% if monitor_url_list -%}
+{{ '  ' ~ monitor_url_list.replace("\n", "\n  ") }}
+{% else -%}
 {% for key, value in monitor_base_urls.items() -%}
 {{ '  ' ~ value }}
 {% endfor -%}
-{% else -%}
-{{ key }} = {{ value.strip().replace("\n", "\n  ") }}
 {% endif -%}
-{% endfor -%}

stack/slapos.cfg

@@ -93,7 +93,7 @@ eggs =
 [versions]
 setuptools = 33.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.5.2+slapos011
+zc.buildout = 2.5.2+slapos012
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -121,14 +121,14 @@ netaddr = 0.7.19
 pbr = 2.0.0
 plone.recipe.command = 1.1
 prettytable = 0.7.2
-psutil = 5.2.0
+psutil = 5.4.3
 pyOpenSSL = 17.2.0
 pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.10.0
 slapos.cookbook = 1.0.53
-slapos.core = 1.4.3
+slapos.core = 1.4.4
 slapos.extension.strip = 0.4
 slapos.libnetworkcache = 0.15
 slapos.rebootstrap = 4.1
@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
 paramiko = 2.1.3
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 Flask = 0.12
 
 # Required by:
@@ -160,7 +160,7 @@ ipaddress = 1.0.18
 jsonschema = 2.6.0
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
 netifaces = 0.10.4
 
@@ -173,15 +173,15 @@ packaging = 16.8
 pycparser = 2.17
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 supervisor = 3.3.3
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 uritemplate = 3.0.0
 
 # Required by:
-# slapos.core==1.4.3
+# slapos.core==1.4.4
 zope.interface = 4.3.3
 
 [networkcache]