Merge branch 'master' into ivan

CHANGES.txt

 Changes
 =======
+0.102 (2015-05-22)
+------------------
+ * kvm-recipe: vm of kvm-cluster can get ipv4/hostname of all other vm in the same cluster
+ * simplehttpserver-recipe: simple http server to serve files
+
+0.101 (2015-04-29)
+------------------
+ * kvm recipe: new parameters: external-disk-format, numa and cpu-options.
+ * kvm recipe: allow guest VM to connect to host http service via a local predefined ipv4 address (guestfwd).
+
+0.100 (2015-04-20)
+------------------
+ * re6stnet recipe: re6st-registry log can now be reopened with SIGUSR1
+ * re6stnet recipe: re6st certificate generation is improved.
+
+0.99 (2015-04-10)
+-----------------
+ * re6stnet: new recipe to deploy re6st registry (re6st master) with slapos.
+
+0.98 (2015-04-09)
+-----------------
+ * shellinabox: do not run in debug mode, it is much slower !
+
+0.97 (2015-03-26)
+-----------------
+ * switch softwaretype recipe: the recipe is backward compatible with old slapos node packages.
+ * kvm recipe: Avoid getting wrong storage path when creating kvm external disk
+
+0.96 (2015-03-20)
+-----------------
+ * slap configuration: recipe can read from master network information releated to a tap interface
+ * slap configuration: recipe will setup data folder in DATA directory of computer partition if disk is mounted
+ * switch softwaretype recipe: also generate tap network information when they exist
+ * switch softwaretype recipe: generate configuration for DATA directory when disk is mounted
+
+0.95 (2015-02-14)
+-----------------
+
+ * resiliency stack: allow web takeover to work inside of webrunner/erp5testnode.
+ * resiliency takeover script: create lock file stating that takeover has been done.
+
+0.94 (2015-02-06)
+-----------------
+ * kvm: allow to configure tap and nat interface at the same time with use-nat and use-tap [d3d65916]
+ * kvm: use -netdev to configure network interface as -net is now obsolete [27baa9d4]
 
 0.85 (2013-12-03)
 -----------------

component/apache/buildout.cfg

@@ -30,9 +30,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
 recipe = slapos.recipe.cmmi
-version = 2.4.10
+version = 2.4.12
 url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
-md5sum = 44543dff14a4ebc1e9e2d86780507156
+md5sum = b8dc8367a57a8d548a9b4ce16d264a13
 configure-command = cp -ar ${apr:location}/apr-${apr:version} srclib/apr/; cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util; ./configure
 configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
                     --disable-static

component/automake/buildout.cfg

@@ -8,7 +8,7 @@ parts =
 
 [automake]
 recipe = slapos.recipe.cmmi
-md5sum = 140e084223bf463a24c1a28427c6aec7
-url = http://ftp.gnu.org/gnu/automake/automake-1.12.6.tar.xz
+md5sum = 9a1ddb0e053474d9d1105cfe39b0c48d
+url = http://ftp.gnu.org/gnu/automake/automake-1.15.tar.xz
 environment =
   PATH =${autoconf:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/babeld/buildout.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg 
+
+parts = babeld
+
+[babeld]
+recipe = slapos.recipe.cmmi
+url = https://lab.nexedi.cn/rafael/babeld/repository/archive.tar.gz?ref=v1.6.0-nxd1 
+md5sum = 1f269d01321103873b8d245df19984f0 
+configure-command =
+  echo "No configure.."
+
+make-targets = 
+  install PREFIX=${buildout:parts-directory}/${:_buildout_section_name_} 

component/binutils/buildout.cfg

-[buildout]
-parts =
-  binutils
-
-extends =
-  ../bison/buildout.cfg
-  ../zlib/buildout.cfg
-
-[binutils]
-recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/binutils/binutils-2.21.1.tar.bz2
-md5sum = bde820eac53fa3a8d8696667418557ad
-environment =
-  PATH=${bison:location}/bin:%(PATH)s
-  CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/bison/buildout.cfg

 [buildout]
 extends =
   ../m4/buildout.cfg
+  ../xz-utils/buildout.cfg
 parts =
   bison
 
 [bison]
-patch-options = -p1
-patches =
-  ${:_profile_base_location_}/bison-drop.gets.patch#ac06cbaa298ac686d0b0c04bc03e6ad8
-
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/bison/bison-2.5.tar.bz2
-md5sum = 9dba20116b13fc61a0846b0058fbe004
+url = http://ftp.gnu.org/gnu/bison/bison-3.0.2.tar.xz
+md5sum = 146be9ff9fbd27497f0bf2286a5a2082
 environment =
   M4=${m4:location}/bin/m4
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/coreutils/buildout.cfg

 [buildout]
 extends =
+  ../gmp/buildout.cfg
   ../xz-utils/buildout.cfg
 parts =
   coreutils-output
@@ -12,7 +13,7 @@ configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_} --enable-install-program=tr,basename,uname,cat,cp,ls
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
-  LDFLAGS =-Wl,--as-needed
+  LDFLAGS=-Wl,--as-needed -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib
 
 [coreutils-output]
 # Shared binary location to ease migration

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2
-md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd
+url = http://curl.haxx.se/download/curl-7.42.1.tar.bz2
+md5sum = 296945012ce647b94083ed427c1877a8
 configure-options =
   --disable-static
   --disable-ldap

component/dash/buildout.cfg

@@ -6,8 +6,8 @@ parts = dash-output
 
 [dash]
 recipe = slapos.recipe.cmmi
-url = http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.7.tar.gz
-md5sum = f6cedb10ae7258adb5ab17a10ae80d51
+url = http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.8.tar.gz
+md5sum = 5c152209680dab3c319e8923f6c51378
 configure-options =
   --disable-static
   --disable-fnmatch

component/dbus/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [dbus]
 recipe = slapos.recipe.cmmi
-url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.8.tar.gz
-md5sum = b9f4a18ee3faa1e07c04aa1d83239c43
+url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.16.tar.gz
+md5sum = 020824a38850501e7d6ba8307a7c5ac3
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-options =
   --disable-static
@@ -25,8 +25,8 @@ environment =
 
 [dbus-glib]
 recipe = slapos.recipe.cmmi
-url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.102.tar.gz
-md5sum = f76b8558fd575d0106c3a556eaa49184
+url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.104.tar.gz
+md5sum = 5497d2070709cf796f1878c75a72a039
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/dtach/buildout.cfg

+[buildout]
+
+parts = dtach
+
+[dtach-build]
+recipe = slapos.recipe.cmmi
+version = 0.8
+url = http://freefr.dl.sourceforge.net/project/dtach/dtach/0.8/dtach-${:version}.tar.gz
+md5sum = ec5999f3b6bb67da19754fcb2e5221f3
+keep-compile-dir = true
+make-targets = 
+  
+[dtach]
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+dtach-bin = dtach
+command = 
+  mkdir -p ${:location}/bin
+  cp -rp ${dtach-build:compile-directory}/dtach-${dtach-build:version}/${:dtach-bin} ${:location}/bin
\ No newline at end of file

component/file/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.20.tar.gz
-md5sum = 5d5e13eb3e0e13839da869a31790faf2
+url = ftp://ftp.astron.com/pub/file/file-5.22.tar.gz
+md5sum = 8fb13e5259fe447e02c4a37bc7225add
 configure-options =
   --disable-static
 environment =

component/firefox/buildout.cfg

@@ -26,12 +26,12 @@ depends =
   ${liberation-fonts:location}
   ${ipaex-fonts:location}
 
-version = 33.1
+version = 36.0.4
 
 # MD5SUMs are available at :
 # https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS
-x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 1106b9c56da48a3e5c5ef965fea70078
-x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 4201d94f32b6b778456e5681cdcacd76
+x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 6f50b82a6c020d0f29a04cfcd9fe8282
+x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 119d9369a3b1be282936c35bf036414f
 
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')

component/fluentd/buildout.cfg

+[buildout]
+
+extends =
+  ../../component/git/buildout.cfg
+  ../../component/ruby/buildout.cfg
+  ../../component/icu/buildout.cfg
+
+[fluentd]
+recipe = rubygemsrecipe
+ruby-executable = ${ruby:location}/bin/ruby
+gems =
+  fluentd
+gem-options = --with-icu-lib=${icu:location}/lib/ --with-icu-dir=${icu:location}/
+environment =
+  LDFLAGS = -L${icu:location}/lib -Wl,-rpath=${icu:location}/lib
+  CFLAGS = -I${icu:location}/include

component/fonts/buildout.cfg

@@ -26,12 +26,12 @@ md5sum = 5c781723a0d9ed6188960defba8e91cf
 # http://ipafont.ipa.go.jp/
 [ipaex-fonts]
 <= fonts-base
-url = http://ipafont.ipa.go.jp/ipaexfont/IPAexfont00201.php
+url = http://dl.sourceforge.jp/ipafonts/57330/IPAexfont00201.zip
 md5sum = 7bf84182a04a9632268dbcb03f100d05
 
 [ipa-fonts]
 <= fonts-base
-url = http://ipafont.ipa.go.jp/ipafont/IPAfont00303.php
+url = http://dl.sourceforge.jp/ipafonts/51868/IPAfont00303.zip
 md5sum = 39a828acf27790adbe4944dfb4d94bb1
 
 [ocrb-fonts]
@@ -43,3 +43,36 @@ md5sum = 9f2acd83291a31dbe053912f4115db75
 <= fonts-base
 url = ftp://ftp.free.fr/mirrors/ftp.debian.org/pool/main/f/fonts-android/fonts-android_4.3.orig.tar.xz
 md5sum = 2d41d5342eb5f61591ddeec5b80da74d
+
+# Microsoft's TrueType core fonts
+# non-free so not enabled by default
+[msttcore-fonts]
+location = ${fonts:location}/${:_buildout_section_name_}
+recipe = slapos.recipe.build
+script =
+  from zc.buildout.download import Download
+  d = self.options['location']
+  fonts = []
+  download = lambda x, dl=Download(self.buildout['buildout']): (
+    dl("http://downloads.sf.net/corefonts/%%s32.exe" %% name, md5sum=md5sum)
+    for md5sum, name in (x.split() for x in x.splitlines() if x))
+  extract = lambda x, d=d, p7z="${p7zip:location}/bin/7za": any(
+    subprocess.check_call((p7z, "x", "-ssc-", path, "*.ttf"), cwd=d)
+    for path, is_temp in x)
+  try: fonts += download(self.options['fonts']); os.makedirs(d); extract(fonts)
+  except: shutil.rmtree(d, ignore_errors=True); raise
+  finally: any(os.remove(path) for path, is_temp in fonts if is_temp)
+slapos_promise =
+slapos_update_promise = ${:slapos_promise}
+fonts =
+  cbdc2fdd7d2ed0832795e86a8b9ee19a  andale
+  9637df0e91703179f0723ec095a36cb5  arial
+  c9089ae0c3b3d0d8c4b0a95979bb9ff0  arialb
+  2b30de40bb5e803a0452c7715fc835d1  comic
+  4e412c772294403ab62fb2d247d85c60  courie
+  4d90016026e2da447593b41a8d8fa8bd  georgi
+  7907c7dd6684e9bade91cff82683d9d7  impact
+  ed39c8ef91b9fb80f76f702568291bd5  times
+  0d7ea16cac6261f8513a061fbfcdb2b5  trebuc
+  12d2a75f8156e10607be1eaa8e8ef120  verdan
+  230a1d13a365b22815f502eb24d9149b  webdin

component/freetype/buildout.cfg

@@ -13,8 +13,8 @@ parts =
 
 [freetype]
 recipe = slapos.recipe.cmmi
-url = http://download.savannah.gnu.org/releases/freetype/freetype-2.5.3.tar.bz2
-md5sum = d6b60f06bfc046e43ab2a6cbfd171d65
+url = http://download.savannah.gnu.org/releases/freetype/freetype-2.5.5.tar.bz2
+md5sum = 2a7a314927011d5030903179cf183be0
 pkg_config_depends = ${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-options =

component/gateone/buildout.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../python-kerberos/buildout.cfg
+  ../git/buildout.cfg
+
+develop = 
+  gateone-repository
+
+parts = 
+  gateone-develop
+  gateone
+
+[gateone-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://github.com/liftoff/GateOne.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[gateone-develop]
+recipe = zc.recipe.egg:develop
+setup = ${gateone-repository:location}
+
+[gateone]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${python-kerberos:egg}
+  tornado
+  setuptools
+  pyOpenSSL
+  futures
+  PIL
+  gateone
+

component/gcc/buildout.cfg

@@ -2,10 +2,8 @@
 # Mostly required to support languages different than C or C++
 [buildout]
 extends =
-  ../m4/buildout.cfg
   ../gmp/buildout.cfg
   ../xz-utils/buildout.cfg
-  ../zip/buildout.cfg
 
 parts =
   gcc
@@ -23,8 +21,8 @@ environment =
 
 [mpc]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz
-md5sum = 68fadff3358fb3e7976c7a398a0af4c3
+url = http://ftp.gnu.org/gnu/mpc/mpc-1.0.3.tar.gz
+md5sum = d6a1d5f8ddea3abd2cc3e98f58352d26
 configure-options =
   --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
@@ -34,29 +32,18 @@ environment =
 
 [isl]
 recipe = slapos.recipe.cmmi
-url = ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.12.2.tar.bz2
-md5sum = e039bfcfb6c2ab039b8ee69bf883e824
+url = ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.14.tar.bz2
+md5sum = acd347243fca5609e3df37dba47fd0bb
 configure-options =
   --with-gmp-prefix=${gmp:location}
   --disable-static
 environment =
   LDFLAGS=-Wl,-rpath=${gmp:location}/lib
 
-[cloog]
-recipe = slapos.recipe.cmmi
-url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.18.1.tar.gz
-md5sum = e34fca0540d840e5d0f6427e98c92252
-configure-options =
-  --with-gmp-prefix=${gmp:location}
-  --with-isl-prefix=${isl:location}
-  --disable-static
-environment =
-  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib
-
 [gcc-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2
-md5sum = 4df8ee253b7f3863ad0b86359cd39c43
+url = http://ftp.gnu.org/gnu/gcc/gcc-5.1.0/gcc-5.1.0.tar.bz2
+md5sum = d5525b1127d07d215960e6051c5da35e
 # make install does not work when several core are used
 make-targets = install -j1
 
@@ -70,9 +57,8 @@ configure-options =
   --with-mpc=${mpc:location}
   --enable-languages="c,c++"
   --with-isl=${isl:location}
-  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
 [gcc-minimal]
 <= gcc-common
@@ -98,6 +84,5 @@ configure-options =
   --with-mpc=${mpc:location}
   --enable-languages="c,c++,fortran"
   --with-isl=${isl:location}
-  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib

component/gdal/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+  ../curl/buildout.cfg
+  ../geos/buildout.cfg
+  ../giflib/buildout.cfg
+  ../jasper/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../pcre/buildout.cfg
+  ../proj4/buildout.cfg
+  ../sqlite3/buildout.cfg
+  ../webp/buildout.cfg
+
+parts =
+  gdal-python
+
+[gdal]
+recipe = slapos.recipe.cmmi
+version = 1.11.1
+url = http://download.osgeo.org/gdal/${:version}/gdal-${:version}.tar.xz
+md5sum = 2e126d7c6605691d38f3e71b945f5c73
+configure-options =
+  --with-curl=${curl:location}/bin/curl-config
+  --with-expat=${libexpat:location}
+  --with-geos=${geos:location}/bin/geos-config
+  --with-gif=${giflib:location}
+  --with-jasper=${jasper:location}
+  --with-jpeg=${libjpeg:location}
+  --with-libtiff=${libtiff:location}
+  --with-libz=${zlib:location}
+  --with-png=${libpng:location}
+  --with-static-proj4=${proj4:location}
+  --with-sqlite3=${sqlite3:location}
+  --with-wepb=${webp:location}
+  --with-xml2=${libxml2:location}/bin/xml2-config
+environment =
+  CPPFLAGS=-I${pcre:location}/include
+  LDFLAGS=-L${pcre:location}/lib -Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib -Wl,-rpath=${curl:location}/lib -Wl,-rpath=${geos:location}/lib -Wl,-rpath=${giflib:location}/lib -Wl,-rpath=${jasper:location}/lib -Wl,-rpath=${jbigkit:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libjpeg:location}/lib -Wl,-rpath=${libpng:location}/lib -Wl,-rpath=${libtiff:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${webp:location}/lib -Wl,-rpath=${zlib:location}/lib
+
+[gdal-python]
+recipe = zc.recipe.egg:custom
+egg = GDAL ==${gdal:version}
+setup-eggs = ${numpy:egg}
+rpath = ${:library-dirs}
+include-dirs =
+  ${gdal:location}/include
+library-dirs =
+  ${gdal:location}/lib
+environment = gdal-python-env
+
+[gdal-python-env]
+PATH = ${gdal:location}/bin:%(PATH)s

component/geos/buildout.cfg

+[buildout]
+parts =
+  geos
+
+[geos]
+recipe = slapos.recipe.cmmi
+version = 3.4.2
+url = http://download.osgeo.org/geos/geos-${:version}.tar.bz2
+md5sum = fc5df2d926eb7e67f988a43a92683bae
+configure-options =
+  --disable-dependency-tracking
+  --disable-static

component/gettext/buildout.cfg

@@ -2,16 +2,16 @@
 parts =
   gettext
 extends =
-  ../ncurses/buildout.cfg
   ../libxml2/buildout.cfg
+  ../lunzip/buildout.cfg
+  ../ncurses/buildout.cfg
   ../perl/buildout.cfg
   ../zlib/buildout.cfg
-  ../xz-utils/buildout.cfg
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.3.tar.xz
-md5sum = 092c3f460553ceb4a638ff81d36434c4
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.4.tar.lz
+md5sum = 3e2c6f64efbffd97520ec3b998682c6b
 
 configure-options =
   --disable-static
@@ -29,6 +29,6 @@ configure-options =
   --without-xz
 
 environment =
-  PATH=${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${perl:location}/bin:${lunzip:location}/bin:%(PATH)s
   CPPFLAGS=-I${libxml2:location}/include -I${zlib:location}/include -I${ncurses:location}/include
   LDFLAGS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib

component/ghostscript/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../fontconfig/buildout.cfg
   ../freetype/buildout.cfg
-  ../libjpeg/buildout.cfg
   ../libtiff/buildout.cfg
   ../libxml2/buildout.cfg
   ../pkgconfig/buildout.cfg
@@ -33,8 +32,8 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.15.tar.bz2
-md5sum = 0ee21d975c67a4883ba66bcc332418f5
+url = http://downloads.ghostscript.com/public/ghostscript-9.16.tar.bz2
+md5sum = 21732fd6e39acc283bc623b8842cbfbb
 
 [ghostscript-fonts]
 recipe = hexagonit.recipe.download

component/git/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
   ../pcre/buildout.cfg
+  ../perl/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 parts =
@@ -15,16 +16,16 @@ parts =
 
 [git]
 recipe = slapos.recipe.cmmi
-url = https://www.kernel.org/pub/software/scm/git/git-2.1.0.tar.xz
-md5sum = 47b1f55d9a16be112f7ae2c778a9b30c
+url = https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.xz
+md5sum = 43e01f9d96ba8c11611e0eef0d9f9f28
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}
   --with-zlib=${zlib:location}
   --with-libpcre=${pcre:location}
   --with-expat=${libexpat:location}
+  --with-perl=${perl:location}/bin/perl
 environment =
-  NO_PERL=y
   NO_PYTHON=y
   NO_TCLTK=y
   PATH=${curl:location}/bin:${gettext:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/glib/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [glib]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/glib-2.40.0.tar.xz
-md5sum = 05fb7cb17eacbc718e90366a1eae60d9
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/glib-2.42.1.tar.xz
+md5sum = 89c4119e50e767d3532158605ee9121a
 configure-options =
   --disable-static
   --disable-selinux

component/glibmm/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [glibmm]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/glibmm-2.40.0.tar.xz
-md5sum = f62754f4f5c9030f8ff43c7ed20556c2
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/glibmm-2.42.0.tar.xz
+md5sum = 7c52cc42085d30ac3b73d74c3f2eb22e
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig
 configure-options =
   --disable-documentation

component/glpk/buildout.cfg

+[buildout]
+parts =
+  glpk
+
+[glpk]
+recipe = slapos.recipe.cmmi
+url = http://ftp.gnu.org/gnu/glpk/glpk-4.55.tar.gz
+md5sum = c632a7a631b8aed02e28eafcd99477f7

component/gmp/buildout.cfg

@@ -9,7 +9,7 @@ parts =
 [gmp]
 recipe = slapos.recipe.cmmi
 version = 6.0.0a
-url = ftp://ftp.gmplib.org/pub/gmp-6.0.0/gmp-${:version}.tar.xz
+url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
 md5sum = 1e6da4e434553d2811437aa42c7f7c76
 configure-options =
   --enable-cxx

component/gnutls/buildout.cfg

@@ -4,20 +4,20 @@ extends =
   ../nettle/buildout.cfg
   ../p11-kit/buildout.cfg
   ../pkgconfig/buildout.cfg
-  ../xz-utils/buildout.cfg
+  ../lunzip/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = gnutls
 
 [gpg-error]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.17.tar.bz2
-md5sum = b4f8d8b9ff14aed41f279aa844563539
+url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.19.tar.bz2
+md5sum = c04c16245b92829281f43b5bef7d16da
 
 [gcrypt]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.2.tar.bz2
-md5sum = b54395a93cb1e57619943c082da09d5f
+url = ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.3.tar.bz2
+md5sum = 4262c3aadf837500756c2051a5c4ae5e
 configure-options =
   --with-gpg-error-prefix=${gpg-error:location}
 environment =
@@ -26,8 +26,8 @@ environment =
 
 [gnutls]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.20.tar.xz
-md5sum = 637ef52191bf87a597240a49cc533972
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.21.tar.lz
+md5sum = a653d93a5a26434474c603470cc86666
 configure-options =
   --disable-static
   --disable-libdane
@@ -35,7 +35,7 @@ configure-options =
   --disable-crywrap
   --without-tpm
 environment =
-  PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${lunzip:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${nettle:location}/lib/pkgconfig:${p11-kit:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
   CPPFLAGS=-I${gmp:location}/include -I${zlib:location}/include
   LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/graphviz/buildout.cfg

@@ -6,19 +6,14 @@ parts =
   graphviz
 extends =
   ../bzip2/buildout.cfg
-  ../fontconfig/buildout.cfg
-  ../freetype/buildout.cfg
-  ../libpng/buildout.cfg
-  ../glib/buildout.cfg
   ../gtk-2/buildout.cfg
   ../pkgconfig/buildout.cfg
-  ../xorg/buildout.cfg
   ../zlib/buildout.cfg
 
 [graphviz]
 recipe = slapos.recipe.cmmi
-url = http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.36.0.tar.gz
-md5sum = 1f41664dba0c93109ac8b71216bf2b57
+url = http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz
+md5sum = 5b6a829b2ac94efcd5fa3c223ed6d3ae
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}
 configure-options =
   --with-included-ltdl

component/grep/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 
 [grep]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/grep/grep-2.20.tar.xz
-md5sum = 2cbea44a4f1548aee20b9ff2d3076908
+url = http://ftp.gnu.org/gnu/grep/grep-2.21.tar.xz
+md5sum = 43c48064d6409862b8a850db83c8038a
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include

component/groonga/buildout.cfg

@@ -6,13 +6,15 @@ parts =
   groonga
 
 extends =
+  ../jemalloc/buildout.cfg
+  ../libstemmer/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../zlib/buildout.cfg
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-4.0.7.tar.gz
-md5sum = 5d11546f921377d040f700bb74f1d7e4
+url = http://packages.groonga.org/source/groonga/groonga-5.0.4.tar.gz
+md5sum = 4c83e9b12e9fad96a5b51874ddf66dc4
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
@@ -23,16 +25,20 @@ configure-options =
   --disable-benchmark
   --disable-document
   --disable-groonga-httpd
+  --with-jemalloc
+  --with-libstemmer
+  --with-libstemmer-include=${libstemmer:location}/include
+  --with-libstemmer-lib=${libstemmer:location}/lib
   --without-mecab
+  --without-kytea
 environment =
-  CPPFLAGS =-I${zlib:location}/include
-  LDFLAGS =-Wl,-rpath=${zlib:location}/lib -L${zlib:location}/lib
+  CPPFLAGS=-I${libstemmer:location}/include -I${libstemmer:location}/include -I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${jemalloc:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${libstemmer:location}/lib -L${libstemmer:location}/lib -Wl,-rpath=${zlib:location}/lib -L${zlib:location}/lib
 
 [groonga-normalizer-mysql]
 recipe = slapos.recipe.cmmi
-version = 1.0.6
-url = http://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-${:version}.tar.gz
-md5sum = 12740a835cfdf4b0dac66c3f2f152d84
+url = http://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-1.1.0.tar.gz
+md5sum = 0a0b523547999f90d347982fe9d03c0e
 location = ${groonga:location}
 configure-options =
   --disable-static

component/gtk-2/buildout.cfg

 [buildout]
 extends =
   ../autoconf/buildout.cfg
-  ../libtool/buildout.cfg
   ../automake/buildout.cfg
   ../bzip2/buildout.cfg
   ../fontconfig/buildout.cfg
@@ -12,7 +11,8 @@ extends =
   ../libjpeg/buildout.cfg
   ../libpng/buildout.cfg
   ../libtiff/buildout.cfg
-  ../libxml2/buildout.cfg
+  ../libtool/buildout.cfg
+  ../m4/buildout.cfg
   ../perl/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
@@ -43,6 +43,7 @@ configure-options =
   --enable-fc
   --disable-lto
 environment =
+  M4=${m4:location}/bin/m4
   PATH=${freetype:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${zlib:location}/include
@@ -65,8 +66,8 @@ environment =
 
 [pango]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/pango-1.36.3.tar.xz
-md5sum = d9532826e95bdb374355deebc42441bb
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/pango-1.36.8.tar.xz
+md5sum = 217a9a753006275215fa9fa127760ece
 pkg_config_depends = ${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
 configure-options =
   --disable-static
@@ -79,8 +80,8 @@ environment =
 
 [gdk-pixbuf]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/gdk-pixbuf-2.30.7.tar.xz
-md5sum = 1195d26f14adfe99ff0878d37678a70a
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/gdk-pixbuf-2.31.1.tar.xz
+md5sum = 74cde211f5b7ac1015d1a7c9feee037c
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}
 configure-options =
   --disable-static
@@ -96,8 +97,8 @@ environment =
 
 [atk]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/atk-2.12.0.tar.xz
-md5sum = 930238dec55fdbf8eda9975b44f07b76
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/atk-2.14.0.tar.xz
+md5sum = ecb7ca8469a5650581b1227d78051b8b
 environment =
   PATH=${glib:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
@@ -106,8 +107,8 @@ environment =
 
 [gtk-2]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/gtk+-2.24.23.tar.xz
-md5sum = 0be39fbed4ca125645175cd6e22f2548
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/gtk+-2.24.25.tar.xz
+md5sum = 612350704dd3aacb95355a4981930c6f
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/gtkmm/buildout.cfg

 [buildout]
 extends =
   ../bzip2/buildout.cfg
-  ../fontconfig/buildout.cfg
   ../freetype/buildout.cfg
   ../glibmm/buildout.cfg
   ../gtk-2/buildout.cfg
-  ../libpng/buildout.cfg
-  ../libxml2/buildout.cfg
   ../perl/buildout.cfg
   ../xorg/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
+
 parts =
   gtkmm
 

component/haproxy/buildout.cfg

@@ -11,8 +11,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.8.tar.gz
-md5sum = 7bffa1afa069d90ce03b7cd9aa0557cd
+url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.11.tar.gz
+md5sum = 5500a79d0d2b238d4a1e9749bd0c2cb2
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/inkscape/buildout.cfg

@@ -5,7 +5,6 @@ extends =
   ../bzip2/buildout.cfg
   ../freetype/buildout.cfg
   ../garbage-collector/buildout.cfg
-  ../gcc/buildout.cfg
   ../gettext/buildout.cfg
   ../glibmm/buildout.cfg
   ../gtk-2/buildout.cfg

component/ipython/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+  ../matplotlib/buildout.cfg
+
+parts =
+  ipython
+
+[ipython-env]
+<= numpy-env
+
+[ipython]
+recipe = zc.recipe.egg:custom
+egg = ipython
+environment = ipython-env
+setup-eggs = ${numpy:egg}
+
+[ipython_notebook]
+recipe = zc.recipe.egg:scripts
+eggs = ipython[notebook]
+  ${matplotlib:egg}
+environment = ipython-env
+setup-eggs = ${numpy:egg}
+initialization =
+  # https://github.com/ipython/ipython/issues/5420
+  import os
+  os.environ['PYTHONPATH'] = ':'.join(sys.path)

component/jasper/CVE-2014-8137.patch

+Description: CVE-2014-8137: double-free in in jas_iccattrval_destroy()
+Origin: vendor, https://bugzilla.redhat.com/attachment.cgi?id=967283,
+ https://bugzilla.redhat.com/attachment.cgi?id=967284
+Bug-Debian: https://bugs.debian.org/773463
+Bug-RedHat: https://bugzilla.redhat.com/show_bug.cgi?id=1173157
+Forwarded: no
+Author: Tomas Hoger <thoger@redhat.com>
+Last-Update: 2014-12-20
+
+--- a/src/libjasper/base/jas_icc.c
++++ b/src/libjasper/base/jas_icc.c
+@@ -1010,7 +1010,6 @@ static int jas_icccurv_input(jas_iccattr
+ 	return 0;
+ 
+ error:
+-	jas_icccurv_destroy(attrval);
+ 	return -1;
+ }
+ 
+@@ -1128,7 +1127,6 @@ static int jas_icctxtdesc_input(jas_icca
+ #endif
+ 	return 0;
+ error:
+-	jas_icctxtdesc_destroy(attrval);
+ 	return -1;
+ }
+ 
+@@ -1207,8 +1205,6 @@ static int jas_icctxt_input(jas_iccattrv
+ 		goto error;
+ 	return 0;
+ error:
+-	if (txt->string)
+-		jas_free(txt->string);
+ 	return -1;
+ }
+ 
+@@ -1329,7 +1325,6 @@ static int jas_icclut8_input(jas_iccattr
+ 		goto error;
+ 	return 0;
+ error:
+-	jas_icclut8_destroy(attrval);
+ 	return -1;
+ }
+ 
+@@ -1498,7 +1493,6 @@ static int jas_icclut16_input(jas_iccatt
+ 		goto error;
+ 	return 0;
+ error:
+-	jas_icclut16_destroy(attrval);
+ 	return -1;
+ }
+ 
+--- a/src/libjasper/jp2/jp2_dec.c
++++ b/src/libjasper/jp2/jp2_dec.c
+@@ -291,7 +291,10 @@ jas_image_t *jp2_decode(jas_stream_t *in
+ 	case JP2_COLR_ICC:
+ 		iccprof = jas_iccprof_createfrombuf(dec->colr->data.colr.iccp,
+ 		  dec->colr->data.colr.iccplen);
+-		assert(iccprof);
++		if (!iccprof) {
++			jas_eprintf("error: failed to parse ICC profile\n");
++			goto error;
++		}
+ 		jas_iccprof_gethdr(iccprof, &icchdr);
+ 		jas_eprintf("ICC Profile CS %08x\n", icchdr.colorspc);
+ 		jas_image_setclrspc(dec->image, fromiccpcs(icchdr.colorspc));

component/jasper/CVE-2014-8138.patch

+Description: CVE-2014-8138: heap overflow in jp2_decode()
+Origin: vendor, https://bugzilla.redhat.com/attachment.cgi?id=967280
+Bug-Debian: https://bugs.debian.org/773463
+Bug-RedHat: https://bugzilla.redhat.com/show_bug.cgi?id=1173162
+Forwarded: no
+Author: Tomas Hoger <thoger@redhat.com>
+Last-Update: 2014-12-20
+
+--- a/src/libjasper/jp2/jp2_dec.c
++++ b/src/libjasper/jp2/jp2_dec.c
+@@ -389,6 +389,11 @@ jas_image_t *jp2_decode(jas_stream_t *in
+ 	/* Determine the type of each component. */
+ 	if (dec->cdef) {
+ 		for (i = 0; i < dec->numchans; ++i) {
++			/* Is the channel number reasonable? */
++			if (dec->cdef->data.cdef.ents[i].channo >= dec->numchans) {
++				jas_eprintf("error: invalid channel number in CDEF box\n");
++				goto error;
++			}
+ 			jas_image_setcmpttype(dec->image,
+ 			  dec->chantocmptlut[dec->cdef->data.cdef.ents[i].channo],
+ 			  jp2_getct(jas_image_clrspc(dec->image),

component/jasper/CVE-2014-9029.patch

+Description: CVE-2014-9029: Heap overflows in libjasper
+Origin: vendor
+Forwarded: no
+Author: Tomas Hoger <thoger@redhat.com>
+Reviewed-by: Salvatore Bonaccorso <carnil@debian.org>
+Last-Update: 2014-11-28
+
+--- a/src/libjasper/jpc/jpc_dec.c
++++ b/src/libjasper/jpc/jpc_dec.c
+@@ -1280,7 +1280,7 @@ static int jpc_dec_process_coc(jpc_dec_t
+ 	jpc_coc_t *coc = &ms->parms.coc;
+ 	jpc_dec_tile_t *tile;
+ 
+-	if (JAS_CAST(int, coc->compno) > dec->numcomps) {
++	if (JAS_CAST(int, coc->compno) >= dec->numcomps) {
+ 		jas_eprintf("invalid component number in COC marker segment\n");
+ 		return -1;
+ 	}
+@@ -1306,7 +1306,7 @@ static int jpc_dec_process_rgn(jpc_dec_t
+ 	jpc_rgn_t *rgn = &ms->parms.rgn;
+ 	jpc_dec_tile_t *tile;
+ 
+-	if (JAS_CAST(int, rgn->compno) > dec->numcomps) {
++	if (JAS_CAST(int, rgn->compno) >= dec->numcomps) {
+ 		jas_eprintf("invalid component number in RGN marker segment\n");
+ 		return -1;
+ 	}
+@@ -1355,7 +1355,7 @@ static int jpc_dec_process_qcc(jpc_dec_t
+ 	jpc_qcc_t *qcc = &ms->parms.qcc;
+ 	jpc_dec_tile_t *tile;
+ 
+-	if (JAS_CAST(int, qcc->compno) > dec->numcomps) {
++	if (JAS_CAST(int, qcc->compno) >= dec->numcomps) {
+ 		jas_eprintf("invalid component number in QCC marker segment\n");
+ 		return -1;
+ 	}

component/jasper/buildout.cfg

@@ -13,6 +13,9 @@ patches =
   ${:_profile_base_location_}/misc-fixes.patch#1202be8418907dfe58f819f7b47da24f
   ${:_profile_base_location_}/fix-filename-buffer-overflow.patch#38403f9c82a18547beca16c9c6f4ce7a
   ${:_profile_base_location_}/CVE-2011-4516-and-CVE-2011-4517.patch#a9676718ed016f66a3c76acf764c9e72
+  ${:_profile_base_location_}/CVE-2014-9029.patch#d69195cf17878f024cc0b580045ec314
+  ${:_profile_base_location_}/CVE-2014-8137.patch#bc5103b9a33315538106bf6652383a10
+  ${:_profile_base_location_}/CVE-2014-8138.patch#bfb9604fe84b6e686fea29bd760cf34d
 # jasper configure script is not executable by default
 configure-command =
   /bin/sh ./configure --prefix=${buildout:parts-directory}/${:_buildout_section_name_} --disable-static --enable-shared --disable-opengl

component/libdmtx/buildout.cfg

@@ -6,7 +6,6 @@ extends =
   ../bzip2/buildout.cfg
   ../imagemagick/buildout.cfg
   ../jbigkit/buildout.cfg
-  ../libpng/buildout.cfg
   ../zlib/buildout.cfg
 parts =
   libdmtx

component/libffi/buildout.cfg

@@ -4,9 +4,9 @@ parts =
 
 [libffi]
 recipe = slapos.recipe.cmmi
-version = 3.1
+version = 3.2.1
 url = ftp://sourceware.org/pub/libffi/libffi-${:version}.tar.gz
-md5sum = f5898b29bbfd70502831a212d9249d10
+md5sum = 83b89587607e3eb65c70d361f13bab43
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 patch-options = -p1
 patches =

component/libpng/buildout.cfg

@@ -18,15 +18,15 @@ environment =
 
 [libpng12]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.2.51.tar.xz
-md5sum = 4efba67fa5aa2b785c6fcec2cc3e90c9
+url = http://download.sourceforge.net/libpng/libpng-1.2.52.tar.xz
+md5sum = 49d5c71929bf69a172147c47b9309fbe
 
 [libpng15]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.19.tar.xz
-md5sum = 1e2afb372b9979b2133829ee8136f30f
+url = http://download.sourceforge.net/libpng/libpng-1.5.21.tar.xz
+md5sum = 22eaa2e90352fcd153bed0cdecf542cb
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.14.tar.xz
-md5sum = aefd9fda2178acab908707737174a0cd
+url = http://download.sourceforge.net/libpng/libpng-1.6.16.tar.xz
+md5sum = 23b7286b5d4a86de950fd2ffc5cac742

component/libreoffice-bin/buildout.cfg

@@ -9,14 +9,14 @@ parts =
 [libreoffice-bin]
 recipe = slapos.recipe.build
 # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-version = 4.3.4.1
+version = 4.4.1.2
 url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
 # supported architectures md5sums
-md5sum_x86 = 6fddbe8508fd2e6d5b942ad360013da2
-md5sum_x86-64 = b3b0eab681d2cd2d22e3c065530b36c2
+md5sum_x86 = 23b740e84bc2b8774d50ac97dd553afd
+md5sum_x86-64 = 8de6f3d1d4af3826a3a1695a47444948
 
 # where office code can be found?
-officedir = libreoffice4.3
+officedir = libreoffice4.4
 
 # script to install
 script =

component/librsvg/buildout.cfg

@@ -2,13 +2,11 @@
 parts = librsvg
 extends =
   ../bzip2/buildout.cfg
-  ../pkgconfig/buildout.cfg
-  ../libpng/buildout.cfg
-  ../libxml2/buildout.cfg
-  ../zlib/buildout.cfg
   ../gtk-2/buildout.cfg
-  ../xorg/buildout.cfg
+  ../libxml2/buildout.cfg
+  ../pkgconfig/buildout.cfg
   ../xz-utils/buildout.cfg
+  ../zlib/buildout.cfg
 
 [libcroco]
 recipe = slapos.recipe.cmmi
@@ -23,8 +21,8 @@ environment =
 
 [librsvg]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/librsvg-2.40.2.tar.xz
-md5sum = acdecdb9f08f3bf662a68bf7dafb8b82
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/librsvg-2.40.5.tar.xz
+md5sum = c2b044fccf415902a052d0e978e0ea60
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${zlib:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/libsigc/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 
 [libsigc]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/libsigc++-2.3.1.tar.xz
-md5sum = 06a0569e8ba161d3428e7daa838682f8
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/libsigc++-2.4.0.tar.xz
+md5sum = c6cd2259f5ef973e4c8178d0abbdbfa7
 configure-options =
   --disable-documentation
 environment =

component/libstemmer/buildout.cfg

+# libstemmer
+# http://snowball.tartarus.org/
+
+[libstemmer]
+recipe = slapos.recipe.cmmi
+url = http://snowball.tartarus.org/dist/libstemmer_c.tgz
+md5sum = 6f32f8f81cd6fa0150333ab540af5e27
+patch-options = -p0
+patches =
+  ${:_profile_base_location_}/libstemmer.Makefile.patch#f337bfe457ecf00d3297043a153d402a
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+configure-command = true
+make-options = DESTDIR=${:location}
+

component/libstemmer/libstemmer.Makefile.patch

+--- Makefile.orig	2014-12-29 15:39:57.433970435 +0100
++++ Makefile	2014-12-29 15:54:42.115248445 +0100
+@@ -1,9 +1,19 @@
+ include mkinc.mak
+-CFLAGS=-Iinclude
+-all: libstemmer.o stemwords
+-libstemmer.o: $(snowball_sources:.c=.o)
+-	$(AR) -cru $@ $^
+-stemwords: examples/stemwords.o libstemmer.o
+-	$(CC) -o $@ $^
++CFLAGS+=-Iinclude -fPIC
++DESTDIR=/usr/local
++all: libstemmer.so libstemmer.a stemwords
++libstemmer.so: $(snowball_sources:.c=.o)
++	$(CC) $(CFLAGS) -shared $(LDFLAGS) -Wl,-soname,$@.0d -o $@.0d.0.0 $^
++	ln -s $@.0d.0.0 $@.0d
++	ln -s $@.0d.0.0 $@
++libstemmer.a: $(snowball_sources:.c=.o)
++	$(AR) -crs $@ $^
++stemwords: examples/stemwords.o libstemmer.so
++	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
+ clean:
+-	rm -f stemwords *.o src_c/*.o runtime/*.o libstemmer/*.o
++	rm -f stemwords libstemmer.so* libstemmer.a *.o src_c/*.o runtime/*.o libstemmer/*.o
++install: all
++	mkdir -p $(DESTDIR)/include
++	mkdir -p $(DESTDIR)/lib
++	cp -a include/*.h $(DESTDIR)/include/
++	cp -a libstemmer.* $(DESTDIR)/lib/

component/libtasn1/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [libtasn1]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.2.tar.gz
-md5sum = 414df906df421dee0a5cf7548788d153
+url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.5.tar.gz
+md5sum = 81d272697545e82d39f6bd14854b68f0
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/libtiff/CVE-2012-4564.patch

-Index: tiff-4.0.3/tools/ppm2tiff.c
-===================================================================
---- tiff-4.0.3.orig/tools/ppm2tiff.c	2013-06-23 10:36:50.779629492 -0400
-+++ tiff-4.0.3/tools/ppm2tiff.c	2013-06-23 10:36:50.775629494 -0400
-@@ -89,6 +89,7 @@
- 	int c;
- 	extern int optind;
- 	extern char* optarg;
-+	tmsize_t scanline_size;
- 
- 	if (argc < 2) {
- 	    fprintf(stderr, "%s: Too few arguments\n", argv[0]);
-@@ -237,8 +238,16 @@
- 	}
- 	if (TIFFScanlineSize(out) > linebytes)
- 		buf = (unsigned char *)_TIFFmalloc(linebytes);
--	else
--		buf = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(out));
-+	else {
-+		scanline_size = TIFFScanlineSize(out);
-+		if (scanline_size != 0)
-+			buf = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(out));
-+		else {
-+			fprintf(stderr, "%s: scanline size overflow\n",infile);
-+			(void) TIFFClose(out);
-+			exit(-2);
-+			}
-+		}
- 	if (resolution > 0) {
- 		TIFFSetField(out, TIFFTAG_XRESOLUTION, resolution);
- 		TIFFSetField(out, TIFFTAG_YRESOLUTION, resolution);

component/libtiff/CVE-2013-1960.patch

-Index: tiff-4.0.3/tools/tiff2pdf.c
-===================================================================
---- tiff-4.0.3.orig/tools/tiff2pdf.c	2013-06-23 10:36:50.979629486 -0400
-+++ tiff-4.0.3/tools/tiff2pdf.c	2013-06-23 10:36:50.975629486 -0400
-@@ -3341,33 +3341,56 @@
- 	uint32 height){
- 
- 	tsize_t i=0;
--	uint16 ri =0;
--	uint16 v_samp=1;
--	uint16 h_samp=1;
--	int j=0;
--	
--	i++;
--	
--	while(i<(*striplength)){
-+
-+	while (i < *striplength) {
-+		tsize_t datalen;
-+		uint16 ri;
-+		uint16 v_samp;
-+		uint16 h_samp;
-+		int j;
-+		int ncomp;
-+
-+		/* marker header: one or more FFs */
-+		if (strip[i] != 0xff)
-+			return(0);
-+		i++;
-+		while (i < *striplength && strip[i] == 0xff)
-+			i++;
-+		if (i >= *striplength)
-+			return(0);
-+		/* SOI is the only pre-SOS marker without a length word */
-+		if (strip[i] == 0xd8)
-+			datalen = 0;
-+		else {
-+			if ((*striplength - i) <= 2)
-+				return(0);
-+			datalen = (strip[i+1] << 8) | strip[i+2];
-+			if (datalen < 2 || datalen >= (*striplength - i))
-+				return(0);
-+		}
- 		switch( strip[i] ){
--			case 0xd8:
--				/* SOI - start of image */
-+			case 0xd8:	/* SOI - start of image */
- 				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), 2);
- 				*bufferoffset+=2;
--				i+=2;
- 				break;
--			case 0xc0:
--			case 0xc1:
--			case 0xc3:
--			case 0xc9:
--			case 0xca:
-+			case 0xc0:	/* SOF0 */
-+			case 0xc1:	/* SOF1 */
-+			case 0xc3:	/* SOF3 */
-+			case 0xc9:	/* SOF9 */
-+			case 0xca:	/* SOF10 */
- 				if(no==0){
--					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
--					for(j=0;j<buffer[*bufferoffset+9];j++){
--						if( (buffer[*bufferoffset+11+(2*j)]>>4) > h_samp) 
--							h_samp = (buffer[*bufferoffset+11+(2*j)]>>4);
--						if( (buffer[*bufferoffset+11+(2*j)] & 0x0f) > v_samp) 
--							v_samp = (buffer[*bufferoffset+11+(2*j)] & 0x0f);
-+					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
-+					ncomp = buffer[*bufferoffset+9];
-+					if (ncomp < 1 || ncomp > 4)
-+						return(0);
-+					v_samp=1;
-+					h_samp=1;
-+					for(j=0;j<ncomp;j++){
-+						uint16 samp = buffer[*bufferoffset+11+(3*j)];
-+						if( (samp>>4) > h_samp) 
-+							h_samp = (samp>>4);
-+						if( (samp & 0x0f) > v_samp) 
-+							v_samp = (samp & 0x0f);
- 					}
- 					v_samp*=8;
- 					h_samp*=8;
-@@ -3381,45 +3404,43 @@
-                                           (unsigned char) ((height>>8) & 0xff);
- 					buffer[*bufferoffset+6]=
-                                             (unsigned char) (height & 0xff);
--					*bufferoffset+=strip[i+2]+2;
--					i+=strip[i+2]+2;
--
-+					*bufferoffset+=datalen+2;
-+					/* insert a DRI marker */
- 					buffer[(*bufferoffset)++]=0xff;
- 					buffer[(*bufferoffset)++]=0xdd;
- 					buffer[(*bufferoffset)++]=0x00;
- 					buffer[(*bufferoffset)++]=0x04;
- 					buffer[(*bufferoffset)++]=(ri >> 8) & 0xff;
- 					buffer[(*bufferoffset)++]= ri & 0xff;
--				} else {
--					i+=strip[i+2]+2;
- 				}
- 				break;
--			case 0xc4:
--			case 0xdb:
--				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
--				*bufferoffset+=strip[i+2]+2;
--				i+=strip[i+2]+2;
-+			case 0xc4: /* DHT */
-+			case 0xdb: /* DQT */
-+				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
-+				*bufferoffset+=datalen+2;
- 				break;
--			case 0xda:
-+			case 0xda: /* SOS */
- 				if(no==0){
--					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
--					*bufferoffset+=strip[i+2]+2;
--					i+=strip[i+2]+2;
-+					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
-+					*bufferoffset+=datalen+2;
- 				} else {
- 					buffer[(*bufferoffset)++]=0xff;
- 					buffer[(*bufferoffset)++]=
-                                             (unsigned char)(0xd0 | ((no-1)%8));
--					i+=strip[i+2]+2;
- 				}
--				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), (*striplength)-i-1);
--				*bufferoffset+=(*striplength)-i-1;
-+				i += datalen + 1;
-+				/* copy remainder of strip */
-+				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i]), *striplength - i);
-+				*bufferoffset+= *striplength - i;
- 				return(1);
- 			default:
--				i+=strip[i+2]+2;
-+				/* ignore any other marker */
-+				break;
- 		}
-+		i += datalen + 1;
- 	}
--	
- 
-+	/* failed to find SOS marker */
- 	return(0);
- }
- #endif

component/libtiff/CVE-2013-4231.patch

-Description: Buffer overflow in gif2tiff
-Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2450
-Bug-Debian: http://bugs.debian.org/719303
-
-Index: tiff-4.0.3/tools/gif2tiff.c
-===================================================================
---- tiff-4.0.3.orig/tools/gif2tiff.c	2013-08-22 11:46:11.960846910 -0400
-+++ tiff-4.0.3/tools/gif2tiff.c	2013-08-22 11:46:11.956846910 -0400
-@@ -333,6 +333,8 @@
-     int status = 1;
- 
-     datasize = getc(infile);
-+    if (datasize > 12)
-+	return 0;
-     clear = 1 << datasize;
-     eoi = clear + 1;
-     avail = clear + 2;

component/libtiff/CVE-2013-4232.patch

-Description: use after free in tiff2pdf
-Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2449
-Bug-Debian: http://bugs.debian.org/719303
-
-Index: tiff-4.0.3/tools/tiff2pdf.c
-===================================================================
---- tiff-4.0.3.orig/tools/tiff2pdf.c	2013-08-22 11:46:37.292847242 -0400
-+++ tiff-4.0.3/tools/tiff2pdf.c	2013-08-22 11:46:37.292847242 -0400
-@@ -2461,7 +2461,8 @@
- 					(unsigned long) t2p->tiff_datasize, 
- 					TIFFFileName(input));
- 				t2p->t2p_error = T2P_ERR_ERROR;
--			  _TIFFfree(buffer);
-+				_TIFFfree(buffer);
-+				return(0);
- 			} else {
- 				buffer=samplebuffer;
- 				t2p->tiff_datasize *= t2p->tiff_samplesperpixel;

component/libtiff/CVE-2013-4244.patch

-Description: OOB write in gif2tiff
-Bug-Redhat: https://bugzilla.redhat.com/show_bug.cgi?id=996468
-
-Index: tiff-4.0.3/tools/gif2tiff.c
-===================================================================
---- tiff-4.0.3.orig/tools/gif2tiff.c	2013-08-24 11:17:13.546447901 -0400
-+++ tiff-4.0.3/tools/gif2tiff.c	2013-08-24 11:17:13.546447901 -0400
-@@ -400,6 +400,10 @@
-     }
- 
-     if (oldcode == -1) {
-+        if (code >= clear) {
-+            fprintf(stderr, "bad input: code=%d is larger than clear=%d\n",code, clear);
-+            return 0;
-+        }
- 	*(*fill)++ = suffix[code];
- 	firstchar = oldcode = code;
- 	return 1;

component/libtiff/buildout.cfg

@@ -14,12 +14,7 @@ url = http://www.imagemagick.org/download/delegates/tiff-${:version}.tar.gz
 md5sum = 051c1068e6a0627f461948c365290410
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/CVE-2012-4564.patch#45667ee618dbe78acce1129706556124
-  ${:_profile_base_location_}/CVE-2013-1960.patch#21a3d119cd3eeadd35ccc355fbd748cf
-  ${:_profile_base_location_}/CVE-2013-1961.patch#bb219740a815b9b47698b83d0ae9f82a
-  ${:_profile_base_location_}/CVE-2013-4231.patch#f6ff024c8df861a6dbb5a0ecd8a0f853
-  ${:_profile_base_location_}/CVE-2013-4232.patch#b439184b3a5f434a3e3235f611b54a89
-  ${:_profile_base_location_}/CVE-2013-4244.patch#2acff059c6156953aadb436b475e5acb
+  ${:_profile_base_location_}/tiff_4.0.3-12.3.debian.patch#c246dc095f57f869b35cf40a32aa8366
 configure-options =
   --disable-static
   --without-x

component/libtiff/CVE-2013-1961.patch → component/libtiff/tiff_4.0.3-12.3.debian.patch

+Description: call glFlush() in tiffgt
+Author: Micksa (micksa-launchpad)
+Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2401
+Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/tiff/+bug/797166
+
+Index: tiff-4.0.3/tools/tiffgt.c
+===================================================================
+--- tiff-4.0.3.orig/tools/tiffgt.c	2013-06-23 10:36:50.575629499 -0400
++++ tiff-4.0.3/tools/tiffgt.c	2013-06-23 10:36:50.571629497 -0400
+@@ -287,6 +287,7 @@
+ raster_draw(void)
+ {
+   glDrawPixels(img.width, img.height, GL_RGBA, GL_UNSIGNED_BYTE, (const GLvoid *) raster);
++  glFlush();
+ }
+ 
+ static void
+Index: tiff-4.0.3/tools/ppm2tiff.c
+===================================================================
+--- tiff-4.0.3.orig/tools/ppm2tiff.c	2013-06-23 10:36:50.779629492 -0400
++++ tiff-4.0.3/tools/ppm2tiff.c	2013-06-23 10:36:50.775629494 -0400
+@@ -89,6 +89,7 @@
+ 	int c;
+ 	extern int optind;
+ 	extern char* optarg;
++	tmsize_t scanline_size;
+ 
+ 	if (argc < 2) {
+ 	    fprintf(stderr, "%s: Too few arguments\n", argv[0]);
+@@ -237,8 +238,16 @@
+ 	}
+ 	if (TIFFScanlineSize(out) > linebytes)
+ 		buf = (unsigned char *)_TIFFmalloc(linebytes);
+-	else
+-		buf = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(out));
++	else {
++		scanline_size = TIFFScanlineSize(out);
++		if (scanline_size != 0)
++			buf = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(out));
++		else {
++			fprintf(stderr, "%s: scanline size overflow\n",infile);
++			(void) TIFFClose(out);
++			exit(-2);
++			}
++		}
+ 	if (resolution > 0) {
+ 		TIFFSetField(out, TIFFTAG_XRESOLUTION, resolution);
+ 		TIFFSetField(out, TIFFTAG_YRESOLUTION, resolution);
+Index: tiff-4.0.3/tools/tiff2pdf.c
+===================================================================
+--- tiff-4.0.3.orig/tools/tiff2pdf.c	2013-06-23 10:36:50.979629486 -0400
++++ tiff-4.0.3/tools/tiff2pdf.c	2013-06-23 10:36:50.975629486 -0400
+@@ -3341,33 +3341,56 @@
+ 	uint32 height){
+ 
+ 	tsize_t i=0;
+-	uint16 ri =0;
+-	uint16 v_samp=1;
+-	uint16 h_samp=1;
+-	int j=0;
+-	
+-	i++;
+-	
+-	while(i<(*striplength)){
++
++	while (i < *striplength) {
++		tsize_t datalen;
++		uint16 ri;
++		uint16 v_samp;
++		uint16 h_samp;
++		int j;
++		int ncomp;
++
++		/* marker header: one or more FFs */
++		if (strip[i] != 0xff)
++			return(0);
++		i++;
++		while (i < *striplength && strip[i] == 0xff)
++			i++;
++		if (i >= *striplength)
++			return(0);
++		/* SOI is the only pre-SOS marker without a length word */
++		if (strip[i] == 0xd8)
++			datalen = 0;
++		else {
++			if ((*striplength - i) <= 2)
++				return(0);
++			datalen = (strip[i+1] << 8) | strip[i+2];
++			if (datalen < 2 || datalen >= (*striplength - i))
++				return(0);
++		}
+ 		switch( strip[i] ){
+-			case 0xd8:
+-				/* SOI - start of image */
++			case 0xd8:	/* SOI - start of image */
+ 				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), 2);
+ 				*bufferoffset+=2;
+-				i+=2;
+ 				break;
+-			case 0xc0:
+-			case 0xc1:
+-			case 0xc3:
+-			case 0xc9:
+-			case 0xca:
++			case 0xc0:	/* SOF0 */
++			case 0xc1:	/* SOF1 */
++			case 0xc3:	/* SOF3 */
++			case 0xc9:	/* SOF9 */
++			case 0xca:	/* SOF10 */
+ 				if(no==0){
+-					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
+-					for(j=0;j<buffer[*bufferoffset+9];j++){
+-						if( (buffer[*bufferoffset+11+(2*j)]>>4) > h_samp) 
+-							h_samp = (buffer[*bufferoffset+11+(2*j)]>>4);
+-						if( (buffer[*bufferoffset+11+(2*j)] & 0x0f) > v_samp) 
+-							v_samp = (buffer[*bufferoffset+11+(2*j)] & 0x0f);
++					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
++					ncomp = buffer[*bufferoffset+9];
++					if (ncomp < 1 || ncomp > 4)
++						return(0);
++					v_samp=1;
++					h_samp=1;
++					for(j=0;j<ncomp;j++){
++						uint16 samp = buffer[*bufferoffset+11+(3*j)];
++						if( (samp>>4) > h_samp) 
++							h_samp = (samp>>4);
++						if( (samp & 0x0f) > v_samp) 
++							v_samp = (samp & 0x0f);
+ 					}
+ 					v_samp*=8;
+ 					h_samp*=8;
+@@ -3381,45 +3404,43 @@
+                                           (unsigned char) ((height>>8) & 0xff);
+ 					buffer[*bufferoffset+6]=
+                                             (unsigned char) (height & 0xff);
+-					*bufferoffset+=strip[i+2]+2;
+-					i+=strip[i+2]+2;
+-
++					*bufferoffset+=datalen+2;
++					/* insert a DRI marker */
+ 					buffer[(*bufferoffset)++]=0xff;
+ 					buffer[(*bufferoffset)++]=0xdd;
+ 					buffer[(*bufferoffset)++]=0x00;
+ 					buffer[(*bufferoffset)++]=0x04;
+ 					buffer[(*bufferoffset)++]=(ri >> 8) & 0xff;
+ 					buffer[(*bufferoffset)++]= ri & 0xff;
+-				} else {
+-					i+=strip[i+2]+2;
+ 				}
+ 				break;
+-			case 0xc4:
+-			case 0xdb:
+-				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
+-				*bufferoffset+=strip[i+2]+2;
+-				i+=strip[i+2]+2;
++			case 0xc4: /* DHT */
++			case 0xdb: /* DQT */
++				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
++				*bufferoffset+=datalen+2;
+ 				break;
+-			case 0xda:
++			case 0xda: /* SOS */
+ 				if(no==0){
+-					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), strip[i+2]+2);
+-					*bufferoffset+=strip[i+2]+2;
+-					i+=strip[i+2]+2;
++					_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), datalen+2);
++					*bufferoffset+=datalen+2;
+ 				} else {
+ 					buffer[(*bufferoffset)++]=0xff;
+ 					buffer[(*bufferoffset)++]=
+                                             (unsigned char)(0xd0 | ((no-1)%8));
+-					i+=strip[i+2]+2;
+ 				}
+-				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i-1]), (*striplength)-i-1);
+-				*bufferoffset+=(*striplength)-i-1;
++				i += datalen + 1;
++				/* copy remainder of strip */
++				_TIFFmemcpy(&(buffer[*bufferoffset]), &(strip[i]), *striplength - i);
++				*bufferoffset+= *striplength - i;
+ 				return(1);
+ 			default:
+-				i+=strip[i+2]+2;
++				/* ignore any other marker */
++				break;
+ 		}
++		i += datalen + 1;
+ 	}
+-	
+ 
++	/* failed to find SOS marker */
+ 	return(0);
+ }
+ #endif
 Index: tiff-4.0.3/contrib/dbs/xtiff/xtiff.c
 ===================================================================
 --- tiff-4.0.3.orig/contrib/dbs/xtiff/xtiff.c	2013-06-23 10:36:51.163629483 -0400
@@ -766,3 +960,941 @@ Index: tiff-4.0.3/tools/tiff2bw.c
  	TIFFSetField(out, TIFFTAG_IMAGEDESCRIPTION, thing);
  	TIFFSetField(out, TIFFTAG_SOFTWARE, "tiff2bw");
  	outbuf = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(out));
+Description: Buffer overflow in gif2tiff
+Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2450
+Bug-Debian: http://bugs.debian.org/719303
+
+Index: tiff-4.0.3/tools/gif2tiff.c
+===================================================================
+--- tiff-4.0.3.orig/tools/gif2tiff.c	2013-08-22 11:46:11.960846910 -0400
++++ tiff-4.0.3/tools/gif2tiff.c	2013-08-22 11:46:11.956846910 -0400
+@@ -333,6 +333,8 @@
+     int status = 1;
+ 
+     datasize = getc(infile);
++    if (datasize > 12)
++	return 0;
+     clear = 1 << datasize;
+     eoi = clear + 1;
+     avail = clear + 2;
+Description: use after free in tiff2pdf
+Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2449
+Bug-Debian: http://bugs.debian.org/719303
+
+Index: tiff-4.0.3/tools/tiff2pdf.c
+===================================================================
+--- tiff-4.0.3.orig/tools/tiff2pdf.c	2013-08-22 11:46:37.292847242 -0400
++++ tiff-4.0.3/tools/tiff2pdf.c	2013-08-22 11:46:37.292847242 -0400
+@@ -2461,7 +2461,8 @@
+ 					(unsigned long) t2p->tiff_datasize, 
+ 					TIFFFileName(input));
+ 				t2p->t2p_error = T2P_ERR_ERROR;
+-			  _TIFFfree(buffer);
++				_TIFFfree(buffer);
++				return(0);
+ 			} else {
+ 				buffer=samplebuffer;
+ 				t2p->tiff_datasize *= t2p->tiff_samplesperpixel;
+Description: OOB write in gif2tiff
+Bug-Redhat: https://bugzilla.redhat.com/show_bug.cgi?id=996468
+
+Index: tiff-4.0.3/tools/gif2tiff.c
+===================================================================
+--- tiff-4.0.3.orig/tools/gif2tiff.c	2013-08-24 11:17:13.546447901 -0400
++++ tiff-4.0.3/tools/gif2tiff.c	2013-08-24 11:17:13.546447901 -0400
+@@ -400,6 +400,10 @@
+     }
+ 
+     if (oldcode == -1) {
++        if (code >= clear) {
++            fprintf(stderr, "bad input: code=%d is larger than clear=%d\n",code, clear);
++            return 0;
++        }
+ 	*(*fill)++ = suffix[code];
+ 	firstchar = oldcode = code;
+ 	return 1;
+Index: tiff/tools/gif2tiff.c
+===================================================================
+--- tiff.orig/tools/gif2tiff.c
++++ tiff/tools/gif2tiff.c
+@@ -280,6 +280,10 @@ readgifimage(char* mode)
+         fprintf(stderr, "no colormap present for image\n");
+         return (0);
+     }
++    if (width == 0 || height == 0) {
++        fprintf(stderr, "Invalid value of width or height\n");
++        return(0);
++    }
+     if ((raster = (unsigned char*) _TIFFmalloc(width*height+EXTRAFUDGE)) == NULL) {
+         fprintf(stderr, "not enough memory for image\n");
+         return (0);
+@@ -404,6 +408,10 @@ process(register int code, unsigned char
+             fprintf(stderr, "bad input: code=%d is larger than clear=%d\n",code, clear);
+             return 0;
+         }
++        if (*fill >= raster + width*height) {
++            fprintf(stderr, "raster full before eoi code\n");
++            return 0;
++        }
+ 	*(*fill)++ = suffix[code];
+ 	firstchar = oldcode = code;
+ 	return 1;
+@@ -434,6 +442,10 @@ process(register int code, unsigned char
+     }
+     oldcode = incode;
+     do {
++        if (*fill >= raster + width*height) {
++            fprintf(stderr, "raster full before eoi code\n");
++            return 0;
++        }
+ 	*(*fill)++ = *--stackp;
+     } while (stackp > stack);
+     return 1;
+Description: fix for Debian bug #741451
+ tiffcp crashes when converting JPEG-encoded TIFF to a different
+ encoding (like none or lzw). For example this will probably fail:
+ .
+        tiffcp -c none jpeg_encoded_file.tif output.tif
+ .
+ The reason is that when the input file contains JPEG data,
+ the tiffcp code forces conversion to RGB space. However,
+ the output normally inherits YCbCr subsampling parameters
+ from the input, which leads to a smaller working buffer
+ than necessary. The buffer is subsequently overrun inside
+ cpStripToTile() (called from writeBufferToContigTiles).
+ Note that the resulting TIFF file would be scrambled even
+ if tiffcp wouldn't crash, since the output file would contain
+ RGB data intepreted as subsampled YCbCr values.
+ .
+ This patch fixes the problem by forcing RGB space on the output
+ TIF if the input is JPEG-encoded and output is *not* JPEG-encoded.
+Origin: upstream, http://bugzilla.maptools.org/show_bug.cgi?id=2480
+Author: Tomasz Buchert <tomasz.buchert@inria.fr>
+Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2480
+Bug-Debian: http://bugs.debian.org/741451
+
+--- a/tools/tiffcp.c
++++ b/tools/tiffcp.c
+@@ -629,6 +629,12 @@
+ 		TIFFSetField(out, TIFFTAG_PHOTOMETRIC,
+ 		    samplesperpixel == 1 ?
+ 		    PHOTOMETRIC_LOGL : PHOTOMETRIC_LOGLUV);
++	else if (input_compression == COMPRESSION_JPEG &&
++		samplesperpixel == 3) {
++		/* RGB conversion was forced above
++		   hence the output will be of the same type */
++		TIFFSetField(out, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB);
++	}
+ 	else
+ 		CopyTag(TIFFTAG_PHOTOMETRIC, 1, TIFF_SHORT);
+ 	if (fillorder != 0)
+Description: CVE-2014-9330
+ Integer overflow in bmp2tiff
+Origin: upstream, http://bugzilla.maptools.org/show_bug.cgi?id=2494
+Bug: http://bugzilla.maptools.org/show_bug.cgi?id=2494
+Bug-Debian: http://bugs.debian.org/773987
+
+Index: tiff/tools/bmp2tiff.c
+===================================================================
+--- tiff.orig/tools/bmp2tiff.c
++++ tiff/tools/bmp2tiff.c
+@@ -1,4 +1,4 @@
+-/* $Id: bmp2tiff.c,v 1.23 2010-03-10 18:56:49 bfriesen Exp $
++/* $Id: bmp2tiff.c,v 1.24 2014-12-21 15:15:32 erouault Exp $
+  *
+  * Project:  libtiff tools
+  * Purpose:  Convert Windows BMP files in TIFF.
+@@ -403,6 +403,13 @@ main(int argc, char* argv[])
+ 
+ 		width = info_hdr.iWidth;
+ 		length = (info_hdr.iHeight > 0) ? info_hdr.iHeight : -info_hdr.iHeight;
++        if( width <= 0 || length <= 0 )
++        {
++            TIFFError(infilename,
++                  "Invalid dimensions of BMP file" );
++            close(fd);
++            return -1;
++        }
+ 
+ 		switch (info_hdr.iBitCount)
+ 		{
+@@ -593,6 +600,14 @@ main(int argc, char* argv[])
+ 
+ 			compr_size = file_hdr.iSize - file_hdr.iOffBits;
+ 			uncompr_size = width * length;
++            /* Detect int overflow */
++            if( uncompr_size / width != length )
++            {
++                TIFFError(infilename,
++                    "Invalid dimensions of BMP file" );
++                close(fd);
++                return -1;
++            }
+ 			comprbuf = (unsigned char *) _TIFFmalloc( compr_size );
+ 			if (!comprbuf) {
+ 				TIFFError(infilename,
+From 0782c759084daaf9e4de7ee6be7543081823455e Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 20:58:29 +0000
+Subject: [PATCH] * tools/tiff2bw.c: when Photometric=RGB, the utility only
+ works if SamplesPerPixel = 3. Enforce that
+ http://bugzilla.maptools.org/show_bug.cgi?id=2485 (CVE-2014-8127)
+
+---
+ ChangeLog       | 6 ++++++
+ tools/tiff2bw.c | 5 +++++
+ 2 files changed, 11 insertions(+)
+
+diff --git a/tools/tiff2bw.c b/tools/tiff2bw.c
+index 22467cd..94b8e31 100644
+--- a/tools/tiff2bw.c
++++ b/tools/tiff2bw.c
+@@ -171,6 +171,11 @@ main(int argc, char* argv[])
+ 		    argv[optind], samplesperpixel);
+ 		return (-1);
+ 	}
++	if( photometric == PHOTOMETRIC_RGB && samplesperpixel != 3) {
++		fprintf(stderr, "%s: Bad samples/pixel %u for PHOTOMETRIC_RGB.\n",
++		    argv[optind], samplesperpixel);
++		return (-1);
++	}
+ 	TIFFGetField(in, TIFFTAG_BITSPERSAMPLE, &bitspersample);
+ 	if (bitspersample != 8) {
+ 		fprintf(stderr,
+From 3996fa0f84f4a8b7e65fe4b8f0681711022034ea Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 20:04:31 +0000
+Subject: [PATCH] * tools/pal2rgb.c, tools/thumbnail.c: fix crash by disabling
+ TIFFTAG_INKNAMES copying. The right fix would be to properly copy it, but not
+ worth the burden for those esoteric utilities.
+ http://bugzilla.maptools.org/show_bug.cgi?id=2484 (CVE-2014-8127)
+
+---
+ ChangeLog         | 7 +++++++
+ tools/pal2rgb.c   | 2 +-
+ tools/thumbnail.c | 2 +-
+ 3 files changed, 9 insertions(+), 2 deletions(-)
+
+diff --git a/tools/pal2rgb.c b/tools/pal2rgb.c
+index bfe7899..3fc3de3 100644
+--- a/tools/pal2rgb.c
++++ b/tools/pal2rgb.c
+@@ -372,7 +372,7 @@ static struct cpTag {
+     { TIFFTAG_CLEANFAXDATA,		1, TIFF_SHORT },
+     { TIFFTAG_CONSECUTIVEBADFAXLINES,	1, TIFF_LONG },
+     { TIFFTAG_INKSET,			1, TIFF_SHORT },
+-    { TIFFTAG_INKNAMES,			1, TIFF_ASCII },
++    /*{ TIFFTAG_INKNAMES,			1, TIFF_ASCII },*/ /* Needs much more complicated logic. See tiffcp */
+     { TIFFTAG_DOTRANGE,			2, TIFF_SHORT },
+     { TIFFTAG_TARGETPRINTER,		1, TIFF_ASCII },
+     { TIFFTAG_SAMPLEFORMAT,		1, TIFF_SHORT },
+diff --git a/tools/thumbnail.c b/tools/thumbnail.c
+index c50bbff..73f9c34 100644
+--- a/tools/thumbnail.c
++++ b/tools/thumbnail.c
+@@ -257,7 +257,7 @@ static struct cpTag {
+     { TIFFTAG_CLEANFAXDATA,		1, TIFF_SHORT },
+     { TIFFTAG_CONSECUTIVEBADFAXLINES,	1, TIFF_LONG },
+     { TIFFTAG_INKSET,			1, TIFF_SHORT },
+-    { TIFFTAG_INKNAMES,			1, TIFF_ASCII },
++    /*{ TIFFTAG_INKNAMES,			1, TIFF_ASCII },*/ /* Needs much more complicated logic. See tiffcp */
+     { TIFFTAG_DOTRANGE,			2, TIFF_SHORT },
+     { TIFFTAG_TARGETPRINTER,		1, TIFF_ASCII },
+     { TIFFTAG_SAMPLEFORMAT,		1, TIFF_SHORT },
+From 1f7359b00663804d96c3a102bcb6ead9812c1509 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Tue, 23 Dec 2014 10:15:35 +0000
+Subject: [PATCH] * libtiff/tif_read.c: fix several invalid comparisons of a
+ uint64 value with <= 0 by casting it to int64 first. This solves crashing bug
+ on corrupted images generated by afl.
+
+---
+ ChangeLog          | 6 ++++++
+ libtiff/tif_read.c | 6 +++---
+ 2 files changed, 9 insertions(+), 3 deletions(-)
+
+diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
+index 2ba822a..dfc5b07 100644
+--- a/libtiff/tif_read.c
++++ b/libtiff/tif_read.c
+@@ -458,7 +458,7 @@ TIFFReadRawStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+ 		return ((tmsize_t)(-1));
+ 	}
+ 	bytecount = td->td_stripbytecount[strip];
+-	if (bytecount <= 0) {
++	if ((int64)bytecount <= 0) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 		TIFFErrorExt(tif->tif_clientdata, module,
+ 			     "%I64u: Invalid strip byte count, strip %lu",
+@@ -498,7 +498,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
+ 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
+ 	{
+ 		uint64 bytecount = td->td_stripbytecount[strip];
+-		if (bytecount <= 0) {
++		if ((int64)bytecount <= 0) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			TIFFErrorExt(tif->tif_clientdata, module,
+ 				"Invalid strip byte count %I64u, strip %lu",
+@@ -801,7 +801,7 @@ TIFFFillTile(TIFF* tif, uint32 tile)
+ 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
+ 	{
+ 		uint64 bytecount = td->td_stripbytecount[tile];
+-		if (bytecount <= 0) {
++		if ((int64)bytecount <= 0) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			TIFFErrorExt(tif->tif_clientdata, module,
+ 				"%I64u: Invalid tile byte count, tile %lu",
+From 662f74445b2fea2eeb759c6524661118aef567ca Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 15:15:31 +0000
+Subject: [PATCH] Fix various crasher bugs on fuzzed images. *
+ libtiff/tif_dir.c: TIFFSetField(): refuse to set negative values for
+ TIFFTAG_XRESOLUTION and TIFFTAG_YRESOLUTION that cause asserts when writing
+ the directory * libtiff/tif_dirread.c: TIFFReadDirectory(): refuse to read
+ ColorMap or TransferFunction if BitsPerSample has not yet been read,
+ otherwise reading it later will cause user code to crash if BitsPerSample > 1
+ * libtiff/tif_getimage.c: TIFFRGBAImageOK(): return FALSE if LOGLUV with
+ SamplesPerPixel != 3, or if CIELAB with SamplesPerPixel != 3 or BitsPerSample
+ != 8 * libtiff/tif_next.c: in the "run mode", use tilewidth for tiled images
+ instead of imagewidth to avoid crash * tools/bmp2tiff.c: fix crash due to int
+ overflow related to input BMP dimensions * tools/tiff2pdf.c: fix crash due to
+ invalid tile count (should likely be checked by libtiff too). Detect invalid
+ settings of BitsPerSample/SamplesPerPixel for CIELAB / ITULAB *
+ tools/tiffcrop.c: fix crash due to invalid TileWidth/TileHeight *
+ tools/tiffdump.c: fix crash due to overflow of entry count.
+
+---
+ ChangeLog              | 19 +++++++++++++++++++
+ libtiff/tif_dir.c      | 21 +++++++++++++++++++--
+ libtiff/tif_dirread.c  | 17 +++++++++++++++++
+ libtiff/tif_getimage.c | 15 +++++++++++++++
+ libtiff/tif_next.c     |  2 ++
+ tools/bmp2tiff.c       | 15 +++++++++++++++
+ tools/tiff2pdf.c       | 41 +++++++++++++++++++++++++++++++++++++++++
+ tools/tiffcrop.c       |  7 ++++---
+ tools/tiffdump.c       |  9 ++++++---
+ 9 files changed, 138 insertions(+), 8 deletions(-)
+
+diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
+index 98cf66d..ab43a28 100644
+--- a/libtiff/tif_dir.c
++++ b/libtiff/tif_dir.c
+@@ -160,6 +160,7 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 	TIFFDirectory* td = &tif->tif_dir;
+ 	int status = 1;
+ 	uint32 v32, i, v;
++    double dblval;
+ 	char* s;
+ 	const TIFFField *fip = TIFFFindField(tif, tag, TIFF_ANY);
+ 	uint32 standard_tag = tag;
+@@ -284,10 +285,16 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 			setDoubleArrayOneValue(&td->td_smaxsamplevalue, va_arg(ap, double), td->td_samplesperpixel);
+ 		break;
+ 	case TIFFTAG_XRESOLUTION:
+-		td->td_xresolution = (float) va_arg(ap, double);
++        dblval = va_arg(ap, double);
++        if( dblval < 0 )
++            goto badvaluedouble;
++		td->td_xresolution = (float) dblval;
+ 		break;
+ 	case TIFFTAG_YRESOLUTION:
+-		td->td_yresolution = (float) va_arg(ap, double);
++        dblval = va_arg(ap, double);
++        if( dblval < 0 )
++            goto badvaluedouble;
++		td->td_yresolution = (float) dblval;
+ 		break;
+ 	case TIFFTAG_PLANARCONFIG:
+ 		v = (uint16) va_arg(ap, uint16_vap);
+@@ -694,6 +701,16 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 		va_end(ap);
+         }
+ 	return (0);
++badvaluedouble:
++        {
++        const TIFFField* fip=TIFFFieldWithTag(tif,tag);
++        TIFFErrorExt(tif->tif_clientdata, module,
++             "%s: Bad value %f for \"%s\" tag",
++             tif->tif_name, dblval,
++             fip ? fip->field_name : "Unknown");
++        va_end(ap);
++        }
++    return (0);
+ }
+ 
+ /*
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 391c823..f66c9a7 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -3430,6 +3430,8 @@ TIFFReadDirectory(TIFF* tif)
+ 	const TIFFField* fip;
+ 	uint32 fii=FAILED_FII;
+         toff_t nextdiroff;
++    int bitspersample_read = FALSE;
++
+ 	tif->tif_diroff=tif->tif_nextdiroff;
+ 	if (!TIFFCheckDirOffset(tif,tif->tif_nextdiroff))
+ 		return 0;           /* last offset or bad offset (IFD looping) */
+@@ -3706,6 +3708,8 @@ TIFFReadDirectory(TIFF* tif)
+ 					}
+ 					if (!TIFFSetField(tif,dp->tdir_tag,value))
+ 						goto bad;
++                    if( dp->tdir_tag == TIFFTAG_BITSPERSAMPLE )
++                        bitspersample_read = TRUE;
+ 				}
+ 				break;
+ 			case TIFFTAG_SMINSAMPLEVALUE:
+@@ -3763,6 +3767,19 @@ TIFFReadDirectory(TIFF* tif)
+ 					uint32 countrequired;
+ 					uint32 incrementpersample;
+ 					uint16* value=NULL;
++                    /* It would be dangerous to instanciate those tag values */
++                    /* since if td_bitspersample has not yet been read (due to */
++                    /* unordered tags), it could be read afterwards with a */
++                    /* values greater than the default one (1), which may cause */
++                    /* crashes in user code */
++                    if( !bitspersample_read )
++                    {
++                        fip = TIFFFieldWithTag(tif,dp->tdir_tag);
++                        TIFFWarningExt(tif->tif_clientdata,module,
++                                       "Ignoring %s since BitsPerSample tag not found",
++                                       fip ? fip->field_name : "unknown tagname");
++                        continue;
++                    }
+ 					countpersample=(1L<<tif->tif_dir.td_bitspersample);
+ 					if ((dp->tdir_tag==TIFFTAG_TRANSFERFUNCTION)&&(dp->tdir_count==(uint64)countpersample))
+ 					{
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index 074d32a..396ad08 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -182,8 +182,23 @@ TIFFRGBAImageOK(TIFF* tif, char emsg[1024])
+ 				    "Planarconfiguration", td->td_planarconfig);
+ 				return (0);
+ 			}
++			if( td->td_samplesperpixel != 3 )
++            {
++                sprintf(emsg,
++                        "Sorry, can not handle image with %s=%d",
++                        "Samples/pixel", td->td_samplesperpixel);
++                return 0;
++            }
+ 			break;
+ 		case PHOTOMETRIC_CIELAB:
++            if( td->td_samplesperpixel != 3 || td->td_bitspersample != 8 )
++            {
++                sprintf(emsg,
++                        "Sorry, can not handle image with %s=%d and %s=%d",
++                        "Samples/pixel", td->td_samplesperpixel,
++                        "Bits/sample", td->td_bitspersample);
++                return 0;
++            }
+ 			break;
+ 		default:
+ 			sprintf(emsg, "Sorry, can not handle image with %s=%d",
+diff --git a/libtiff/tif_next.c b/libtiff/tif_next.c
+index 55e2537..a53c716 100644
+--- a/libtiff/tif_next.c
++++ b/libtiff/tif_next.c
+@@ -102,6 +102,8 @@ NeXTDecode(TIFF* tif, uint8* buf, tmsize_t occ, uint16 s)
+ 		default: {
+ 			uint32 npixels = 0, grey;
+ 			uint32 imagewidth = tif->tif_dir.td_imagewidth;
++            if( isTiled(tif) )
++                imagewidth = tif->tif_dir.td_tilewidth;
+ 
+ 			/*
+ 			 * The scanline is composed of a sequence of constant
+diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
+index dfda963..f202b41 100644
+--- a/tools/tiff2pdf.c
++++ b/tools/tiff2pdf.c
+@@ -1167,6 +1167,15 @@ void t2p_read_tiff_init(T2P* t2p, TIFF* input){
+ 		if( (TIFFGetField(input, TIFFTAG_PLANARCONFIG, &xuint16) != 0)
+ 			&& (xuint16 == PLANARCONFIG_SEPARATE ) ){
+ 				TIFFGetField(input, TIFFTAG_SAMPLESPERPIXEL, &xuint16);
++                if( (t2p->tiff_tiles[i].tiles_tilecount % xuint16) != 0 )
++                {
++                    TIFFError(
++                        TIFF2PDF_MODULE, 
++                        "Invalid tile count, %s", 
++                        TIFFFileName(input));
++                    t2p->t2p_error = T2P_ERR_ERROR;
++                    return;
++                }
+ 				t2p->tiff_tiles[i].tiles_tilecount/= xuint16;
+ 		}
+ 		if( t2p->tiff_tiles[i].tiles_tilecount > 0){
+@@ -1552,6 +1561,22 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
+ #endif
+ 			break;
+ 		case PHOTOMETRIC_CIELAB:
++            if( t2p->tiff_samplesperpixel != 3){
++                TIFFError(
++                    TIFF2PDF_MODULE, 
++                    "Unsupported samplesperpixel = %d for CIELAB", 
++                    t2p->tiff_samplesperpixel);
++                t2p->t2p_error = T2P_ERR_ERROR;
++                return;
++            }
++            if( t2p->tiff_bitspersample != 8){
++                TIFFError(
++                    TIFF2PDF_MODULE, 
++                    "Invalid bitspersample = %d for CIELAB", 
++                    t2p->tiff_bitspersample);
++                t2p->t2p_error = T2P_ERR_ERROR;
++                return;
++            }
+ 			t2p->pdf_labrange[0]= -127;
+ 			t2p->pdf_labrange[1]= 127;
+ 			t2p->pdf_labrange[2]= -127;
+@@ -1567,6 +1592,22 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
+ 			t2p->pdf_colorspace=T2P_CS_LAB;
+ 			break;
+ 		case PHOTOMETRIC_ITULAB:
++            if( t2p->tiff_samplesperpixel != 3){
++                TIFFError(
++                    TIFF2PDF_MODULE, 
++                    "Unsupported samplesperpixel = %d for ITULAB", 
++                    t2p->tiff_samplesperpixel);
++                t2p->t2p_error = T2P_ERR_ERROR;
++                return;
++            }
++            if( t2p->tiff_bitspersample != 8){
++                TIFFError(
++                    TIFF2PDF_MODULE, 
++                    "Invalid bitspersample = %d for ITULAB", 
++                    t2p->tiff_bitspersample);
++                t2p->t2p_error = T2P_ERR_ERROR;
++                return;
++            }
+ 			t2p->pdf_labrange[0]=-85;
+ 			t2p->pdf_labrange[1]=85;
+ 			t2p->pdf_labrange[2]=-75;
+diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
+index f5530bb..4088463 100644
+--- a/tools/tiffcrop.c
++++ b/tools/tiffcrop.c
+@@ -1205,9 +1205,10 @@ static int writeBufferToContigTiles (TIFF* out, uint8* buf, uint32 imagelength,
+   tsize_t tilesize = TIFFTileSize(out);
+   unsigned char *tilebuf = NULL;
+ 
+-  TIFFGetField(out, TIFFTAG_TILELENGTH, &tl);
+-  TIFFGetField(out, TIFFTAG_TILEWIDTH, &tw);
+-  TIFFGetField(out, TIFFTAG_BITSPERSAMPLE, &bps);
++  if( !TIFFGetField(out, TIFFTAG_TILELENGTH, &tl) ||
++      !TIFFGetField(out, TIFFTAG_TILEWIDTH, &tw) ||
++      !TIFFGetField(out, TIFFTAG_BITSPERSAMPLE, &bps) )
++      return 1;
+ 
+   tile_buffsize = tilesize;
+   if (tilesize < (tsize_t)(tl * tile_rowsize))
+diff --git a/tools/tiffdump.c b/tools/tiffdump.c
+index cf5d62f..8247765 100644
+--- a/tools/tiffdump.c
++++ b/tools/tiffdump.c
+@@ -374,6 +374,8 @@ ReadDirectory(int fd, unsigned int ix, uint64 off)
+ 		void* datamem;
+ 		uint64 dataoffset;
+ 		int datatruncated;
++        int datasizeoverflow;
++
+ 		tag = *(uint16*)dp;
+ 		if (swabflag)
+ 			TIFFSwabShort(&tag);
+@@ -412,13 +414,14 @@ ReadDirectory(int fd, unsigned int ix, uint64 off)
+ 		else
+ 			typewidth = datawidth[type];
+ 		datasize = count*typewidth;
++        datasizeoverflow = (typewidth > 0 && datasize / typewidth != count);
+ 		datafits = 1;
+ 		datamem = dp;
+ 		dataoffset = 0;
+ 		datatruncated = 0;
+ 		if (!bigtiff)
+ 		{
+-			if (datasize>4)
++			if (datasizeoverflow || datasize>4)
+ 			{
+ 				uint32 dataoffset32;
+ 				datafits = 0;
+@@ -432,7 +435,7 @@ ReadDirectory(int fd, unsigned int ix, uint64 off)
+ 		}
+ 		else
+ 		{
+-			if (datasize>8)
++			if (datasizeoverflow || datasize>8)
+ 			{
+ 				datafits = 0;
+ 				datamem = NULL;
+@@ -442,7 +445,7 @@ ReadDirectory(int fd, unsigned int ix, uint64 off)
+ 			}
+ 			dp += sizeof(uint64);
+ 		}
+-		if (datasize>0x10000)
++		if (datasizeoverflow || datasize>0x10000)
+ 		{
+ 			datatruncated = 1;
+ 			count = 0x10000/typewidth;
+From 3206e0c752a62da1ae606867113ed3bf9bf73306 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 19:53:59 +0000
+Subject: [PATCH] * tools/thumbnail.c: fix out-of-buffer write
+ http://bugzilla.maptools.org/show_bug.cgi?id=2489 (CVE-2014-8128)
+
+---
+ ChangeLog         | 5 +++++
+ tools/thumbnail.c | 8 +++++++-
+ 2 files changed, 12 insertions(+), 1 deletion(-)
+
+diff --git a/tools/thumbnail.c b/tools/thumbnail.c
+index fab63f6..c50bbff 100644
+--- a/tools/thumbnail.c
++++ b/tools/thumbnail.c
+@@ -568,7 +568,13 @@ setImage1(const uint8* br, uint32 rw, uint32 rh)
+ 	    err -= limit;
+ 	    sy++;
+ 	    if (err >= limit)
+-		rows[nrows++] = br + bpr*sy;
++		{
++			/* We should perhaps error loudly, but I can't make sense of that */
++			/* code... */
++			if( nrows == 256 )
++				break;
++			rows[nrows++] = br + bpr*sy;
++		}
+ 	}
+ 	setrow(row, nrows, rows);
+ 	row += tnw;
+From 8b6e80fca434525497e5a31c3309a3bab5b3c1c8 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 18:52:42 +0000
+Subject: [PATCH] * tools/thumbnail.c, tools/tiffcmp.c: only read/write
+ TIFFTAG_GROUP3OPTIONS or TIFFTAG_GROUP4OPTIONS if compression is
+ COMPRESSION_CCITTFAX3 or COMPRESSION_CCITTFAX4
+ http://bugzilla.maptools.org/show_bug.cgi?id=2493 (CVE-2014-8128)
+
+---
+ ChangeLog         |  7 +++++++
+ tools/thumbnail.c | 21 ++++++++++++++++++++-
+ tools/tiffcmp.c   | 17 +++++++++++++++--
+ 3 files changed, 42 insertions(+), 3 deletions(-)
+
+diff --git a/tools/thumbnail.c b/tools/thumbnail.c
+index a98a881..fab63f6 100644
+--- a/tools/thumbnail.c
++++ b/tools/thumbnail.c
+@@ -274,7 +274,26 @@ cpTags(TIFF* in, TIFF* out)
+ {
+     struct cpTag *p;
+     for (p = tags; p < &tags[NTAGS]; p++)
+-	cpTag(in, out, p->tag, p->count, p->type);
++	{
++		/* Horrible: but TIFFGetField() expects 2 arguments to be passed */
++		/* if we request a tag that is defined in a codec, but that codec */
++		/* isn't used */
++		if( p->tag == TIFFTAG_GROUP3OPTIONS )
++		{
++			uint16 compression;
++			if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++				compression != COMPRESSION_CCITTFAX3 )
++				continue;
++		}
++		if( p->tag == TIFFTAG_GROUP4OPTIONS )
++		{
++			uint16 compression;
++			if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++				compression != COMPRESSION_CCITTFAX4 )
++				continue;
++		}
++		cpTag(in, out, p->tag, p->count, p->type);
++	}
+ }
+ #undef NTAGS
+ 
+diff --git a/tools/tiffcmp.c b/tools/tiffcmp.c
+index 508a461..d6392af 100644
+--- a/tools/tiffcmp.c
++++ b/tools/tiffcmp.c
+@@ -260,6 +260,7 @@ tiffcmp(TIFF* tif1, TIFF* tif2)
+ static int
+ cmptags(TIFF* tif1, TIFF* tif2)
+ {
++	uint16 compression1, compression2;
+ 	CmpLongField(TIFFTAG_SUBFILETYPE,	"SubFileType");
+ 	CmpLongField(TIFFTAG_IMAGEWIDTH,	"ImageWidth");
+ 	CmpLongField(TIFFTAG_IMAGELENGTH,	"ImageLength");
+@@ -276,8 +277,20 @@ cmptags(TIFF* tif1, TIFF* tif2)
+ 	CmpShortField(TIFFTAG_SAMPLEFORMAT,	"SampleFormat");
+ 	CmpFloatField(TIFFTAG_XRESOLUTION,	"XResolution");
+ 	CmpFloatField(TIFFTAG_YRESOLUTION,	"YResolution");
+-	CmpLongField(TIFFTAG_GROUP3OPTIONS,	"Group3Options");
+-	CmpLongField(TIFFTAG_GROUP4OPTIONS,	"Group4Options");
++	if( TIFFGetField(tif1, TIFFTAG_COMPRESSION, &compression1) &&
++		compression1 == COMPRESSION_CCITTFAX3 &&
++		TIFFGetField(tif2, TIFFTAG_COMPRESSION, &compression2) &&
++		compression2 == COMPRESSION_CCITTFAX3 )
++	{
++		CmpLongField(TIFFTAG_GROUP3OPTIONS,	"Group3Options");
++	}
++	if( TIFFGetField(tif1, TIFFTAG_COMPRESSION, &compression1) &&
++		compression1 == COMPRESSION_CCITTFAX4 &&
++		TIFFGetField(tif2, TIFFTAG_COMPRESSION, &compression2) &&
++		compression2 == COMPRESSION_CCITTFAX4 )
++	{
++		CmpLongField(TIFFTAG_GROUP4OPTIONS,	"Group4Options");
++	}
+ 	CmpShortField(TIFFTAG_RESOLUTIONUNIT,	"ResolutionUnit");
+ 	CmpShortField(TIFFTAG_PLANARCONFIG,	"PlanarConfiguration");
+ 	CmpLongField(TIFFTAG_ROWSPERSTRIP,	"RowsPerStrip");
+From 266bc48054b018a2f1d74562aa48eb2f509436d5 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 17:36:36 +0000
+Subject: [PATCH] * tools/tiff2pdf.c: check return code of TIFFGetField() when
+ reading TIFFTAG_SAMPLESPERPIXEL
+
+---
+ ChangeLog        |  5 +++++
+ tools/tiff2pdf.c | 10 +++++++++-
+ 2 files changed, 14 insertions(+), 1 deletion(-)
+
+Index: tiff-4.0.3/tools/tiff2pdf.c
+===================================================================
+--- tiff-4.0.3.orig/tools/tiff2pdf.c
++++ tiff-4.0.3/tools/tiff2pdf.c
+@@ -1164,7 +1164,15 @@ void t2p_read_tiff_init(T2P* t2p, TIFF*
+ 			t2p->tiff_pages[i].page_tilecount;
+ 		if( (TIFFGetField(input, TIFFTAG_PLANARCONFIG, &xuint16) != 0)
+ 			&& (xuint16 == PLANARCONFIG_SEPARATE ) ){
+-				TIFFGetField(input, TIFFTAG_SAMPLESPERPIXEL, &xuint16);
++				if( !TIFFGetField(input, TIFFTAG_SAMPLESPERPIXEL, &xuint16) )
++				{
++					TIFFError(
++                        TIFF2PDF_MODULE, 
++                        "Missing SamplesPerPixel, %s", 
++                        TIFFFileName(input));
++                    t2p->t2p_error = T2P_ERR_ERROR;
++                    return;
++				}
+                 if( (t2p->tiff_tiles[i].tiles_tilecount % xuint16) != 0 )
+                 {
+                     TIFFError(
+From cd82b5267ad4c10eb91e4ee8a716a81362cf851c Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Sun, 21 Dec 2014 18:07:48 +0000
+Subject: [PATCH] * libtiff/tif_next.c: check that BitsPerSample = 2. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2487 (CVE-2014-8129)
+
+---
+ ChangeLog          |  5 +++++
+ libtiff/tif_next.c | 17 +++++++++++++++++
+ 2 files changed, 22 insertions(+)
+
+diff --git a/libtiff/tif_next.c b/libtiff/tif_next.c
+index a53c716..d834196 100644
+--- a/libtiff/tif_next.c
++++ b/libtiff/tif_next.c
+@@ -141,10 +141,27 @@ NeXTDecode(TIFF* tif, uint8* buf, tmsize_t occ, uint16 s)
+ 	return (0);
+ }
+ 
++static int
++NeXTPreDecode(TIFF* tif, uint16 s)
++{
++	static const char module[] = "NeXTPreDecode";
++	TIFFDirectory *td = &tif->tif_dir;
++	(void)s;
++
++	if( td->td_bitspersample != 2 )
++	{
++		TIFFErrorExt(tif->tif_clientdata, module, "Unsupported BitsPerSample = %d",
++					 td->td_bitspersample);
++		return (0);
++	}
++	return (1);
++}
++	
+ int
+ TIFFInitNeXT(TIFF* tif, int scheme)
+ {
+ 	(void) scheme;
++	tif->tif_predecode = NeXTPreDecode;  
+ 	tif->tif_decoderow = NeXTDecode;  
+ 	tif->tif_decodestrip = NeXTDecode;  
+ 	tif->tif_decodetile = NeXTDecode;
+From 40a5955cbf0df62b1f9e9bd7d9657b0070725d19 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Mon, 29 Dec 2014 12:09:11 +0000
+Subject: [PATCH] * libtiff/tif_next.c: add new tests to check that we don't
+ read outside of the compressed input stream buffer.
+
+* libtiff/tif_getimage.c: in OJPEG case, fix checks on strile width/height
+---
+ ChangeLog              |  9 +++++++++
+ libtiff/tif_getimage.c | 12 +++++++-----
+ libtiff/tif_next.c     |  4 +++-
+ 3 files changed, 19 insertions(+), 6 deletions(-)
+
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index a4f46d9..3ad8ee7 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -1871,7 +1871,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile)
+ 
+     (void) y;
+     fromskew = (fromskew * 10) / 4;
+-    if ((h & 3) == 0 && (w & 1) == 0) {
++    if ((w & 3) == 0 && (h & 1) == 0) {
+         for (; h >= 2; h -= 2) {
+             x = w>>2;
+             do {
+@@ -1948,7 +1948,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile)
+     /* XXX adjust fromskew */
+     do {
+ 	x = w>>2;
+-	do {
++	while(x>0) {
+ 	    int32 Cb = pp[4];
+ 	    int32 Cr = pp[5];
+ 
+@@ -1959,7 +1959,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile)
+ 
+ 	    cp += 4;
+ 	    pp += 6;
+-	} while (--x);
++		x--;
++	}
+ 
+         if( (w&3) != 0 )
+         {
+@@ -2050,7 +2051,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile)
+ 	fromskew = (fromskew * 4) / 2;
+ 	do {
+ 		x = w>>1;
+-		do {
++		while(x>0) {
+ 			int32 Cb = pp[2];
+ 			int32 Cr = pp[3];
+ 
+@@ -2059,7 +2060,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile)
+ 
+ 			cp += 2;
+ 			pp += 4;
+-		} while (--x);
++			x --;
++		}
+ 
+ 		if( (w&1) != 0 )
+ 		{
+diff --git a/libtiff/tif_next.c b/libtiff/tif_next.c
+index d834196..dd669cc 100644
+--- a/libtiff/tif_next.c
++++ b/libtiff/tif_next.c
+@@ -71,7 +71,7 @@ NeXTDecode(TIFF* tif, uint8* buf, tmsize_t occ, uint16 s)
+ 		TIFFErrorExt(tif->tif_clientdata, module, "Fractional scanlines cannot be read");
+ 		return (0);
+ 	}
+-	for (row = buf; occ > 0; occ -= scanline, row += scanline) {
++	for (row = buf; cc > 0 && occ > 0; occ -= scanline, row += scanline) {
+ 		n = *bp++, cc--;
+ 		switch (n) {
+ 		case LITERALROW:
+@@ -90,6 +90,8 @@ NeXTDecode(TIFF* tif, uint8* buf, tmsize_t occ, uint16 s)
+ 			 * The scanline has a literal span that begins at some
+ 			 * offset.
+ 			 */
++			if( cc < 4 )
++				goto bad;
+ 			off = (bp[0] * 256) + bp[1];
+ 			n = (bp[2] * 256) + bp[3];
+ 			if (cc < 4+n || off+n > scanline)
+Picked from CVE: diff -u -r1.14 -r1.15
+http://bugzilla.maptools.org/show_bug.cgi?id=2501
+
+Author: Even Rouault <even.rouault@spatialys.com>
+
+--- tiff-4.0.3.orig/tools/tiffdither.c
++++ tiff-4.0.3/tools/tiffdither.c
+@@ -39,6 +39,7 @@
+ #endif
+ 
+ #include "tiffio.h"
++#include "tiffiop.h"
+ 
+ #define	streq(a,b)	(strcmp(a,b) == 0)
+ #define	strneq(a,b,n)	(strncmp(a,b,n) == 0)
+@@ -56,7 +57,7 @@ static	void usage(void);
+  * Floyd-Steinberg error propragation with threshold.
+  * This code is stolen from tiffmedian.
+  */
+-static void
++static int
+ fsdither(TIFF* in, TIFF* out)
+ {
+ 	unsigned char *outline, *inputline, *inptr;
+@@ -68,14 +69,19 @@ fsdither(TIFF* in, TIFF* out)
+ 	int lastline, lastpixel;
+ 	int bit;
+ 	tsize_t outlinesize;
++	int errcode = 0;
+ 
+ 	imax = imagelength - 1;
+ 	jmax = imagewidth - 1;
+ 	inputline = (unsigned char *)_TIFFmalloc(TIFFScanlineSize(in));
+-	thisline = (short *)_TIFFmalloc(imagewidth * sizeof (short));
+-	nextline = (short *)_TIFFmalloc(imagewidth * sizeof (short));
++	thisline = (short *)_TIFFmalloc(TIFFSafeMultiply(tmsize_t, imagewidth, sizeof (short)));
++	nextline = (short *)_TIFFmalloc(TIFFSafeMultiply(tmsize_t, imagewidth, sizeof (short)));
+ 	outlinesize = TIFFScanlineSize(out);
+ 	outline = (unsigned char *) _TIFFmalloc(outlinesize);
++	if (! (inputline && thisline && nextline && outline)) {
++	    fprintf(stderr, "Out of memory.\n");
++	    goto skip_on_error;
++	}
+ 
+ 	/*
+ 	 * Get first line
+@@ -93,7 +99,7 @@ fsdither(TIFF* in, TIFF* out)
+ 		nextline = tmpptr;
+ 		lastline = (i == imax);
+ 		if (TIFFReadScanline(in, inputline, i, 0) <= 0)
+-			break;
++			goto skip_on_error;
+ 		inptr = inputline;
+ 		nextptr = nextline;
+ 		for (j = 0; j < imagewidth; ++j)
+@@ -131,13 +137,18 @@ fsdither(TIFF* in, TIFF* out)
+ 			}
+ 		}
+ 		if (TIFFWriteScanline(out, outline, i-1, 0) < 0)
+-			break;
++			goto skip_on_error;
+ 	}
++	goto exit_label;
++
+   skip_on_error:
++	errcode = 1;
++  exit_label:
+ 	_TIFFfree(inputline);
+ 	_TIFFfree(thisline);
+ 	_TIFFfree(nextline);
+ 	_TIFFfree(outline);
++	return errcode;
+ }
+ 
+ static	uint16 compression = COMPRESSION_PACKBITS;
+Patches by Petr Gajdos (pgajdos@suse.cz) from
+http://bugzilla.maptools.org/show_bug.cgi?id=2499
+
+--- tiff-4.0.3.orig/libtiff/tif_dirinfo.c
++++ tiff-4.0.3/libtiff/tif_dirinfo.c
+@@ -141,6 +141,8 @@ tiffFields[] = {
+ 	{ TIFFTAG_FAXDCS, -1, -1, TIFF_ASCII, 0, TIFF_SETGET_ASCII, TIFF_SETGET_ASCII, FIELD_CUSTOM, TRUE, FALSE, "FaxDcs", NULL },
+ 	{ TIFFTAG_STONITS, 1, 1, TIFF_DOUBLE, 0, TIFF_SETGET_DOUBLE, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 0, 0, "StoNits", NULL },
+ 	{ TIFFTAG_INTEROPERABILITYIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_UNDEFINED, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 0, 0, "InteroperabilityIFDOffset", NULL },
++	{ TIFFTAG_CONSECUTIVEBADFAXLINES, 1, 1, TIFF_LONG, 0, TIFF_SETGET_UINT32, TIFF_SETGET_UINT32, FIELD_CUSTOM, TRUE, FALSE, "ConsecutiveBadFaxLines", NULL },
++        { TIFFTAG_PREDICTOR, 1, 1, TIFF_SHORT, 0, TIFF_SETGET_UINT16, TIFF_SETGET_UINT16, FIELD_CUSTOM, FALSE, FALSE, "Predictor", NULL },
+ 	/* begin DNG tags */
+ 	{ TIFFTAG_DNGVERSION, 4, 4, TIFF_BYTE, 0, TIFF_SETGET_C0_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 0, 0, "DNGVersion", NULL },
+ 	{ TIFFTAG_DNGBACKWARDVERSION, 4, 4, TIFF_BYTE, 0, TIFF_SETGET_C0_UINT8, TIFF_SETGET_UNDEFINED, FIELD_CUSTOM, 0, 0, "DNGBackwardVersion", NULL },

component/libtool/buildout.cfg

 [buildout]
+extends =
+  ../xz-utils/buildout.cfg
 parts = libtool
 
 [libtool]
 recipe = slapos.recipe.cmmi
-md5sum = d2f3b7d4627e69e13514a40e72a24d50
-url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.2.tar.gz
+md5sum = ee9c087775aeb98ce53a9c69da865a55
+url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.5.tar.xz
 configure-options =
   --disable-static
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/libxml2/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [libxml2]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.xmlsoft.org/libxml2/libxml2-2.9.1.tar.gz
-md5sum = 9c0cfef285d5c4a5c80d00904ddab380
+url = ftp://ftp.xmlsoft.org/libxml2/libxml2-2.9.2.tar.gz
+md5sum = 9e6a9aca9d155737868b3dc5fd82f788
 configure-options =
   --disable-static
   --without-python

component/libyaml/buildout.cfg

@@ -5,5 +5,5 @@ parts =
 
 [libyaml]
 recipe = slapos.recipe.cmmi
-url = http://pyyaml.org/download/libyaml/yaml-0.1.4.tar.gz
-md5sum = 36c852831d02cf90508c29852361d01b
+url = http://pyyaml.org/download/libyaml/yaml-0.1.6.tar.gz
+md5sum = 5fe00cda18ca5daeb43762b80c38e06e

component/lunzip/buildout.cfg

+[buildout]
+parts =
+  lunzip
+
+[lunzip]
+recipe = slapos.recipe.cmmi
+url = http://download.savannah.gnu.org/releases/lzip/lunzip/lunzip-1.6.tar.gz
+md5sum = 5e6ad4fe91f235be64227bc9930986fe
+configure-options =
+  --disable-static

component/lzo/buildout.cfg

+[buildout]
+
+parts =
+  lzo
+
+[lzo]
+recipe = slapos.recipe.cmmi
+url = http://www.oberhumer.com/opensource/lzo/download/lzo-2.09.tar.gz 
+md5sum = c7ffc9a103afe2d1bba0b015e7aa887f 

component/make/buildout.cfg

 [buildout]
-extends =
-  ../autoconf/buildout.cfg
-  ../automake/buildout.cfg
+parts =
+  make
 
 [make]
 # make 3.82 breaks too many things. Stick with 3.81.

component/manpy/buildout.cfg

+[buildout]
+extends =
+  ../scipy/buildout.cfg
+  ../rpy2/buildout.cfg
+  ../pulp/buildout.cfg
+  ../git/buildout.cfg
+  ../mysql-python/buildout.cfg
+
+parts = 
+  manpy
+
+[dream-repository.git]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/dream.git
+branch = master
+git-executable = ${git:location}/bin/git
+develop = true
+
+[dream-build]
+; use a develop egg to easily develop and because dream already exist on pypi
+; https://pypi.python.org/pypi/dream
+recipe = zc.recipe.egg:develop
+setup = ${dream-repository.git:location}
+
+[manpy-env]
+<= numpy-env
+
+[manpy]
+dependency = ${dream-build:setup}
+recipe = zc.recipe.egg:script
+eggs = dream
+environment = manpy-env
+setup-eggs =
+  ${scipy:egg}
+  ${mysql-python:egg}
+initialization =
+  ${pulp:initialization}
+  ${rpy2:initialization}

component/mariadb/buildout.cfg

@@ -7,7 +7,6 @@ extends =
   ../groonga/buildout.cfg
   ../jemalloc/buildout.cfg
   ../libaio/buildout.cfg
-  ../libevent/buildout.cfg
   ../libxml2/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
@@ -21,9 +20,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 10.0.14
+version = 10.0.19
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
-md5sum = 80fea71de54a9cfa7f5508df53d3f06d
+md5sum = aeaf101c688515dc8f73a5250e6c1df9
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
@@ -41,12 +40,13 @@ configure-options =
   -DENABLE_DTRACE=0
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
+  -DWITH_JEMALLOC=yes
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
+  -DWITHOUT_MROONGA_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
   -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
   -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
   -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
-  -DWITHOUT_TOKUDB=true
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
   CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${zlib:location}/include
@@ -57,8 +57,8 @@ environment =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-4.07.tar.gz
-md5sum = fcfca7934b1f8df981fca579d3cd41bd
+url = http://packages.groonga.org/source/mroonga/mroonga-5.03.tar.gz
+md5sum = 4f5413d5c94ebc44caeeb5cb62d346ed
 configure-command = mkdir fake_mariadb_source && ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql && ./configure
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

component/matplotlib/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+  ../pkgconfig/buildout.cfg
+  ../libpng/buildout.cfg
+  ../freetype/buildout.cfg
+
+parts =
+  matplotlib
+
+[matplotlib-env]
+<= numpy-env
+PATH = ${pkgconfig:location}/bin:%(PATH)s
+PKG_CONFIG_PATH = ${libpng:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+CFLAGS = -I${libpng:location}/include -I${freetype:location}/include
+
+[matplotlib]
+recipe = zc.recipe.egg:custom
+egg = matplotlib
+environment = matplotlib-env
+setup-eggs = ${numpy:egg}
+rpath =
+  ${libpng:location}/lib
+  ${freetype:location}/lib
+

component/mesa/buildout.cfg

@@ -38,6 +38,9 @@ environment =
   ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
   AUTOCONF=${autoconf:location}/bin/autoconf
   AUTOMAKE=${automake:location}/bin/automake
+make-options =
+  -j1
+  LIBTOOL=${libtool:location}/bin/libtool
 
 [glu]
 recipe = slapos.recipe.cmmi

component/msgpack-python/buildout.cfg

+[buildout]
+parts =
+  msgpack-python
+
+[msgpack-python]
+recipe = zc.recipe.egg:custom
+egg = msgpack-python

component/netcat/buildout.cfg

+[buildout]
+
+parts = netcat
+
+[netcat]
+recipe = slapos.recipe.cmmi
+url = http://sourceforge.net/projects/netcat/files/netcat/0.7.1/netcat-0.7.1.tar.gz
+md5sum = 088def25efe04dcdd1f8369d8926ab34
\ No newline at end of file

component/nginx/buildout.cfg

@@ -11,8 +11,8 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.7.6.tar.gz
-md5sum = dd444e5333e0d324bec480e2ff67870a
+url = http://nginx.org/download/nginx-1.7.8.tar.gz
+md5sum = fd5ab813fc1853cd8efe580ead577c3e
 
 [nginx]
 <= nginx-common

component/noVNC/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [noVNC]
 recipe = hexagonit.recipe.download
-# Post-0.4 release from January 2013
-url = http://github.com/kanaka/noVNC/zipball/3b2acc2258d36137a37edfbe0f03a3099189c49d
-md5sum = a276be8fa193652bb5de8a271603f11f
+# version-0.5.1 release from 29 Nov 2014
+url = https://github.com/kanaka/noVNC/archive/v0.5.1.tar.gz
+md5sum = ac55b2316b2164b6e09ae3bd89c37cb6
 strip-top-level-dir = true

component/nodejs/buildout.cfg

@@ -9,7 +9,22 @@ parts =
   nodejs
 
 [nodejs]
-<= nodejs-0.8
+<= nodejs-0.10
+
+[nodejs-0.10]
+# Server-side Javascript.
+recipe = slapos.recipe.cmmi
+url = http://nodejs.org/dist/v0.10.36/node-v0.10.36.tar.gz
+md5sum = 4b3527b830f2dacaba01aececd509c6f
+configure-options =
+  --openssl-includes=${openssl:location}/include
+  --openssl-libpath=${openssl:location}/lib
+environment =
+  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
+  CPPFLAGS=-I${zlib:location}/include
+  LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
 
 [nodejs-0.8]
 # Server-side Javascript.

component/numpy/buildout.cfg

@@ -2,21 +2,19 @@
 
 extends =
   ../gcc/buildout.cfg
-  ../libatlas/buildout.cfg
-
-version = versions
+  ../openblas/buildout.cfg
 
 [numpy-env]
 PATH = ${gcc-fortran:location}/bin:%(PATH)s
-BLAS = ${libatlas:location}/lib/libcblas.a
-LAPACK = ${libatlas:location}/lib/liblapack.a
-ATLAS = ${libatlas:location}/lib/libatlas.a
+OPENBLAS = ${openblas:location}/lib/libopenblas.so
+LAPACK = ${openblas:location}/lib/libopenblas.so
+ATLAS = ${openblas:location}/lib/libopenblas.so
 
 [numpy]
 recipe = zc.recipe.egg:custom
 egg = numpy
 environment = numpy-env
-# no need to add libatlas location because they are statically linked.
 rpath =
   ${gcc-fortran:location}/lib
   ${gcc-fortran:location}/lib64
+  ${openblas:location}/lib

component/openblas/buildout.cfg

+[buildout]
+parts =
+  openblas
+
+[openblas]
+recipe = slapos.recipe.cmmi
+url = http://github.com/xianyi/OpenBLAS/tarball/v0.2.13
+md5sum = 74adf4c0d0d82bff4774be5bf2134183
+configure-command = true
+make-options =
+  PREFIX="${buildout:parts-directory}/${:_buildout_section_name_}"
+  BINARY="$(uname -m | grep -q x86_64 && echo 64 || echo 32)"
+  NO_STATIC=1
+  USE_OPENMP=1
+  USE_THREAD=1
+# to build generic binary that supports multiple architecture in one binary
+# DYNAMIC_ARCH=1
+environment =
+  PATH=${gcc-fortran:location}/bin:%(PATH)s
+  LD_LIBRARY_PATH=${gcc-fortran:location}/lib:${gcc-fortran:location}/lib64
+  LDFLAGS=-Wl,-rpath=${gcc-fortran:location}/lib -Wl,-rpath=${gcc-fortran:location}/lib64

component/openldap/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [openldap]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-2.4.33.tgz
-md5sum = 5adae44897647c15ce5abbff313bc85a
+url = ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-2.4.40.tgz
+md5sum = 423c1f23d2a0cb96b3e9baf7e9d7dda7
 configure-options =
   --disable-static
   --disable-slapd

component/openssl/buildout.cfg

@@ -16,13 +16,14 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.1j.tar.gz
-md5sum = f7175c9cd3c39bb1907ac8bba9df8ed3
+url = https://www.openssl.org/source/openssl-1.0.1n.tar.gz
+md5sum = 139568bd5a56fa49b72a290d37113f30
 patch-binary = ${patch:location}/bin/patch
 patches =
-  ${:_profile_base_location_}/openssl-nodoc.patch#b4887a7b4e18402447bc6227d2493b92
-  ${:_profile_base_location_}/openssl-exlibs.patch#dfb8979460d6d75f2d23d1ea83bbb40a
-patch-options = -p0
+  ${:_profile_base_location_}/openssl-nodoc.patch#dd1345ae7ed18ccf176bc6b77c516f98
+  ${:_profile_base_location_}/openssl-exlibs.patch#fba5c873cf974ba80a973be41da3c738
+  ${:_profile_base_location_}/openssl-1.0.1m-parallel-build.patch#a1ddd93b5b296473c7446131deb31f93
+patch-options = -p1
 configure-command = ./config
 configure-options =
   -I${zlib:location}/include
@@ -33,10 +34,6 @@ configure-options =
   shared no-idea no-mdc2 no-rc5 zlib
   -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib
   && make depend
-
-# it seems that parallel build sometimes fails for openssl.
-make-options =
-  -j1
 make-targets =
   all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
 environment =

component/openssl/openssl-1.0.1m-parallel-build.patch

+https://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/dev-libs/openssl/files/
+http://rt.openssl.org/Ticket/Display.html?id=2084
+
+--- openssl-1.0.1m/crypto/Makefile
++++ openssl-1.0.1m/crypto/Makefile
+@@ -85,11 +85,11 @@
+ 	@if [ -z "$(THIS)" ]; then $(MAKE) -f $(TOP)/Makefile reflect THIS=$@; fi
+ 
+ subdirs:
+-	@target=all; $(RECURSIVE_MAKE)
++	+@target=all; $(RECURSIVE_MAKE)
+ 
+ files:
+ 	$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
+-	@target=files; $(RECURSIVE_MAKE)
++	+@target=files; $(RECURSIVE_MAKE)
+ 
+ links:
+ 	@$(PERL) $(TOP)/util/mklink.pl ../include/openssl $(EXHEADER)
+@@ -100,7 +100,7 @@
+ # lib: $(LIB): are splitted to avoid end-less loop
+ lib:	$(LIB)
+ 	@touch lib
+-$(LIB):	$(LIBOBJ)
++$(LIB):	$(LIBOBJ) | subdirs
+ 	$(AR) $(LIB) $(LIBOBJ)
+ 	[ -z "$(FIPSLIBDIR)" ] || $(AR) $(LIB) $(FIPSLIBDIR)fipscanister.o
+ 	$(RANLIB) $(LIB) || echo Never mind.
+@@ -111,7 +111,7 @@
+ 	fi
+ 
+ libs:
+-	@target=lib; $(RECURSIVE_MAKE)
++	+@target=lib; $(RECURSIVE_MAKE)
+ 
+ install:
+ 	@[ -n "$(INSTALLTOP)" ] # should be set by top Makefile...
+@@ -120,7 +120,7 @@
+ 	(cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \
+ 	chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \
+ 	done;
+-	@target=install; $(RECURSIVE_MAKE)
++	+@target=install; $(RECURSIVE_MAKE)
+ 
+ lint:
+ 	@target=lint; $(RECURSIVE_MAKE)
+--- openssl-1.0.1m/crypto/objects/Makefile
++++ openssl-1.0.1m/crypto/objects/Makefile
+@@ -44,11 +44,11 @@
+ # objects.pl both reads and writes obj_mac.num
+ obj_mac.h: objects.pl objects.txt obj_mac.num
+ 	$(PERL) objects.pl objects.txt obj_mac.num obj_mac.h
+-	@sleep 1; touch obj_mac.h; sleep 1
+ 
+-obj_xref.h: objxref.pl obj_xref.txt obj_mac.num
++# This doesn't really need obj_mac.h, but since that rule reads & writes
++# obj_mac.num, we can't run in parallel with it.
++obj_xref.h: objxref.pl obj_xref.txt obj_mac.num obj_mac.h
+ 	$(PERL) objxref.pl obj_mac.num obj_xref.txt > obj_xref.h
+-	@sleep 1; touch obj_xref.h; sleep 1
+ 
+ files:
+ 	$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
+--- openssl-1.0.1m/engines/Makefile
++++ openssl-1.0.1m/engines/Makefile
+@@ -72,7 +72,7 @@
+ 
+ all:	lib subdirs
+ 
+-lib:	$(LIBOBJ)
++lib:	$(LIBOBJ) | subdirs
+ 	@if [ -n "$(SHARED_LIBS)" ]; then \
+ 		set -e; \
+ 		for l in $(LIBNAMES); do \
+@@ -89,7 +89,7 @@
+ 
+ subdirs:
+ 	echo $(EDIRS)
+-	@target=all; $(RECURSIVE_MAKE)
++	+@target=all; $(RECURSIVE_MAKE)
+ 
+ files:
+ 	$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
+@@ -128,7 +128,7 @@
+ 			  mv -f $(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines/$$pfx$$l$$sfx.new $(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines/$$pfx$$l$$sfx ); \
+ 		done; \
+ 	fi
+-	@target=install; $(RECURSIVE_MAKE)
++	+@target=install; $(RECURSIVE_MAKE)
+ 
+ tags:
+ 	ctags $(SRC)
+--- openssl-1.0.1m/Makefile.org
++++ openssl-1.0.1m/Makefile.org
+@@ -273,17 +273,17 @@
+ build_libs: build_crypto build_ssl build_engines
+ 
+ build_crypto:
+-	@dir=crypto; target=all; $(BUILD_ONE_CMD)
+-build_ssl:
+-	@dir=ssl; target=all; $(BUILD_ONE_CMD)
+-build_engines:
+-	@dir=engines; target=all; $(BUILD_ONE_CMD)
+-build_apps:
+-	@dir=apps; target=all; $(BUILD_ONE_CMD)
+-build_tests:
+-	@dir=test; target=all; $(BUILD_ONE_CMD)
+-build_tools:
+-	@dir=tools; target=all; $(BUILD_ONE_CMD)
++	+@dir=crypto; target=all; $(BUILD_ONE_CMD)
++build_ssl: build_crypto
++	+@dir=ssl; target=all; $(BUILD_ONE_CMD)
++build_engines: build_crypto
++	+@dir=engines; target=all; $(BUILD_ONE_CMD)
++build_apps: build_libs
++	+@dir=apps; target=all; $(BUILD_ONE_CMD)
++build_tests: build_libs
++	+@dir=test; target=all; $(BUILD_ONE_CMD)
++build_tools: build_libs
++	+@dir=tools; target=all; $(BUILD_ONE_CMD)
+ 
+ all_testapps: build_libs build_testapps
+ build_testapps:
+@@ -538,9 +538,9 @@
+ dist_pem_h:
+ 	(cd crypto/pem; $(MAKE) -e $(BUILDENV) pem.h; $(MAKE) clean)
+ 
+-install: all install_docs install_sw
++install: install_docs install_sw
+ 
+-install_sw:
++install_dirs:
+ 	@$(PERL) $(TOP)/util/mkdir-p.pl $(INSTALL_PREFIX)$(INSTALLTOP)/bin \
+ 		$(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR) \
+ 		$(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines \
+@@ -549,12 +549,19 @@
+ 		$(INSTALL_PREFIX)$(OPENSSLDIR)/misc \
+ 		$(INSTALL_PREFIX)$(OPENSSLDIR)/certs \
+ 		$(INSTALL_PREFIX)$(OPENSSLDIR)/private
++	@$(PERL) $(TOP)/util/mkdir-p.pl \
++		$(INSTALL_PREFIX)$(MANDIR)/man1 \
++		$(INSTALL_PREFIX)$(MANDIR)/man3 \
++		$(INSTALL_PREFIX)$(MANDIR)/man5 \
++		$(INSTALL_PREFIX)$(MANDIR)/man7
++
++install_sw: install_dirs
+ 	@set -e; headerlist="$(EXHEADER)"; for i in $$headerlist;\
+ 	do \
+ 	(cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \
+ 	chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \
+ 	done;
+-	@set -e; target=install; $(RECURSIVE_BUILD_CMD)
++	+@set -e; target=install; $(RECURSIVE_BUILD_CMD)
+ 	@set -e; liblist="$(LIBS)"; for i in $$liblist ;\
+ 	do \
+ 		if [ -f "$$i" ]; then \
+@@ -634,12 +641,7 @@
+ 		done; \
+ 	done
+ 
+-install_docs:
+-	@$(PERL) $(TOP)/util/mkdir-p.pl \
+-		$(INSTALL_PREFIX)$(MANDIR)/man1 \
+-		$(INSTALL_PREFIX)$(MANDIR)/man3 \
+-		$(INSTALL_PREFIX)$(MANDIR)/man5 \
+-		$(INSTALL_PREFIX)$(MANDIR)/man7
++install_docs: install_dirs
+ 	@pod2man="`cd ./util; ./pod2mantest $(PERL)`"; \
+ 	here="`pwd`"; \
+ 	filecase=; \
+--- openssl-1.0.1m/Makefile.shared
++++ openssl-1.0.1m/Makefile.shared
+@@ -105,6 +105,7 @@
+     SHAREDFLAGS="$${SHAREDFLAGS:-$(CFLAGS) $(SHARED_LDFLAGS)}"; \
+     LIBPATH=`for x in $$LIBDEPS; do echo $$x; done | sed -e 's/^ *-L//;t' -e d | uniq`; \
+     LIBPATH=`echo $$LIBPATH | sed -e 's/ /:/g'`; \
++    [ -e $$SHLIB$$SHLIB_SOVER$$SHLIB_SUFFIX ] && exit 0; \
+     LD_LIBRARY_PATH=$$LIBPATH:$$LD_LIBRARY_PATH \
+     $${SHAREDCMD} $${SHAREDFLAGS} \
+ 	-o $$SHLIB$$SHLIB_SOVER$$SHLIB_SUFFIX \
+@@ -122,6 +123,7 @@
+ 			done; \
+ 		fi; \
+ 		if [ -n "$$SHLIB_SOVER" ]; then \
++			[ -e "$$SHLIB$$SHLIB_SUFFIX" ] || \
+ 			( $(SET_X); rm -f $$SHLIB$$SHLIB_SUFFIX; \
+ 			  ln -s $$prev $$SHLIB$$SHLIB_SUFFIX ); \
+ 		fi; \
+--- openssl-1.0.1m/test/Makefile
++++ openssl-1.0.1m/test/Makefile
+@@ -130,7 +130,7 @@
+ tags:
+ 	ctags $(SRC)
+ 
+-tests:	exe apps $(TESTS)
++tests:	exe $(TESTS)
+ 
+ apps:
+ 	@(cd ..; $(MAKE) DIRS=apps all)
+@@ -388,118 +388,118 @@
+ 		link_app.$${shlib_target}
+ 
+ $(RSATEST)$(EXE_EXT): $(RSATEST).o $(DLIBCRYPTO)
+-	@target=$(RSATEST); $(BUILD_CMD)
++	+@target=$(RSATEST); $(BUILD_CMD)
+ 
+ $(BNTEST)$(EXE_EXT): $(BNTEST).o $(DLIBCRYPTO)
+-	@target=$(BNTEST); $(BUILD_CMD)
++	+@target=$(BNTEST); $(BUILD_CMD)
+ 
+ $(ECTEST)$(EXE_EXT): $(ECTEST).o $(DLIBCRYPTO)
+-	@target=$(ECTEST); $(BUILD_CMD)
++	+@target=$(ECTEST); $(BUILD_CMD)
+ 
+ $(EXPTEST)$(EXE_EXT): $(EXPTEST).o $(DLIBCRYPTO)
+-	@target=$(EXPTEST); $(BUILD_CMD)
++	+@target=$(EXPTEST); $(BUILD_CMD)
+ 
+ $(IDEATEST)$(EXE_EXT): $(IDEATEST).o $(DLIBCRYPTO)
+-	@target=$(IDEATEST); $(BUILD_CMD)
++	+@target=$(IDEATEST); $(BUILD_CMD)
+ 
+ $(MD2TEST)$(EXE_EXT): $(MD2TEST).o $(DLIBCRYPTO)
+-	@target=$(MD2TEST); $(BUILD_CMD)
++	+@target=$(MD2TEST); $(BUILD_CMD)
+ 
+ $(SHATEST)$(EXE_EXT): $(SHATEST).o $(DLIBCRYPTO)
+-	@target=$(SHATEST); $(BUILD_CMD)
++	+@target=$(SHATEST); $(BUILD_CMD)
+ 
+ $(SHA1TEST)$(EXE_EXT): $(SHA1TEST).o $(DLIBCRYPTO)
+-	@target=$(SHA1TEST); $(BUILD_CMD)
++	+@target=$(SHA1TEST); $(BUILD_CMD)
+ 
+ $(SHA256TEST)$(EXE_EXT): $(SHA256TEST).o $(DLIBCRYPTO)
+-	@target=$(SHA256TEST); $(BUILD_CMD)
++	+@target=$(SHA256TEST); $(BUILD_CMD)
+ 
+ $(SHA512TEST)$(EXE_EXT): $(SHA512TEST).o $(DLIBCRYPTO)
+-	@target=$(SHA512TEST); $(BUILD_CMD)
++	+@target=$(SHA512TEST); $(BUILD_CMD)
+ 
+ $(RMDTEST)$(EXE_EXT): $(RMDTEST).o $(DLIBCRYPTO)
+-	@target=$(RMDTEST); $(BUILD_CMD)
++	+@target=$(RMDTEST); $(BUILD_CMD)
+ 
+ $(MDC2TEST)$(EXE_EXT): $(MDC2TEST).o $(DLIBCRYPTO)
+-	@target=$(MDC2TEST); $(BUILD_CMD)
++	+@target=$(MDC2TEST); $(BUILD_CMD)
+ 
+ $(MD4TEST)$(EXE_EXT): $(MD4TEST).o $(DLIBCRYPTO)
+-	@target=$(MD4TEST); $(BUILD_CMD)
++	+@target=$(MD4TEST); $(BUILD_CMD)
+ 
+ $(MD5TEST)$(EXE_EXT): $(MD5TEST).o $(DLIBCRYPTO)
+-	@target=$(MD5TEST); $(BUILD_CMD)
++	+@target=$(MD5TEST); $(BUILD_CMD)
+ 
+ $(HMACTEST)$(EXE_EXT): $(HMACTEST).o $(DLIBCRYPTO)
+-	@target=$(HMACTEST); $(BUILD_CMD)
++	+@target=$(HMACTEST); $(BUILD_CMD)
+ 
+ $(WPTEST)$(EXE_EXT): $(WPTEST).o $(DLIBCRYPTO)
+-	@target=$(WPTEST); $(BUILD_CMD)
++	+@target=$(WPTEST); $(BUILD_CMD)
+ 
+ $(RC2TEST)$(EXE_EXT): $(RC2TEST).o $(DLIBCRYPTO)
+-	@target=$(RC2TEST); $(BUILD_CMD)
++	+@target=$(RC2TEST); $(BUILD_CMD)
+ 
+ $(BFTEST)$(EXE_EXT): $(BFTEST).o $(DLIBCRYPTO)
+-	@target=$(BFTEST); $(BUILD_CMD)
++	+@target=$(BFTEST); $(BUILD_CMD)
+ 
+ $(CASTTEST)$(EXE_EXT): $(CASTTEST).o $(DLIBCRYPTO)
+-	@target=$(CASTTEST); $(BUILD_CMD)
++	+@target=$(CASTTEST); $(BUILD_CMD)
+ 
+ $(RC4TEST)$(EXE_EXT): $(RC4TEST).o $(DLIBCRYPTO)
+-	@target=$(RC4TEST); $(BUILD_CMD)
++	+@target=$(RC4TEST); $(BUILD_CMD)
+ 
+ $(RC5TEST)$(EXE_EXT): $(RC5TEST).o $(DLIBCRYPTO)
+-	@target=$(RC5TEST); $(BUILD_CMD)
++	+@target=$(RC5TEST); $(BUILD_CMD)
+ 
+ $(DESTEST)$(EXE_EXT): $(DESTEST).o $(DLIBCRYPTO)
+-	@target=$(DESTEST); $(BUILD_CMD)
++	+@target=$(DESTEST); $(BUILD_CMD)
+ 
+ $(RANDTEST)$(EXE_EXT): $(RANDTEST).o $(DLIBCRYPTO)
+-	@target=$(RANDTEST); $(BUILD_CMD)
++	+@target=$(RANDTEST); $(BUILD_CMD)
+ 
+ $(DHTEST)$(EXE_EXT): $(DHTEST).o $(DLIBCRYPTO)
+-	@target=$(DHTEST); $(BUILD_CMD)
++	+@target=$(DHTEST); $(BUILD_CMD)
+ 
+ $(DSATEST)$(EXE_EXT): $(DSATEST).o $(DLIBCRYPTO)
+-	@target=$(DSATEST); $(BUILD_CMD)
++	+@target=$(DSATEST); $(BUILD_CMD)
+ 
+ $(METHTEST)$(EXE_EXT): $(METHTEST).o $(DLIBCRYPTO)
+-	@target=$(METHTEST); $(BUILD_CMD)
++	+@target=$(METHTEST); $(BUILD_CMD)
+ 
+ $(SSLTEST)$(EXE_EXT): $(SSLTEST).o $(DLIBSSL) $(DLIBCRYPTO)
+-	@target=$(SSLTEST); $(FIPS_BUILD_CMD)
++	+@target=$(SSLTEST); $(FIPS_BUILD_CMD)
+ 
+ $(ENGINETEST)$(EXE_EXT): $(ENGINETEST).o $(DLIBCRYPTO)
+-	@target=$(ENGINETEST); $(BUILD_CMD)
++	+@target=$(ENGINETEST); $(BUILD_CMD)
+ 
+ $(EVPTEST)$(EXE_EXT): $(EVPTEST).o $(DLIBCRYPTO)
+-	@target=$(EVPTEST); $(BUILD_CMD)
++	+@target=$(EVPTEST); $(BUILD_CMD)
+ 
+ $(EVPEXTRATEST)$(EXE_EXT): $(EVPEXTRATEST).o $(DLIBCRYPTO)
+-	@target=$(EVPEXTRATEST); $(BUILD_CMD)
++	+@target=$(EVPEXTRATEST); $(BUILD_CMD)
+ 
+ $(ECDSATEST)$(EXE_EXT): $(ECDSATEST).o $(DLIBCRYPTO)
+-	@target=$(ECDSATEST); $(BUILD_CMD)
++	+@target=$(ECDSATEST); $(BUILD_CMD)
+ 
+ $(ECDHTEST)$(EXE_EXT): $(ECDHTEST).o $(DLIBCRYPTO)
+-	@target=$(ECDHTEST); $(BUILD_CMD)
++	+@target=$(ECDHTEST); $(BUILD_CMD)
+ 
+ $(IGETEST)$(EXE_EXT): $(IGETEST).o $(DLIBCRYPTO)
+-	@target=$(IGETEST); $(BUILD_CMD)
++	+@target=$(IGETEST); $(BUILD_CMD)
+ 
+ $(JPAKETEST)$(EXE_EXT): $(JPAKETEST).o $(DLIBCRYPTO)
+-	@target=$(JPAKETEST); $(BUILD_CMD)
++	+@target=$(JPAKETEST); $(BUILD_CMD)
+ 
+ $(ASN1TEST)$(EXE_EXT): $(ASN1TEST).o $(DLIBCRYPTO)
+-	@target=$(ASN1TEST); $(BUILD_CMD)
++	+@target=$(ASN1TEST); $(BUILD_CMD)
+ 
+ $(SRPTEST)$(EXE_EXT): $(SRPTEST).o $(DLIBCRYPTO)
+-	@target=$(SRPTEST); $(BUILD_CMD)
++	+@target=$(SRPTEST); $(BUILD_CMD)
+ 
+ $(HEARTBEATTEST)$(EXE_EXT): $(HEARTBEATTEST).o $(DLIBCRYPTO)
+-	@target=$(HEARTBEATTEST); $(BUILD_CMD_STATIC)
++	+@target=$(HEARTBEATTEST); $(BUILD_CMD_STATIC)
+ 
+ $(CONSTTIMETEST)$(EXE_EXT): $(CONSTTIMETEST).o
+-	@target=$(CONSTTIMETEST) $(BUILD_CMD)
++	+@target=$(CONSTTIMETEST) $(BUILD_CMD)
+ 
+ #$(AESTEST).o: $(AESTEST).c
+ #	$(CC) -c $(CFLAGS) -DINTERMEDIATE_VALUE_KAT -DTRACE_KAT_MCT $(AESTEST).c
+@@ -512,7 +512,7 @@
+ #	fi
+ 
+ dummytest$(EXE_EXT): dummytest.o $(DLIBCRYPTO)
+-	@target=dummytest; $(BUILD_CMD)
++	+@target=dummytest; $(BUILD_CMD)
+ 
+ # DO NOT DELETE THIS LINE -- make depend depends on it.
+ 

component/openssl/openssl-exlibs.patch

---- engines/ccgost/Makefile~	2010-08-24 23:46:34.000000000 +0200
-+++ engines/ccgost/Makefile	2012-03-14 10:11:46.826419864 +0100
+--- a/engines/ccgost/Makefile~	2010-08-24 23:46:34.000000000 +0200
++++ b/engines/ccgost/Makefile	2012-03-14 10:11:46.826419864 +0100
 @@ -7,6 +7,7 @@
  AR= ar r
  CFLAGS= $(INCLUDES) $(CFLAG)

component/openssl/openssl-nodoc.patch

---- Makefile	2012-01-18 14:42:28.000000000 +0100
-+++ Makefile	2012-01-24 17:43:40.000000000 +0100
+--- a/Makefile	2012-01-18 14:42:28.000000000 +0100
++++ b/Makefile	2012-01-24 17:43:40.000000000 +0100
 @@ -494,7 +494,7 @@
  dist_pem_h:
  	(cd crypto/pem; $(MAKE) -e $(BUILDENV) pem.h; $(MAKE) clean)

component/openvpn/buildout.cfg

+[buildout]
+extends =
+  ../lzo/buildout.cfg
+  ../patch/buildout.cfg
+  ../openssl/buildout.cfg
+  ../flex/buildout.cfg
+
+parts =
+  openvpn
+
+[openvpn]
+recipe = slapos.recipe.cmmi
+url = http://swupdate.openvpn.org/community/releases/openvpn-2.3.6.tar.gz
+md5sum = 6ca03fe0fd093e0d01601abee808835c 
+patch-binary = ${patch:location}/bin/patch
+patches =
+  ${:_profile_base_location_}/openvpn-ciphernone.patch#462b53a45da2fa686822618c4faafd19
+patch-options = -p0
+configure-options = 
+  --disable-plugin-auth-pam
+  --enable-static
+/bin/bash: q: command not found
+environment =
+  LZO_LIBS=-L${lzo:location}/lib -llzo2
+  LZO_CFLAGS=-I${lzo:location}/include
+  OPENSSL_SSL_LIBS="-L${openssl:location}/lib -lssl" 
+  OPENSSL_SSL_CFLAGS="-I${openssl:location}/include/" 
+  OPENSSL_CRYPTO_LIBS="-L${openssl:location}/lib -lcrypto" 
+  OPENSSL_CRYPTO_CFLAGS="-I${openssl:location}/include"
+  LDFLAGS =-Wl,-rpath=${lzo:location}/lib -Wl,-rpath=${flex:location}/lib -Wl,-rpath=${openssl:location}/lib
+  CPPFLAGS=-I${lzo:location}/include -I${flex:location}/include
+

component/openvpn/openvpn-ciphernone.patch

+--- src/openvpn/crypto_backend.h
++++ src/openvpn/crypto_backend.h
+@@ -237,8 +237,7 @@
+  *
+  * @return		true iff the cipher is a CBC mode cipher.
+  */
+-bool cipher_kt_mode_cbc(const cipher_kt_t *cipher)
+-  __attribute__((nonnull));
++bool cipher_kt_mode_cbc(const cipher_kt_t *cipher);
+ 
+ /**
+  * Check if the supplied cipher is a supported OFB or CFB mode cipher.
+@@ -247,8 +246,7 @@
+  *
+  * @return		true iff the cipher is a OFB or CFB mode cipher.
+  */
+-bool cipher_kt_mode_ofb_cfb(const cipher_kt_t *cipher)
+-  __attribute__((nonnull));
++bool cipher_kt_mode_ofb_cfb(const cipher_kt_t *cipher);
+ 
+ 
+ /**

component/p7zip/buildout.cfg

+[buildout]
+parts =
+  p7zip
+
+[p7zip]
+recipe = slapos.recipe.cmmi
+url = http://downloads.sf.net/project/p7zip/p7zip/${:version}/p7zip_${:version}_src_all.tar.bz2
+version = 9.38.1
+md5sum = 6cba8402ccab2370d3b70c5e28b3d651
+configure-command = rm -r DOC
+make-options =
+  DEST_HOME=${buildout:parts-directory}/${:_buildout_section_name_}

component/pandas/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+
+parts =
+  pandas
+
+[pandas-env]
+<= numpy-env
+
+[pandas]
+recipe = zc.recipe.egg:custom
+egg = pandas
+environment = pandas-env
+setup-eggs = ${numpy:egg}

component/patch/buildout.cfg

@@ -6,7 +6,7 @@ parts =
 
 [patch]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/gnu/patch/patch-2.7.1.tar.xz
-md5sum = e9ae5393426d3ad783a300a338c09b72
+url = ftp://ftp.gnu.org/gnu/patch/patch-2.7.3.tar.xz
+md5sum = 29b87be845e4662ab0ca0d48a805ecc6
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s

component/perl/buildout.cfg

@@ -7,9 +7,9 @@ parts =
 
 [perl]
 recipe = slapos.recipe.cmmi
-version = 5.20.0
+version = 5.20.2
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
-md5sum = 20cbecd4e9e880ee7a50a136c8b1484e
+md5sum = 21062666f1c627aeb6dbff3c6952738b
 siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =
@@ -25,4 +25,4 @@ configure-command =
     -Dusethreads
 environment =
   PATH=${patch:location}/bin:%(PATH)s
-post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#7fded8308c1676decf77575c6d6b325f:post_make_hook
+post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#539cb3cd0d1090f7f30a8e5a82b37854:post_make_hook

component/perl/perl-create-libs-symlink.py

@@ -15,7 +15,7 @@ def post_make_hook(options, buildout):
     print "ERROR - no libperl.* found!"
     exit
   elif nr_matches > 1:
-    print "WARNING - several libperl.a found, taking only the first one:", matches.join("\n")
+    print "WARNING - several libperl.a found, taking only the first one:", "\n".join(matches)
   
   # matches[0] is a prefix of "location"
   # For the symlink, we want the relative path.

component/poppler/buildout.cfg

@@ -3,24 +3,23 @@ parts = poppler
 extends =
   ../bzip2/buildout.cfg
   ../fontconfig/buildout.cfg
-  ../freetype/buildout.cfg
   ../jbigkit/buildout.cfg
   ../libjpeg/buildout.cfg
   ../libpng/buildout.cfg
   ../libtiff/buildout.cfg
-  ../libxml2/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 
 [poppler]
 recipe = slapos.recipe.cmmi
-url = http://poppler.freedesktop.org/poppler-0.26.0.tar.xz
-md5sum = 76c3b2f18002a0f1e2bbbc7cccd1b957
+url = http://poppler.freedesktop.org/poppler-0.30.0.tar.xz
+md5sum = fb0a9d82b5433c823421a123eb8e9789
 configure-options =
   --disable-cairo-output
   --disable-cms
   --disable-libcurl
+  --disable-libopenjpeg
   --disable-gtk-doc-html
   --disable-gtk-test
   --disable-poppler-cpp

component/proj4/buildout.cfg

+[buildout]
+parts =
+  proj4
+
+[proj4]
+recipe = slapos.recipe.cmmi
+version = 4.8.0
+url = http://download.osgeo.org/proj/proj-${:version}.tar.gz
+md5sum = d815838c92a29179298c126effbb1537
+configure-options =
+  --disable-dependency-tracking

component/pulp/buildout.cfg

+[buildout]
+extends =
+  ../glpk/buildout.cfg
+  ../numpy/buildout.cfg
+
+parts =
+  pulp
+
+# XXX patched pulp version that does not specify pyparsing version number
+# because it conflicts with slapos version
+[pulp-repository.git]
+recipe = slapos.recipe.build:gitclone
+repository = https://gitlab.com/jerome-nexedi/pulp-or.git
+branch = master
+git-executable = ${git:location}/bin/git
+develop = true
+
+[pulp-build]
+recipe = zc.recipe.egg:develop
+setup = ${pulp-repository.git:location}
+
+[pulp-env]
+<= numpy-env
+PATH=${glpk:location}/bin
+
+[pulp]
+dependencies = ${pulp-build:setup}
+recipe = zc.recipe.egg:custom
+egg = pulp
+environment = pulp-env
+setup-eggs = ${numpy:egg}
+initialization =
+  # pulp needs glpk in $PATH
+  import os
+  os.environ['PATH'] = '${glpk:location}/bin' + os.pathsep + os.environ.get('PATH', '')

component/pycrypto-python/buildout.cfg

@@ -5,8 +5,6 @@ extends =
 parts =
   pycrypto-python
 
-versions = versions
-
 [pycrypto-python]
 recipe = zc.recipe.egg:custom
 egg = pycrypto
@@ -22,6 +20,3 @@ rpath =
 ac_cv_func_malloc_0_nonnull = yes
 LDFLAGS = -L${gmp:location}/lib
 CPPFLAGS = -I${gmp:location}/include
-
-[versions]
-pycrypto = 2.6.1

component/python-2.7/buildout.cfg

@@ -27,9 +27,9 @@ python = python2.7
 
 [python2.7]
 recipe = slapos.recipe.cmmi
-package_version = 2.7.8
+package_version = 2.7.10
 package_version_suffix =
-md5sum = d235bdfa75b8396942e360a70487ee00
+md5sum = c685ef0b8e9f27b5e3db5db12b268ac6
 
 # This is actually the default setting for prefix, but we can't use it in
 # other settings in this part if we don't set it explicitly here.
@@ -39,8 +39,7 @@ executable = ${:prefix}/bin/python${:version}
 
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/tls_sni.patch#c95af105e6e96aaa58a50137595872a0
-  ${:_profile_base_location_}/tls_sni_httplib.patch#5c9d00d23b85169df792a936a056cbcc
+  ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
 url =
   http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/python-2.7/fix_compiler_module_issue_20613.patch

+From af98230024f639d4c719b55d0d330912c3ea7b97 Mon Sep 17 00:00:00 2001
+From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+Date: Thu, 11 Dec 2014 23:48:22 +0900
+Subject: [PATCH] Revert https://hg.python.org/cpython/raw-rev/42faa8054c3d.
+ This fixes http://bugs.python.org/issue20613.
+
+---
+ Lib/compiler/pyassem.py   | 257 ++++++++++++++++++++++++++++------------------
+ Lib/compiler/pycodegen.py |   2 +-
+ 2 files changed, 159 insertions(+), 100 deletions(-)
+
+diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
+index f52f7d0..4299a17 100644
+--- a/Lib/compiler/pyassem.py
++++ b/Lib/compiler/pyassem.py
+@@ -21,7 +21,6 @@ class FlowGraph:
+             if self.current:
+                 print "end", repr(self.current)
+                 print "    next", self.current.next
+-                print "    prev", self.current.prev
+                 print "   ", self.current.get_children()
+             print repr(block)
+         self.current = block
+@@ -41,12 +40,13 @@ class FlowGraph:
+         if block is None:
+             block = self.newBlock()
+ 
+-        # Note: If the current block ends with an unconditional control
+-        # transfer, then it is techically incorrect to add an implicit
+-        # transfer to the block graph. Doing so results in code generation
+-        # for unreachable blocks.  That doesn't appear to be very common
+-        # with Python code and since the built-in compiler doesn't optimize
+-        # it out we don't either.
++        # Note: If the current block ends with an unconditional
++        # control transfer, then it is incorrect to add an implicit
++        # transfer to the block graph.  The current code requires
++        # these edges to get the blocks emitted in the right order,
++        # however. :-(  If a client needs to remove these edges, call
++        # pruneEdges().
++
+         self.current.addNext(block)
+         self.startBlock(block)
+ 
+@@ -69,6 +69,8 @@ class FlowGraph:
+     def emit(self, *inst):
+         if self._debug:
+             print "\t", inst
++        if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']:
++            self.current.addOutEdge(self.exit)
+         if len(inst) == 2 and isinstance(inst[1], Block):
+             self.current.addOutEdge(inst[1])
+         self.current.emit(inst)
+@@ -78,9 +80,118 @@ class FlowGraph:
+ 
+         i.e. each node appears before all of its successors
+         """
+-        order = order_blocks(self.entry, self.exit)
++        # XXX make sure every node that doesn't have an explicit next
++        # is set so that next points to exit
++        for b in self.blocks.elements():
++            if b is self.exit:
++                continue
++            if not b.next:
++                b.addNext(self.exit)
++        order = dfs_postorder(self.entry, {})
++        order.reverse()
++        self.fixupOrder(order, self.exit)
++        # hack alert
++        if not self.exit in order:
++            order.append(self.exit)
++
+         return order
+ 
++    def fixupOrder(self, blocks, default_next):
++        """Fixup bad order introduced by DFS."""
++
++        # XXX This is a total mess.  There must be a better way to get
++        # the code blocks in the right order.
++
++        self.fixupOrderHonorNext(blocks, default_next)
++        self.fixupOrderForward(blocks, default_next)
++
++    def fixupOrderHonorNext(self, blocks, default_next):
++        """Fix one problem with DFS.
++
++        The DFS uses child block, but doesn't know about the special
++        "next" block.  As a result, the DFS can order blocks so that a
++        block isn't next to the right block for implicit control
++        transfers.
++        """
++        index = {}
++        for i in range(len(blocks)):
++            index[blocks[i]] = i
++
++        for i in range(0, len(blocks) - 1):
++            b = blocks[i]
++            n = blocks[i + 1]
++            if not b.next or b.next[0] == default_next or b.next[0] == n:
++                continue
++            # The blocks are in the wrong order.  Find the chain of
++            # blocks to insert where they belong.
++            cur = b
++            chain = []
++            elt = cur
++            while elt.next and elt.next[0] != default_next:
++                chain.append(elt.next[0])
++                elt = elt.next[0]
++            # Now remove the blocks in the chain from the current
++            # block list, so that they can be re-inserted.
++            l = []
++            for b in chain:
++                assert index[b] > i
++                l.append((index[b], b))
++            l.sort()
++            l.reverse()
++            for j, b in l:
++                del blocks[index[b]]
++            # Insert the chain in the proper location
++            blocks[i:i + 1] = [cur] + chain
++            # Finally, re-compute the block indexes
++            for i in range(len(blocks)):
++                index[blocks[i]] = i
++
++    def fixupOrderForward(self, blocks, default_next):
++        """Make sure all JUMP_FORWARDs jump forward"""
++        index = {}
++        chains = []
++        cur = []
++        for b in blocks:
++            index[b] = len(chains)
++            cur.append(b)
++            if b.next and b.next[0] == default_next:
++                chains.append(cur)
++                cur = []
++        chains.append(cur)
++
++        while 1:
++            constraints = []
++
++            for i in range(len(chains)):
++                l = chains[i]
++                for b in l:
++                    for c in b.get_children():
++                        if index[c] < i:
++                            forward_p = 0
++                            for inst in b.insts:
++                                if inst[0] == 'JUMP_FORWARD':
++                                    if inst[1] == c:
++                                        forward_p = 1
++                            if not forward_p:
++                                continue
++                            constraints.append((index[c], i))
++
++            if not constraints:
++                break
++
++            # XXX just do one for now
++            # do swaps to get things in the right order
++            goes_before, a_chain = constraints[0]
++            assert a_chain > goes_before
++            c = chains[a_chain]
++            chains.remove(c)
++            chains.insert(goes_before, c)
++
++        del blocks[:]
++        for c in chains:
++            for b in c:
++                blocks.append(b)
++
+     def getBlocks(self):
+         return self.blocks.elements()
+ 
+@@ -94,84 +205,27 @@ class FlowGraph:
+             l.extend(b.getContainedGraphs())
+         return l
+ 
+-
+-def order_blocks(start_block, exit_block):
+-    """Order blocks so that they are emitted in the right order"""
+-    # Rules:
+-    # - when a block has a next block, the next block must be emitted just after
+-    # - when a block has followers (relative jumps), it must be emitted before
+-    #   them
+-    # - all reachable blocks must be emitted
++def dfs_postorder(b, seen):
++    """Depth-first search of tree rooted at b, return in postorder"""
+     order = []
+-
+-    # Find all the blocks to be emitted.
+-    remaining = set()
+-    todo = [start_block]
+-    while todo:
+-        b = todo.pop()
+-        if b in remaining:
+-            continue
+-        remaining.add(b)
+-        for c in b.get_children():
+-            if c not in remaining:
+-                todo.append(c)
+-
+-    # A block is dominated by another block if that block must be emitted
+-    # before it.
+-    dominators = {}
+-    for b in remaining:
+-        if __debug__ and b.next:
+-            assert b is b.next[0].prev[0], (b, b.next)
+-        # Make sure every block appears in dominators, even if no
+-        # other block must precede it.
+-        dominators.setdefault(b, set())
+-        # preceding blocks dominate following blocks
+-        for c in b.get_followers():
+-            while 1:
+-                dominators.setdefault(c, set()).add(b)
+-                # Any block that has a next pointer leading to c is also
+-                # dominated because the whole chain will be emitted at once.
+-                # Walk backwards and add them all.
+-                if c.prev and c.prev[0] is not b:
+-                    c = c.prev[0]
+-                else:
+-                    break
+-
+-    def find_next():
+-        # Find a block that can be emitted next.
+-        for b in remaining:
+-            for c in dominators[b]:
+-                if c in remaining:
+-                    break # can't emit yet, dominated by a remaining block
+-            else:
+-                return b
+-        assert 0, 'circular dependency, cannot find next block'
+-
+-    b = start_block
+-    while 1:
+-        order.append(b)
+-        remaining.discard(b)
+-        if b.next:
+-            b = b.next[0]
++    seen[b] = b
++    for c in b.get_children():
++        if c in seen:
+             continue
+-        elif b is not exit_block and not b.has_unconditional_transfer():
+-            order.append(exit_block)
+-        if not remaining:
+-            break
+-        b = find_next()
++        order = order + dfs_postorder(c, seen)
++    order.append(b)
+     return order
+ 
+-
+ class Block:
+     _count = 0
+ 
+     def __init__(self, label=''):
+         self.insts = []
+-        self.outEdges = set()
++        self.inEdges = misc.Set()
++        self.outEdges = misc.Set()
+         self.label = label
+         self.bid = Block._count
+         self.next = []
+-        self.prev = []
+         Block._count = Block._count + 1
+ 
+     def __repr__(self):
+@@ -187,46 +241,51 @@ class Block:
+ 
+     def emit(self, inst):
+         op = inst[0]
++        if op[:4] == 'JUMP':
++            self.outEdges.add(inst[1])
+         self.insts.append(inst)
+ 
+     def getInstructions(self):
+         return self.insts
+ 
++    def addInEdge(self, block):
++        self.inEdges.add(block)
++
+     def addOutEdge(self, block):
+         self.outEdges.add(block)
+ 
+     def addNext(self, block):
+         self.next.append(block)
+         assert len(self.next) == 1, map(str, self.next)
+-        block.prev.append(self)
+-        assert len(block.prev) == 1, map(str, block.prev)
+ 
+-    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
+-                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP',
+-                        )
++    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE',
++                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
+ 
+-    def has_unconditional_transfer(self):
+-        """Returns True if there is an unconditional transfer to an other block
+-        at the end of this block. This means there is no risk for the bytecode
+-        executer to go past this block's bytecode."""
++    def pruneNext(self):
++        """Remove bogus edge for unconditional transfers
++
++        Each block has a next edge that accounts for implicit control
++        transfers, e.g. from a JUMP_IF_FALSE to the block that will be
++        executed if the test is true.
++
++        These edges must remain for the current assembler code to
++        work. If they are removed, the dfs_postorder gets things in
++        weird orders.  However, they shouldn't be there for other
++        purposes, e.g. conversion to SSA form.  This method will
++        remove the next edge when it follows an unconditional control
++        transfer.
++        """
+         try:
+             op, arg = self.insts[-1]
+         except (IndexError, ValueError):
+             return
+-        return op in self._uncond_transfer
++        if op in self._uncond_transfer:
++            self.next = []
+ 
+     def get_children(self):
+-        return list(self.outEdges) + self.next
+-
+-    def get_followers(self):
+-        """Get the whole list of followers, including the next block."""
+-        followers = set(self.next)
+-        # Blocks that must be emitted *after* this one, because of
+-        # bytecode offsets (e.g. relative jumps) pointing to them.
+-        for inst in self.insts:
+-            if inst[0] in PyFlowGraph.hasjrel:
+-                followers.add(inst[1])
+-        return followers
++        if self.next and self.next[0] in self.outEdges:
++            self.outEdges.remove(self.next[0])
++        return self.outEdges.elements() + self.next
+ 
+     def getContainedGraphs(self):
+         """Return all graphs contained within this block.
+@@ -387,18 +446,18 @@ class PyFlowGraph(FlowGraph):
+             elif inst[0] != "SET_LINENO":
+                 pc = pc + 3
+             opname = inst[0]
+-            if opname in self.hasjrel:
++            if self.hasjrel.has_elt(opname):
+                 oparg = inst[1]
+                 offset = begin[oparg] - pc
+                 insts[i] = opname, offset
+-            elif opname in self.hasjabs:
++            elif self.hasjabs.has_elt(opname):
+                 insts[i] = opname, begin[inst[1]]
+         self.stage = FLAT
+ 
+-    hasjrel = set()
++    hasjrel = misc.Set()
+     for i in dis.hasjrel:
+         hasjrel.add(dis.opname[i])
+-    hasjabs = set()
++    hasjabs = misc.Set()
+     for i in dis.hasjabs:
+         hasjabs.add(dis.opname[i])
+ 
+diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
+index 6515945..3f641ba 100644
+--- a/Lib/compiler/pycodegen.py
++++ b/Lib/compiler/pycodegen.py
+@@ -706,7 +706,7 @@ class CodeGenerator:
+             self.startBlock(anchor)
+             self.emit('POP_BLOCK')
+             self.setups.pop()
+-            self.nextBlock(end)
++            self.startBlock(end)
+ 
+         self.emit('LOAD_CONST', None)
+ 
+-- 
+2.2.0.rc0.207.ga3a616c
+

component/python-2.7/tls_sni.patch

-Description: Support TLS SNI extension in ssl module
-Author: markk
-Bug-Python: http://bugs.python.org/issue5639
-
---- a/Lib/ssl.py
-+++ b/Lib/ssl.py
-@@ -202,6 +202,7 @@
-     def __init__(self, sock, keyfile=None, certfile=None,
-                  server_side=False, cert_reqs=CERT_NONE,
-                  ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-+                 server_hostname=None,
-                  do_handshake_on_connect=True,
-                  suppress_ragged_eofs=True, ciphers=None):
-         # Can't use sock.type as other flags (such as SOCK_NONBLOCK) get
-@@ -238,6 +239,7 @@
-             self._sslobj = _ssl.sslwrap(self._sock, server_side,
-                                         keyfile, certfile,
-                                         cert_reqs, ssl_version, ca_certs,
-+                                        server_hostname,
-                                         ciphers)
-             if do_handshake_on_connect:
-                 self.do_handshake()
-@@ -246,6 +248,7 @@
-         self.cert_reqs = cert_reqs
-         self.ssl_version = ssl_version
-         self.ca_certs = ca_certs
-+        self.server_hostname = server_hostname
-         self.ciphers = ciphers
-         self.do_handshake_on_connect = do_handshake_on_connect
-         self.suppress_ragged_eofs = suppress_ragged_eofs
-@@ -411,7 +414,7 @@
-             raise ValueError("attempt to connect already-connected SSLSocket!")
-         self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile,
-                                     self.cert_reqs, self.ssl_version,
--                                    self.ca_certs, self.ciphers)
-+                                    self.ca_certs, self.server_hostname, self.ciphers)
-         try:
-             if return_errno:
-                 rc = socket.connect_ex(self, addr)
-@@ -452,6 +455,7 @@
-                               cert_reqs=self.cert_reqs,
-                               ssl_version=self.ssl_version,
-                               ca_certs=self.ca_certs,
-+                              server_hostname=None,
-                               ciphers=self.ciphers,
-                               do_handshake_on_connect=self.do_handshake_on_connect,
-                               suppress_ragged_eofs=self.suppress_ragged_eofs),
-@@ -566,7 +570,7 @@
-         sock = sock._sock
- 
-     ssl_sock = _ssl.sslwrap(sock, 0, keyfile, certfile, CERT_NONE,
--                            PROTOCOL_SSLv23, None)
-+                            PROTOCOL_SSLv23, None, None, None)
-     try:
-         sock.getpeername()
-     except socket_error:
---- a/Modules/_ssl.c
-+++ b/Modules/_ssl.c
-@@ -267,7 +267,7 @@
-                enum py_ssl_server_or_client socket_type,
-                enum py_ssl_cert_requirements certreq,
-                enum py_ssl_version proto_version,
--               char *cacerts_file, char *ciphers)
-+               char *cacerts_file, char *server_hostname, char *ciphers)
- {
-     PySSLObject *self;
-     char *errstr = NULL;
-@@ -389,6 +389,14 @@
- 
-     PySSL_BEGIN_ALLOW_THREADS
-     self->ssl = SSL_new(self->ctx); /* New ssl struct */
-+#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
-+    /* If SNI isn't supported, we just don't call it and fail silently,
-+     * as there's not much else we can do.
-+     */
-+    if ((socket_type == PY_SSL_CLIENT) &&
-+             (proto_version != PY_SSL_VERSION_SSL2) && server_hostname)
-+        SSL_set_tlsext_host_name(self->ssl, server_hostname);
-+#endif
-     PySSL_END_ALLOW_THREADS
-     SSL_set_fd(self->ssl, Sock->sock_fd);       /* Set the socket for SSL */
- #ifdef SSL_MODE_AUTO_RETRY
-@@ -431,15 +439,16 @@
-     char *key_file = NULL;
-     char *cert_file = NULL;
-     char *cacerts_file = NULL;
-+    char *server_hostname = NULL;
-     char *ciphers = NULL;
- 
--    if (!PyArg_ParseTuple(args, "O!i|zziizz:sslwrap",
-+    if (!PyArg_ParseTuple(args, "O!i|zziizzz:sslwrap",
-                           PySocketModule.Sock_Type,
-                           &Sock,
-                           &server_side,
-                           &key_file, &cert_file,
-                           &verification_mode, &protocol,
--                          &cacerts_file, &ciphers))
-+                          &cacerts_file, &server_hostname, &ciphers))
-         return NULL;
- 
-     /*
-@@ -452,13 +461,13 @@
- 
-     return (PyObject *) newPySSLObject(Sock, key_file, cert_file,
-                                        server_side, verification_mode,
--                                       protocol, cacerts_file,
-+                                       protocol, cacerts_file, server_hostname,
-                                        ciphers);
- }
- 
- PyDoc_STRVAR(ssl_doc,
- "sslwrap(socket, server_side, [keyfile, certfile, certs_mode, protocol,\n"
--"                              cacertsfile, ciphers]) -> sslobject");
-+"                              cacertsfile, ciphers, server_hostname]) -> sslobject");
- 
- /* SSL object methods */
- 

component/python-2.7/tls_sni_httplib.patch

-Author: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
-Description: Enable TLS SNI support for httplib
-
---- a/Lib/httplib.py	2014-07-31 14:50:21.178088529 +0900
-+++ b/Lib/httplib.py	2014-07-31 20:11:09.279081382 +0900
-@@ -1195,7 +1195,12 @@
-             if self._tunnel_host:
-                 self.sock = sock
-                 self._tunnel()
--            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
-+                server_hostname = self._tunnel_host
-+            else:
-+                server_hostname = self.host
-+
-+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
-+                                        server_hostname=server_hostname)
- 
-     __all__.append("HTTPSConnection")
- 
---- a/Lib/ssl.py	2014-07-31 19:33:21.911968158 +0900
-+++ b/Lib/ssl.py	2014-07-31 19:33:57.428391985 +0900
-@@ -481,14 +481,15 @@
-                 server_side=False, cert_reqs=CERT_NONE,
-                 ssl_version=PROTOCOL_SSLv23, ca_certs=None,
-                 do_handshake_on_connect=True,
--                suppress_ragged_eofs=True, ciphers=None):
-+                suppress_ragged_eofs=True, ciphers=None,
-+                server_hostname=None):
- 
-     return SSLSocket(sock, keyfile=keyfile, certfile=certfile,
-                      server_side=server_side, cert_reqs=cert_reqs,
-                      ssl_version=ssl_version, ca_certs=ca_certs,
-                      do_handshake_on_connect=do_handshake_on_connect,
-                      suppress_ragged_eofs=suppress_ragged_eofs,
--                     ciphers=ciphers)
-+                     ciphers=ciphers, server_hostname=server_hostname)
- 
- 
- # some utility functions

component/python-cliff/buildout.cfg

+[buildout]
+
+[python-cliff]
+recipe = zc.recipe.egg:custom
+egg = cliff
+setup-eggs = ${python-cliff-prep:eggs}
+
+[python-cliff-prep]
+recipe = zc.recipe.egg
+eggs =
+  pbr
+

component/python-cryptography/buildout.cfg

@@ -22,5 +22,6 @@ CPATH = ${openssl:location}/include
 recipe = zc.recipe.egg
 eggs =
   ${python-cffi:egg}
+  enum34
   pycparser
   six

component/python-kerberos/buildout.cfg

+[buildout]
+extends =
+#  ../openssl/buildout.cfg
+  ../kerberos/buildout.cfg
+#  ../pkgconfig/buildout.cfg
+
+parts =
+  python-kerberos
+
+[python-kerberos]
+recipe = zc.recipe.egg:custom
+egg = kerberos
+environment = python-kerberos-env
+#setup-eggs = ${python-cryptography-prep:eggs}
+
+[python-kerberos-env]
+PATH = ${kerberos:location}/bin:%(PATH)s
+#PKG_CONFIG_PATH = ${openssl:location}/lib/pkgconfig
+LD_LIBRARY_PATH = ${kerberos:location}/lib
+CPATH = ${kerberos:location}/include
+
+[python-cryptography-prep]
+recipe = zc.recipe.egg
+eggs =
+  ${python-cffi:egg}
+  enum34
+  pycparser
+  six

component/qemu-kvm/buildout.cfg

@@ -15,10 +15,10 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
-md5sum = 3a897d722457c5a895cd6ac79a28fda0
+url = http://wiki.qemu-project.org/download/qemu-2.2.1.tar.bz2
+md5sum = 833ff4457062456d38d6567f802ffef4
 configure-options =
-  --target-list=x86_64-softmmu
+  --target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
   --enable-system
   --with-system-pixman
   --disable-sdl
@@ -40,25 +40,12 @@ environment =
   PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
   LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
 
-# The following is only available in buildout2, which we don't use yet.
-[kvm-bits64]
-configure-options =
-  --target-list=x86_64-softmmu
-  ${kvm:configure-options}
-
-[kvm-bits32]
-configure-options =
-  --target-list=i386-softmmu
-  ${kvm:configure-options}
-
-
 [debian-amd64-netinst.iso]
-# Download the installer of Debian 7 (Wheezy)
+# Download the installer of Debian 8 (Jessie)
 recipe = hexagonit.recipe.download
-url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso 
+url = http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso
 filename = ${:_buildout_section_name_}
-md5sum = b86774fe4de88be6378ba3d71b8029bd 
+md5sum = 1a311f9afb68d6365211b13b4342c40b
 download-only = true
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
-

component/r-language/buildout.cfg

+[buildout]
+extends =
+  ../bzip2/buildout.cfg
+  ../gcc/buildout.cfg
+  ../pcre/buildout.cfg
+  ../readline/buildout.cfg
+  ../xz-utils/buildout.cfg
+  ../zlib/buildout.cfg
+
+parts =
+  gcc-fortran
+  r-language
+
+[r-language]
+recipe = slapos.recipe.cmmi
+url = http://cran.univ-paris1.fr/src/base/R-3/R-3.2.0.tar.gz
+md5sum = 66fa17ad457d7e618191aa0f52fc402e
+configure-options =
+  --enable-R-shlib
+  --with-readline=yes
+  --without-tcltk
+  --without-cairo
+  --without-libpng
+  --without-jpeglib
+  --without-libtiff
+  --with-system-zlib
+  --with-system-bzlib
+  --with-system-pcre
+  --with-system-xz
+  --without-ICU
+  --without-x
+environment =
+  PATH=${gcc-fortran:location}/bin:%(PATH)s
+  CPPFLAGS=-I${bzip2:location}/include  -I${pcre:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${xz-utils:location}/include -I${zlib:location}/include
+  LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gcc-fortran:location}/lib -Wl,-rpath=${gcc-fortran:location}/lib -L${gcc-fortran:location}/lib64 -Wl,-rpath=${gcc-fortran:location}/lib64 -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/r-language/software.cfg

+# A minimal software release to test the r-language component (depends on gfortran).
+
+[buildout]
+extends = 
+  ../../stack/slapos.cfg
+  ../../component/gcc/buildout.cfg
+  ../../component/r-language/buildout.cfg
+
+parts =
+  eggs
+  gcc-fortran
+  r-language
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.cookbook
+

component/ragel/buildout.cfg

@@ -8,7 +8,7 @@ parts =
 
 [ragel]
 recipe = slapos.recipe.cmmi
-url = http://www.complang.org/ragel/ragel-6.8.tar.gz
+url = http://www.colm.net/files/ragel/ragel-6.8.tar.gz
 md5sum = 1bb39745ac23da449019f9f2cb4b0d01
 configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

component/re6stnet/buildout.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../git/buildout.cfg
+  ../babeld/buildout.cfg
+  ../openvpn/buildout.cfg
+
+develop = 
+  re6stnet-repository
+
+parts = 
+  babeld
+  re6stnet-develop
+  re6stnet
+
+[re6stnet-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/re6stnet.git
+branch = master
+git-executable = ${git:location}/bin/git
+
+[re6stnet-develop]
+recipe = zc.recipe.egg:develop
+setup = ${re6stnet-hack:dir}
+
+[re6stnet-hack]
+recipe = plone.recipe.command
+stop-on-error = true
+dir = ${re6stnet-repository:location}
+command =
+  rm -f "${:dir}/re6stconf.py" && ln -s re6st-conf "${:dir}/re6stconf.py"
+  rm -f "${:dir}/re6stregistry.py" && ln -s re6st-registry "${:dir}/re6stregistry.py"
+  rm -f "${:dir}/re6stnet.py" && ln -s re6stnet "${:dir}/re6stnet.py"
+  sed -i 's#("git",)#("${git:location}/bin/git",)#' ${:dir}/re6st/version.py
+
+update-command = ${:command}
+
+[environment]
+# Note: For now original PATH is appended to the end, as not all tools are
+#       provided by SlapOS
+PATH=${openvpn:location}/sbin:${babeld:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin::${buildout:bin-directory}:${xz-utils:location}/bin:$PATH
+
+[re6stnet]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  pyOpenSSL
+  re6stnet
+
+script =
+  re6stnet
+  re6st-conf
+  re6st-registry
+
+initialization =
+  import os
+  os.environ['PATH'] = os.path.expandvars('${environment:PATH}')
+
+entry-points =
+  re6stnet=re6stnet:main
+  re6st-conf=re6stconf:main
+  re6st-registry=re6stregistry:main
+

component/readline/buildout.cfg

 [buildout]
+extends =
+  ../ncurses/buildout.cfg
+
 parts =
   readline5
   readline
@@ -40,6 +43,10 @@ patches =
   http://ftp.gnu.org/gnu/readline/readline-6.3-patches/readline63-006#71dc6ecce66d1489b96595f55d142a52
   http://ftp.gnu.org/gnu/readline/readline-6.3-patches/readline63-007#062a08ed60679d3c4878710b3d595b65
   http://ftp.gnu.org/gnu/readline/readline-6.3-patches/readline63-008#ee1c04072154826870848d8b218d7b04
+  ${:_profile_base_location_}/configure-ncurses.diff#c2bf0d0cb0ccd3d3c078ba124fffb7d2
 configure-options =
   --enable-multibyte
   --disable-static
+environment =
+  CPPFLAGS=-I${ncurses:location}/include/ncurses
+  LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib

component/readline/configure-ncurses.diff

+diff -ur ../readline-6.3.orig/aclocal.m4 ./aclocal.m4
+--- ../readline-6.3.orig/aclocal.m4	2013-10-21 00:37:50.000000000 +0200
++++ ./aclocal.m4	2015-05-21 11:04:20.243591527 +0200
+@@ -958,10 +958,10 @@
+ fi
+ AC_CACHE_VAL(bash_cv_termcap_lib,
+ [AC_CHECK_FUNC(tgetent, bash_cv_termcap_lib=libc,
+-  [AC_CHECK_LIB(termcap, tgetent, bash_cv_termcap_lib=libtermcap,
+-    [AC_CHECK_LIB(tinfo, tgetent, bash_cv_termcap_lib=libtinfo,
++  [AC_CHECK_LIB(ncurses, tgetent, bash_cv_termcap_lib=libncurses,
++    [AC_CHECK_LIB(termcap, tgetent, bash_cv_termcap_lib=libtermcap,
++      [AC_CHECK_LIB(tinfo, tgetent, bash_cv_termcap_lib=libtinfo,
+         [AC_CHECK_LIB(curses, tgetent, bash_cv_termcap_lib=libcurses,
+-	    [AC_CHECK_LIB(ncurses, tgetent, bash_cv_termcap_lib=libncurses,
+ 	        bash_cv_termcap_lib=gnutermcap)])])])])])
+ if test "X$_bash_needmsg" = "Xyes"; then
+ AC_MSG_CHECKING(which library has the termcap functions)
+Only in ./: aclocal.m4~
+Only in ./: autom4te.cache
+diff -ur ../readline-6.3.orig/configure ./configure
+--- ../readline-6.3.orig/configure	2013-03-13 15:14:53.000000000 +0100
++++ ./configure	2015-05-21 11:29:48.157514191 +0200
+@@ -5793,6 +5793,45 @@
+ if test "x$ac_cv_func_tgetent" = xyes; then :
+   bash_cv_termcap_lib=libc
+ else
++  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lncurses" >&5
++$as_echo_n "checking for tgetent in -lncurses... " >&6; }
++if ${ac_cv_lib_ncurses_tgetent+:} false; then :
++  $as_echo_n "(cached) " >&6
++else
++  ac_check_lib_save_LIBS=$LIBS
++LIBS="-lncurses  $LIBS"
++cat confdefs.h - <<_ACEOF >conftest.$ac_ext
++/* end confdefs.h.  */
++
++/* Override any GCC internal prototype to avoid an error.
++   Use char because int might match the return type of a GCC
++   builtin and then its argument prototype would still apply.  */
++#ifdef __cplusplus
++extern "C"
++#endif
++char tgetent ();
++int
++main ()
++{
++return tgetent ();
++  ;
++  return 0;
++}
++_ACEOF
++if ac_fn_c_try_link "$LINENO"; then :
++  ac_cv_lib_ncurses_tgetent=yes
++else
++  ac_cv_lib_ncurses_tgetent=no
++fi
++rm -f core conftest.err conftest.$ac_objext \
++    conftest$ac_exeext conftest.$ac_ext
++LIBS=$ac_check_lib_save_LIBS
++fi
++{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_tgetent" >&5
++$as_echo "$ac_cv_lib_ncurses_tgetent" >&6; }
++if test "x$ac_cv_lib_ncurses_tgetent" = xyes; then :
++  bash_cv_termcap_lib=libncurses
++else
+   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for tgetent in -ltermcap" >&5
+ $as_echo_n "checking for tgetent in -ltermcap... " >&6; }
+ if ${ac_cv_lib_termcap_tgetent+:} false; then :
+@@ -5910,45 +5949,6 @@
+ if test "x$ac_cv_lib_curses_tgetent" = xyes; then :
+   bash_cv_termcap_lib=libcurses
+ else
+-  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lncurses" >&5
+-$as_echo_n "checking for tgetent in -lncurses... " >&6; }
+-if ${ac_cv_lib_ncurses_tgetent+:} false; then :
+-  $as_echo_n "(cached) " >&6
+-else
+-  ac_check_lib_save_LIBS=$LIBS
+-LIBS="-lncurses  $LIBS"
+-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+-/* end confdefs.h.  */
+-
+-/* Override any GCC internal prototype to avoid an error.
+-   Use char because int might match the return type of a GCC
+-   builtin and then its argument prototype would still apply.  */
+-#ifdef __cplusplus
+-extern "C"
+-#endif
+-char tgetent ();
+-int
+-main ()
+-{
+-return tgetent ();
+-  ;
+-  return 0;
+-}
+-_ACEOF
+-if ac_fn_c_try_link "$LINENO"; then :
+-  ac_cv_lib_ncurses_tgetent=yes
+-else
+-  ac_cv_lib_ncurses_tgetent=no
+-fi
+-rm -f core conftest.err conftest.$ac_objext \
+-    conftest$ac_exeext conftest.$ac_ext
+-LIBS=$ac_check_lib_save_LIBS
+-fi
+-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_tgetent" >&5
+-$as_echo "$ac_cv_lib_ncurses_tgetent" >&6; }
+-if test "x$ac_cv_lib_ncurses_tgetent" = xyes; then :
+-  bash_cv_termcap_lib=libncurses
+-else
+   bash_cv_termcap_lib=gnutermcap
+ fi
+ 
+@@ -6322,10 +6322,7 @@
+   $as_echo_n "(cached) " >&6
+ else
+   if test "$cross_compiling" = yes; then :
+-  { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+-as_fn_error $? "cannot run test program while cross compiling
+-See \`config.log' for more details" "$LINENO" 5; }
++  bash_cv_wcwidth_broken=no
+ else
+   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ /* end confdefs.h.  */
+@@ -6409,6 +6406,7 @@
+ #	*curses*|*termcap*|*termlib*)	;;
+ #	*)			SHLIB_LIBS="$SHLIB_LIBS $TERMCAP_LIB" ;;
+ #	esac
++        SHLIB_LIBS="$SHLIB_LIBS $TERMCAP_LIB"
+ 
+ 
+ 
+diff -ur ../readline-6.3.orig/configure.ac ./configure.ac
+--- ../readline-6.3.orig/configure.ac	2014-02-11 23:12:39.000000000 +0100
++++ ./configure.ac	2015-05-21 11:25:26.164610769 +0200
+@@ -222,6 +222,7 @@
+ #	*curses*|*termcap*|*termlib*)	;;
+ #	*)			SHLIB_LIBS="$SHLIB_LIBS $TERMCAP_LIB" ;;
+ #	esac
++        SHLIB_LIBS="$SHLIB_LIBS $TERMCAP_LIB"
+ 	
+         AC_SUBST(SHOBJ_CC)
+         AC_SUBST(SHOBJ_CFLAGS)
+Only in ./: configure.ac~

component/rpy2/buildout.cfg

+[buildout]
+extends =
+  ../gcc/buildout.cfg
+  ../pcre/buildout.cfg
+  ../r-language/buildout.cfg
+  ../xz-utils/buildout.cfg
+
+# rpy2 needs R in $PATH
+# to use rpy2 you need a wrapper that sets ${r-language:location}/bin in PATH
+# see ../manpy/buildout.cfg as an example
+
+[rpy2_env]
+PATH = ${gcc-fortran:location}/bin:${r-language:location}/bin:%(PATH)s
+
+[rpy2]
+recipe = zc.recipe.egg:custom
+environment = rpy2_env
+egg = rpy2
+initialization =
+  # rpy2 needs R in $PATH
+  import os
+  os.environ['PATH'] = '${r-language:location}/bin' + os.pathsep + os.environ.get('PATH', '')
+rpath =
+  ${gcc-fortran:location}/lib
+  ${gcc-fortran:location}/lib64
+  ${pcre:location}/lib
+  ${readline:location}/lib
+  ${xz-utils:location}/lib
+  ${r-language:location}/lib/R/lib
+library-dirs =
+  ${pcre:location}/lib
+  ${readline:location}/lib
+  ${xz-utils:location}/lib
+include-dirs =
+  ${readline:location}/include

component/rsync/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [rsync]
 recipe = slapos.recipe.cmmi
-url = http://rsync.samba.org/ftp/rsync/src/rsync-3.0.9.tar.gz
-md5sum = 5ee72266fe2c1822333c407e1761b92b
+url = http://rsync.samba.org/ftp/rsync/src/rsync-3.1.1.tar.gz
+md5sum = 43bd6676f0b404326eee2d63be3cdcfe
 make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}

component/ruby/buildout.cfg

@@ -2,32 +2,39 @@
 
 extends =
   ../gdbm/buildout.cfg
-  ../openssl/buildout.cfg
-  ../zlib/buildout.cfg
-  ../readline/buildout.cfg
-  ../ncurses/buildout.cfg
+  ../libffi/buildout.cfg
   ../libyaml/buildout.cfg
-  ../openssl/buildout.cfg
   ../ncurses/buildout.cfg
+  ../openssl/buildout.cfg
   ../readline/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = ruby
 
-[ruby-1.9.3]
+[ruby-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.3-p194.tar.gz
-md5sum = bc0c715c69da4d1d8bd57069c19f6c0e
 configure-options =
   --enable-shared --enable-rpath --disable-install-doc
-  --enable-pthread --enable-ipv6
-  --with-ext=bigdecimal,continuation,curses,date,digest,dl,etc,fcntl,iconv,io,json,nkf,openssl,pathname,psych,pty,racc,readline,ripper,sdbm,socket,stringio,strscan,syck,syslog,zlib
+  --with-out-ext=tk,tk/tkutil,win32,win32ole
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libyaml:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw  -I${gdbm:location}/include -I${openssl:location}/include
-  LDFLAGS=-L${zlib:location}/lib -L${libyaml:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib  -L${gdbm:location}/lib -L${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${libyaml:location}/lib   -Wl,-rpath=${ncurses:location}/lib  -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib
+  PATH=${xz-utils:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gdbm:location}/include -I${libffi:location}/include -I${libyaml:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${openssl:location}/include -I${readline:location}/include -I${zlib:location}/include
+  LDFLAGS=-L${gdbm:location}/lib -L${libffi:location}/lib -L${libyaml:location}/lib -L${ncurses:location}/lib -L${openssl:location}/lib -L${readline:location}/lib -L${zlib:location}/lib -Wl,-rpath=${gdbm:location}/lib  -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${libyaml:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${zlib:location}/lib
   PKG_CONFIG_PATH=${libyaml:location}/lib/
 
-[ruby-1.9]
-<= ruby-1.9.3
+
+[ruby2.1]
+<= ruby-common
+url = http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.6.tar.xz
+md5sum = ec6f10ca331ce947802ede86259513a8
+
+
+[ruby2.2]
+<= ruby-common
+url = http://ftp.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.xz
+md5sum = dbce9b9d79d90f213ba8d448b0b6ed86
+
 
 [ruby]
+<= ruby2.2

component/scikit-learn/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+  ../scipy/buildout.cfg
+
+parts =
+  scikit-learn
+
+[scikit-learn-env]
+<= numpy-env
+
+[scikit-learn]
+recipe = zc.recipe.egg:custom
+egg = scikit-learn
+environment = scikit-learn-env
+setup-eggs = ${numpy:egg}

component/scipy/buildout.cfg

+[buildout]
+extends =
+  ../numpy/buildout.cfg
+
+parts =
+  scipy
+
+[scipy-env]
+<= numpy-env
+
+[scipy]
+recipe = zc.recipe.egg:custom
+egg = scipy
+environment = scipy-env
+setup-eggs = ${numpy:egg}
+rpath = ${numpy:rpath}

component/slapos/buildout.cfg

@@ -114,7 +114,3 @@ eggs =
     ${slapos:eggs}
 interpreter = py
 scripts = py
-
-[versions]
-collective.recipe.template = 1.11
-z3c.recipe.scripts = 1.0.1

component/sqlite3/buildout.cfg

@@ -5,11 +5,14 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2014/sqlite-autoconf-3080701.tar.gz
-md5sum = 8ee4541ebb3e5739e7ef5e9046e30063
+url = http://sqlite.org/2015/sqlite-autoconf-3081002.tar.gz
+md5sum = a18bfc015cd49a1e7a961b7b77bc3b37
 configure-options =
   --disable-static
   --enable-readline
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/dynamic_link.patch#328d3773b49151207d29a5a857f98337
 environment =
   CPPFLAGS=-I${readline:location}/include -I${ncurses:location}/include
   LDFLAGS=-L${buildout:parts-directory}/${:_buildout_section_name_} -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib

component/sqlite3/dynamic_link.patch

+diff -ur sqlite-autoconf-3081002/Makefile.am sqlite-autoconf-3081002/Makefile.am
+--- sqlite-autoconf-3081002/Makefile.am 2015-04-08 16:38:50.000000000 +0200
++++ sqlite-autoconf-3081002/Makefile.am 2015-05-09 14:23:20.000000000 +0200
+@@ -7,7 +7,8 @@
+
+ bin_PROGRAMS = sqlite3
+ sqlite3_SOURCES = shell.c sqlite3.h
+-sqlite3_LDADD = sqlite3.$(OBJEXT) @READLINE_LIBS@
++sqlite3_LDADD = $(top_builddir)/libsqlite3.la @READLINE_LIBS@
++sqlite3_DEPENDENCIES = $(top_builddir)/libsqlite3.la
+
+ include_HEADERS = sqlite3.h sqlite3ext.h
+
+diff -ur sqlite-autoconf-3081002/Makefile.in sqlite-autoconf-3081002/Makefile.in
+--- sqlite-autoconf-3081002/Makefile.in 2015-04-08 16:38:52.000000000 +0200
++++ sqlite-autoconf-3081002/Makefile.in 2015-05-09 14:23:23.000000000 +0200
+@@ -109,7 +109,6 @@
+ PROGRAMS = $(bin_PROGRAMS)
+ am_sqlite3_OBJECTS = shell.$(OBJEXT)
+ sqlite3_OBJECTS = $(am_sqlite3_OBJECTS)
+-sqlite3_DEPENDENCIES = sqlite3.$(OBJEXT)
+ DEFAULT_INCLUDES = -I.@am__isrc@
+ depcomp = $(SHELL) $(top_srcdir)/depcomp
+ am__depfiles_maybe = depfiles
+@@ -274,7 +273,8 @@
+ libsqlite3_la_SOURCES = sqlite3.c
+ libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
+ sqlite3_SOURCES = shell.c sqlite3.h
+-sqlite3_LDADD = sqlite3.$(OBJEXT) @READLINE_LIBS@
++sqlite3_LDADD = $(top_builddir)/libsqlite3.la @READLINE_LIBS@
++sqlite3_DEPENDENCIES = $(top_builddir)/libsqlite3.la
+ include_HEADERS = sqlite3.h sqlite3ext.h
+ EXTRA_DIST = sqlite3.1 tea
+ pkgconfigdir = ${libdir}/pkgconfig

component/squid/buildout.cfg

@@ -6,13 +6,15 @@ parts =
   squid
 extends =
   ../pkgconfig/buildout.cfg
+  ../xz-utils/buildout.cfg
 
 [squid]
-recipe = hexagonit.recipe.cmmi
-url = http://www.squid-cache.org/Versions/v3/3.4/squid-3.4.7.tar.bz2
-md5sum = 9951034b10f7ee0f45a95cfae61c57c2
+recipe = slapos.recipe.cmmi
+url = http://www.squid-cache.org/Versions/v3/3.4/squid-3.4.13.tar.xz
+md5sum = a5f6c978b2d7a99b161c8275e1acb470
 configure-options =
   --disable-dependency-tracking
+  --disable-static
   --disable-translation
   --disable-htcp
   --disable-snmp
@@ -20,6 +22,7 @@ configure-options =
   --disable-icmp
   --disable-esi
   --disable-icap-client
+  --disable-ecap
   --disable-wccp
   --disable-wccpv2
   --disable-eui
@@ -38,4 +41,4 @@ configure-options =
   --disable-auth-ntlm
   --with-krb5-config=no
 Environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/stunnel/buildout.cfg

@@ -6,8 +6,8 @@ parts =
 
 [stunnel]
 recipe = slapos.recipe.cmmi
-url = https://www.stunnel.org/downloads/stunnel-5.02.tar.gz
-md5sum = bb48b1c18cfc0a42708ef996b1a26926
+url = https://www.stunnel.org/downloads/archive/5.x/stunnel-5.14.tar.gz
+md5sum = e716501960dc6856d80f92547298f724
 configure-options =
   --enable-ipv6
   --disable-libwrap

component/tesseract/buildout.cfg

 [buildout]
 extends =
+  ../autoconf/buildout.cfg
   ../automake/buildout.cfg
+  ../jbigkit/buildout.cfg
+  ../leptonica/buildout.cfg
   ../libpng/buildout.cfg
   ../libtool/buildout.cfg
-  ../leptonica/buildout.cfg
-  ../jbigkit/buildout.cfg
   ../webp/buildout.cfg
 
 parts =
@@ -20,9 +21,8 @@ patches =
   ${:_profile_base_location_}/tesseract-3.00-gcc-4.7-build.patch#ca80db3ec489c547b03f3ee48879c1b1
   ${:_profile_base_location_}/tesseract-3.01-remove-bom.patch#2e691858cb492b7c17d23bf0912b3d24
 configure-command =
-  aclocal -I ${libtool:location}/share/aclocal -I config
   libtoolize -f -c
-  libtoolize --automake
+  aclocal -I ${libtool:location}/share/aclocal -I config
   autoheader -f
   automake -c -a -f
   autoconf -Wno-portability
@@ -38,6 +38,8 @@ environment =
   LIBLEPT_HEADERSDIR=${leptonica:location}/include
   CPPFLAGS=-I${leptonica:location}/include
   LDFLAGS =-L${leptonica:location}/lib -Wl,-rpath=${leptonica:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+make-options =
+  LIBTOOL=${libtool:location}/bin/libtool
 
 [tesseract-eng-traineddata]
 recipe = hexagonit.recipe.download

component/w3-validator/buildout.cfg

@@ -35,6 +35,6 @@ depends =
   ${perl-XML-LibXML:location}
   ${perl-libwww-perl:location}
 perl-path = ${perl:location}/bin/perl
-# validator-1_2-release in http://dvcs.w3.org/hg/markup-validator/tags
-validator-url = http://dvcs.w3.org/hg/markup-validator/archive/754b7a64cdb4.tar.bz2
+# validator-1_3-release in https://dvcs.w3.org/hg/markup-validator/tags
+validator-url = https://dvcs.w3.org/hg/markup-validator/archive/6c6c7c1f69c5.tar.bz2
 dtd-url =

component/wendelin.core/buildout.cfg

+[buildout]
+extends = ../git/buildout.cfg
+parts =
+  wendelin.core
+
+
+# wendelin.core installed from released egg from pypi
+[wendelin.core]
+recipe = zc.recipe.egg:custom
+egg = wendelin.core
+
+
+# wendelin.core installed from latest git version
+[wendelin.core-dev]
+recipe = zc.recipe.egg:develop
+egg = wendelin.core
+setup = ${wendelin.core-repository-submoduleinit:location}
+environment = wendelin.core-dev-env
+
+[wendelin.core-dev-env]
+# wendelin.core-dev needs git to build
+PATH = ${git:location}/bin:%(PATH)s
+
+
+[wendelin.core-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.cn/nexedi/wendelin.core.git
+# dir is pretty name as top-level -dev recipe
+location = ${buildout:parts-directory}/wendelin.core-dev
+git-executable = ${git:location}/bin/git
+
+
+# TODO add `git clone --recursive` to slapos.recipe.build:gitclone
+#      and this way merge this -submoduleinit into -repository part
+[wendelin.core-repository-submoduleinit]
+recipe = plone.recipe.command
+command = cd "${wendelin.core-repository:location}" && ${git:location}/bin/git submodule update --init
+stop-on-error = true
+# propagate location of main repo
+location= ${wendelin.core-repository:location}

component/wget/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz
-md5sum = fe102975ab3a6c049777883f1bb9ad07
+url = http://ftp.gnu.org/gnu/wget/wget-1.16.3.tar.xz
+md5sum = d2e4455781a70140ae83b54ca594ce21
 configure-options =
   --enable-ipv6
   --enable-opie
@@ -24,10 +24,10 @@ patch-options =
   -p1
 
 patches =
-  ${:_profile_base_location_}/wget-doc.makefile.patch#aabcc1695c7fb65ac44f295e04a2db78
+  ${:_profile_base_location_}/wget-doc.makefile.patch#0d23cf1ee81268a94699aebbb26058e6
 
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
-  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${openssl:location}/lib
   CPPFLAGS=-I${zlib:location}/include -I${openssl:location}/include -I${pcre:location}/include

component/wget/wget-doc.makefile.patch

@@ -7,16 +7,16 @@
 -SUBDIRS = lib src doc po tests util testenv
 +SUBDIRS = lib src po tests util testenv
  
- EXTRA_DIST = ChangeLog.README MAILING-LIST \
-              msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
+ EXTRA_DIST = MAILING-LIST \
+              msdos/config.h msdos/Makefile.DJ \
 --- a/Makefile.in	2012-08-05 22:17:17.000000000 +0200
 +++ b/Makefile.in	2013-06-21 15:05:04.351269286 +0200
-@@ -1195,7 +1195,7 @@
+@@ -1322,7 +1322,7 @@
  ACLOCAL_AMFLAGS = -I m4
 
  # subdirectories in the distribution
 -SUBDIRS = lib src doc po tests util testenv
 +SUBDIRS = lib src po tests util testenv
- EXTRA_DIST = ChangeLog.README MAILING-LIST \
-              msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
+ EXTRA_DIST = MAILING-LIST \
+              msdos/config.h msdos/Makefile.DJ \
               msdos/Makefile.WC ABOUT-NLS \

component/xz-utils/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [xz-utils]
 recipe = slapos.recipe.cmmi
-url = http://tukaani.org/xz/xz-5.0.5.tar.bz2
-md5sum = db44efe0d53ac4317627624b98c63da0
+url = http://tukaani.org/xz/xz-5.2.0.tar.bz2
+md5sum = 867cc8611760240ebf3440bd6e170bb9
 configure-options =
   --disable-static

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.92'
+version = '0.102'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -87,6 +87,7 @@ setup(name=name,
           'check_page_content = slapos.recipe.check_page_content:Recipe',
           'check_port_listening = slapos.recipe.check_port_listening:Recipe',
           'check_url_available = slapos.recipe.check_url_available:Recipe',
+          'check_parameter = slapos.recipe.check_parameter:Recipe',
           'cloud9 = slapos.recipe.cloud9:Recipe',
           'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe',
           'condor = slapos.recipe.condor:Recipe',
@@ -174,6 +175,7 @@ setup(name=name,
           'request.serialised = slapos.recipe.request:Serialised',
           'request.edge = slapos.recipe.request:RequestEdge',
           'requestoptional = slapos.recipe.request:RequestOptional',
+          're6stnet.registry = slapos.recipe.re6stnet:Recipe',
           'reverseproxy.nginx = slapos.recipe.reverse_proxy_nginx:Recipe',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
@@ -181,6 +183,7 @@ setup(name=name,
           'shellinabox = slapos.recipe.shellinabox:Recipe',
           'signalwrapper= slapos.recipe.signal_wrapper:Recipe',
           'simplelogger = slapos.recipe.simplelogger:Recipe',
+          'simplehttpserver = slapos.recipe.simplehttpserver:Recipe',
           'siptester = slapos.recipe.siptester:SipTesterRecipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
@@ -217,6 +220,6 @@ setup(name=name,
         ],
       },
       test_suite='slapos.test',
-      test_requires=[ 'jsonschema' ],
+      tests_require=[ 'jsonschema' ],
     )
 

slapos/recipe/addresiliency/__init__.py

@@ -43,7 +43,7 @@ class Recipe(GenericSlapRecipe):
         slap_connection = self.buildout['slap-connection']
 
         takeover_wrapper = self.createPythonScript(
-            name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
+            name=self.options['wrapper-takeover'],
             absolute_function='slapos.recipe.addresiliency.takeover.run',
             arguments={
                 'server_url': slap_connection['server-url'],
@@ -53,6 +53,7 @@ class Recipe(GenericSlapRecipe):
                 'partition_id': slap_connection['partition-id'],
                 'software': slap_connection['software-release-url'],
                 'namebase': self.parameter_dict['namebase'],
+                'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
             })
 
         path_list.append(takeover_wrapper)

slapos/recipe/addresiliency/takeover.py

@@ -11,7 +11,8 @@ logging.basicConfig(level=logging.DEBUG)
 
 def takeover(server_url, key_file, cert_file, computer_guid,
              partition_id, software_release, namebase,
-             winner_instance_suffix = None):
+             winner_instance_suffix = None,
+             takeover_triggered_file_path=None):
   """
   This function does
 
@@ -74,6 +75,10 @@ def takeover(server_url, key_file, cert_file, computer_guid,
   # Note: Root instance will reconfigure itself the winning instance (software_type
   # and parameters.)
 
+  # Create "lock" file preventing equeue to run import scripts
+  # XXX hardcoded
+  open(takeover_triggered_file_path, 'w').write('')
+
 def run(args):
   slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
                                                 key_file = args.pop('key_file'),
@@ -81,5 +86,6 @@ def run(args):
                                                 computer_guid = args.pop('computer_id'),
                                                 partition_id = args.pop('partition_id'),
                                                 software_release = args.pop('software'),
-                                                namebase = args.pop('namebase'))
+                                                namebase = args.pop('namebase'),
+                                                takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
 

slapos/recipe/check_parameter/__init__.py

+# vim: set et sts=2:
+##############################################################################
+#
+# Copyright (c) 2015 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import sys
+
+class Recipe(GenericBaseRecipe):
+  """
+  Check listening port promise
+  """
+
+  def install(self):
+    config = dict(
+      value=self.options['value'],
+      python_path=sys.executable,
+    )
+
+    if self.options.get('expected-type') == "ipv6":
+      template = self.getTemplateFilename('check_ipv6.py.in')
+
+    elif self.options.get('expected-type') == "ipv4":
+      template = self.getTemplateFilename('check_ipv4.py.in')
+    else:
+      config["expected-value"] = self.options.get('expected-value')
+ 
+      config["expected-not-value"] = self.options.get('expected-not-value')
+
+      template = self.getTemplateFilename('check_parameter.py.in')
+
+    promise = self.createExecutable(
+      self.options['path'],
+      self.substituteTemplate(template, config))
+
+    return [promise]

slapos/recipe/check_parameter/template/check_ipv4.py.in

+#!%(python_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+import socket
+
+address = "%(value)s"
+
+try:
+  socket.inet_pton(socket.AF_INET, address)
+except AttributeError:  # no inet_pton here, sorry
+  try:
+    socket.inet_aton(address)
+  except socket.error:
+    sys.exit(127)
+  if address.count('.') != 3:
+    sys.exit(127)
+
+except socket.error:  # not a valid address
+   sys.exit(127)

slapos/recipe/check_parameter/template/check_ipv6.py.in

+#!%(python_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+import socket
+import sys
+
+address = "%(value)s"
+
+try:
+  socket.inet_pton(socket.AF_INET6, address)
+except socket.error:  # not a valid address
+  sys.exit(127)

slapos/recipe/check_parameter/template/check_parameter.py.in

+#!%(python_path)s
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+import socket
+import sys
+
+value = "%(value)s"
+expected = "%(expected-value)s"
+not_expected = "%(expected-not-value)s"
+
+if expected != "" and value != expected:
+  print "FAIL: %s != %s" % (value, expected)
+  sys.exit(127)
+
+if not_expected != "" and value == not_expected: 
+  print "FAIL: %s == %s" % (value, not_expected)
+  sys.exit(127)
+

slapos/recipe/dcron.py

@@ -76,10 +76,12 @@ class Part(GenericBaseRecipe):
 day_of_week_dict = dict((name, dow) for dow, name in enumerate(
     "sunday monday tuesday wednesday thursday friday saturday".split())
   for name in (name, name[:3]))
-symbolic_dict = dict(hourly  = '0 * * * *',
+symbolic_dict = dict(minutely = '* * * * *',
+                     hourly   = '0 * * * *',
                      daily    = '0 0 * * *',
+                     weekly   = '0 0 * * 0',
                      monthly  = '0 0 1 * *',
-                     weekly  = '0 0 * * 0')
+                     yearly   = '0 0 1 1 *')
 
 def systemd_to_cron(spec):
   """Convert from systemd.time(7) calendar spec to crontab spec"""
@@ -118,13 +120,22 @@ def systemd_to_cron(spec):
     raise ValueError
   month, day = day
   hour, minute = time
-  spec = minute, hour, day, month, dow
-  for x, (y, z) in zip(spec, ((0, 60), (0, 24), (1, 31), (1, 12))):
+  spec = [minute, hour, day, month, dow]
+  for i, (y, z) in enumerate(((0, 60), (0, 24), (1, 31), (1, 12))):
+    x = spec[i]
     if x != '*':
       for x in x.split(','):
         x = map(int, x.split('/', 1))
-        x[0] -= y
-        if x[0] < 0 or len(x) > 1 and x[0] >= x[1] or z <= sum(x):
+        a = x[0] - y
+        if 0 <= a < z:
+          if len(x) == 1:
+            continue
+          b = x[1]
+          if b > 0:
+            a = (z - a - 1) // b * b
+            if a:
+              spec[i] = '%s-%s/%s' % (x[0], x[0] + a, b)
+              continue
         raise ValueError
   return ' '.join(spec)
 

slapos/recipe/equeue.py

@@ -32,9 +32,13 @@ class Recipe(GenericBaseRecipe):
 
     parameters = [
       '--database', self.options['database'],
-      '-l', self.options['log'],
+      '--logfile', self.options['log'],
+      '--lockfile', self.options['lockfile']
     ]
 
+    if 'takeover-triggered-file-path' in self.options:
+      parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']])
+
     if 'loglevel' in self.options:
       parameters.extend(['--loglevel', self.options['loglevel']])
 

slapos/recipe/generic_mysql/__init__.py

@@ -53,6 +53,15 @@ class Recipe(GenericBaseRecipe):
     else:
       networking = 'skip-networking'
 
+    log_bin = self.options.get('binlog-path', '')
+    if log_bin:
+      log_bin = 'log_bin = %s' % log_bin
+    expire_logs_days = self.options.get('binlog-expire-days')
+    if expire_logs_days > 0:
+      expire_logs_days = 'expire_logs_days = %s' % expire_logs_days
+    else:
+      expire_logs_days = ''
+
     mysql_conf_file = self.createFile(
       self.options['conf-file'],
       self.substituteTemplate(template_filename, {
@@ -62,6 +71,8 @@ class Recipe(GenericBaseRecipe):
         'socket': self.options['socket'],
         'error_log': self.options['error-log'],
         'slow_query_log': self.options['slow-query-log'],
+        'log_bin': log_bin,
+        'expire_logs_days': expire_logs_days,
       })
     )
     path_list.append(mysql_conf_file)
@@ -69,16 +80,19 @@ class Recipe(GenericBaseRecipe):
     mysql_script_list = []
 
     # user defined functions
+    udf_registration = ""
     mroonga = self.options.get('mroonga', 'ha_mroonga.so')
     if mroonga:
-      last_insert_grn_id = "CREATE FUNCTION last_insert_grn_id RETURNS " \
-        "INTEGER SONAME '" + mroonga + "';"
-    else:
-      last_insert_grn_id = ""
+      udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \
+        "INTEGER SONAME '" + mroonga + "';\n"
+      udf_registration += "CREATE FUNCTION mroonga_snippet RETURNS " \
+        "STRING SONAME '" + mroonga + "';\n"
+      udf_registration += "CREATE FUNCTION mroonga_command RETURNS " \
+        "STRING SONAME '" + mroonga + "';\n"
     mysql_script_list.append(self.substituteTemplate(
       self.getTemplateFilename('mysql-init-function.sql.in'),
       {
-        'last_insert_grn_id': last_insert_grn_id,
+        'udf_registration': udf_registration,
       }
     ))
     # real database

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -48,9 +48,8 @@ innodb_locks_unsafe_for_binlog = 1
 #innodb_doublewrite = 0
 #sync_frm = 0
 
-# Uncomment the following if you need binary logging, which is recommended
-# on production instances (either for replication or incremental backups).
-#log-bin=mysql-bin
+%(log_bin)s
+%(expire_logs_days)s
 
 # Force utf8 usage
 collation_server = utf8_unicode_ci

slapos/recipe/generic_mysql/template/mysql-init-function.sql.in

 USE mysql;
 DROP FUNCTION IF EXISTS last_insert_grn_id;
-%(last_insert_grn_id)s
+DROP FUNCTION IF EXISTS mroonga_snippet;
+DROP FUNCTION IF EXISTS mroonga_command;
 DROP FUNCTION IF EXISTS sphinx_snippets;
-#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
+%(udf_registration)s

slapos/recipe/kvm/__init__.py

@@ -45,7 +45,7 @@ class Recipe(GenericBaseRecipe):
 
     path_list = []
 
-    if not self.isTrueValue(self.options.get('use-tap')):
+    if self.isTrueValue(self.options.get('use-nat')):
       # XXX This could be done using Jinja.
       for port in self.options['nat-rules'].split():
         tunnel_port = int(port) + 10000

slapos/recipe/kvm/template/kvm_run.in

@@ -7,6 +7,11 @@ import os
 import socket
 import subprocess
 import urllib
+import gzip
+import shutil
+from random import shuffle
+import glob
+import re
 
 # XXX: give all of this through parameter, don't use this as template, but as module
 qemu_img_path = '%(qemu-img-path)s'
@@ -19,14 +24,30 @@ default_disk_image = '%(default-disk-image)s'
 disk_path = '%(disk-path)s'
 virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
 virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
+virtual_hard_drive_gzipped = '%(virtual-hard-drive-gzipped)s'.strip()
 nat_rules = '%(nat-rules)s'.strip()
 use_tap = '%(use-tap)s'
+use_nat = '%(use-nat)s'
 tap_interface = '%(tap-interface)s'
 listen_ip = '%(ipv4)s'
 mac_address = '%(mac-address)s'
+tap_mac_address = '%(tap-mac-address)s'
 smp_count = '%(smp-count)s'
+smp_options = '%(smp-options)s'.strip()
+numa_list = '%(numa)s'.split()
 ram_size = '%(ram-size)s'
 pid_file_path = '%(pid-file-path)s'
+external_disk_number = %(external-disk-number)s
+external_disk_size = '%(external-disk-size)s'
+external_disk_format = '%(external-disk-format)s'
+disk_storage_dict = {}
+disk_storage_list = """%(disk-storage-list)s""".split('\n')
+map_storage_list = []
+etc_directory = '%(etc-directory)s'.strip()
+httpd_port = %(httpd-port)s
+netcat_bin = '%(netcat-binary)s'.strip()
+cluster_doc_host = '%(cluster-doc-host)s'
+cluster_doc_port = %(cluster-doc-port)s
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -57,25 +78,74 @@ def getSocketStatus(host, port):
     break
   return s
 
+def getMapStorageList(disk_storage_dict, external_disk_number):
+  map_disk_file = os.path.join(etc_directory, '.data-disk-ids')
+  last_disk_num_f = os.path.join(etc_directory, '.data-disk-amount')
+  id_list = []
+  last_amount = 0
+  map_f_exist = os.path.exists(map_disk_file)
+  if os.path.exists(last_disk_num_f):
+    with open(last_disk_num_f, 'r') as lf:
+      last_amount = int(lf.readline())
+  if map_f_exist:
+    with open(map_disk_file, 'r') as mf:
+      # ID are writen in one line: data1 data3 data2 ...
+      content = mf.readline()
+      for id in content.split(' '):
+        if disk_storage_dict.has_key(id):
+          id_list.append(id)
+        else:
+          # Mean that this disk path has been removed (disk unmounted)
+          last_amount -= 1
+  for key in disk_storage_dict:
+    if not key in id_list:
+      id_list.append(key)
+
+  if id_list:
+    if not map_f_exist:
+      # shuffle the list to not write disk in data1, data2, ... everytime 
+      shuffle(id_list)
+    if external_disk_number < last_amount:
+      # Drop created disk is not allowed
+      external_disk_number = last_amount
+    with open(map_disk_file, 'w') as mf:
+      mf.write(' '.join(id_list))
+    with open(last_disk_num_f, 'w') as lf:
+      lf.write('%%s' %% external_disk_number)
+  return id_list, external_disk_number
+
 # Download existing hard drive if needed at first boot
 if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
   print('Downloading virtual hard drive...')
   try:
-    urllib.urlretrieve(virtual_hard_drive_url, disk_path)
+    downloaded_disk = disk_path
+    if virtual_hard_drive_gzipped == 'True':
+      downloaded_disk = '%%s.gz' %% disk_path
+    urllib.urlretrieve(virtual_hard_drive_url, downloaded_disk)
   except:
-    os.remove(disk_path)
+    os.remove(downloaded_disk)
     raise
   md5sum = virtual_hard_drive_md5sum.strip()
   if md5sum:
     print('Checking MD5 checksum...')
-    local_md5sum = md5Checksum(disk_path)
+    local_md5sum = md5Checksum(downloaded_disk)
     if local_md5sum != md5sum:
-      os.remove(disk_path)
+      os.remove(downloaded_disk)
       raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
           local_md5sum, md5sum))
     print('MD5sum check passed.')
   else:
     print('Warning: not checksum specified.')
+  if downloaded_disk.endswith('.gz'):
+    try:
+      with open(disk_path, 'w') as disk:
+        with gzip.open(downloaded_disk, 'rb') as disk_gz:
+          shutil.copyfileobj(disk_gz, disk)
+    except Exception:
+      if os.path.exists(disk_path):
+        os.remove(disk_path)
+      raise
+    os.remove(downloaded_disk)
 
 # Create disk if doesn't exist
 # XXX: move to Buildout profile
@@ -85,25 +155,97 @@ if not os.path.exists(disk_path):
       disk_path, '%%sG' %% disk_size])
   print('Done.')
 
+# Check and create external disk
+additional_disk_list = []
+for storage in disk_storage_list:
+  if storage:
+    key, val = storage.split(' ')
+    disk_storage_dict[key.strip()] = val.strip()
+
+if not external_disk_format in ['qcow2', 'raw', 'vdi', 'vmdk', 'cloop']:
+  external_disk_format = 'qcow2'
+
+map_storage_list, external_disk_number = getMapStorageList(disk_storage_dict,
+                                                      int(external_disk_number))
+
+assert len(map_storage_list) == len(disk_storage_dict)
+if disk_storage_dict:
+  if external_disk_number > 0:
+    index = 0
+    while (index < len(disk_storage_dict)) and (index < external_disk_number):
+      path = disk_storage_dict[map_storage_list[index]]
+      if os.path.exists(path):
+        disk_filepath = os.path.join(path,
+                                  'kvm_virtual_disk.%%s' %% external_disk_format)
+        disk_list = glob.glob('%%s.*' %% os.path.join(path, 'kvm_virtual_disk'))
+        if disk_list == []:
+          print('Creating one additional virtual hard drive...')
+          subprocess.Popen([qemu_img_path, 'create' ,'-f', '%%s' %% external_disk_format,
+              disk_filepath, '%%sG' %% external_disk_size])
+        else:
+          # Cannot change or recreate if disk is exists
+          disk_filepath = disk_list[0]
+        additional_disk_list.append(disk_filepath)
+      else:
+        print('Data folder %%s was not used to create external disk %%r' %% (index +1))
+      index += 1
+
+
 # Generate network parameters
 # XXX: use_tap should be a boolean
+tap_network_parameter = []
+nat_network_parameter = []
+numa_parameter = []
+number = -1
+if use_nat == 'True':
+  number += 1
+  rules = 'user,id=lan%%s,' %% number + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip,
+                        int(port) + 10000, port) for port in nat_rules.split())
+  if httpd_port > 0:
+    rules += ',guestfwd=tcp:10.0.2.100:80-cmd:%%s %%s %%s' %% (netcat_bin,
+                                                        listen_ip, httpd_port)
+  if cluster_doc_host and cluster_doc_port > 0:
+    rules += ',guestfwd=tcp:10.0.2.101:443-cmd:%%s %%s %%s' %% (netcat_bin,
+                                           cluster_doc_host, cluster_doc_port)
+  nat_network_parameter = ['-netdev', rules,
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
 if use_tap == 'True':
-  qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
-else:
-  qemu_network_parameter = 'user,' + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, int(port) + 10000, port) for port in nat_rules.split())
+  number += 1
+  tap_network_parameter = ['-netdev',
+          'tap,id=lan%%s,ifname=%%s,script=no,downscript=no' %% (number,
+            tap_interface),
+          '-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, tap_mac_address)]
 
+smp = smp_count
+if smp_options:
+  for option in smp_options.split(','):
+    key, val = option.split('=')
+    if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
+      smp += ',%%s=%%s' %% (key, val)
 kvm_argument_list = [qemu_path,
-  '-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
-  '-net', qemu_network_parameter,
-  '-smp', smp_count,
-  '-m', ram_size,
+  '-enable-kvm', '-smp', smp,
+  '-m', ram_size, '-vga', 'std',
   '-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
   '-vnc', '%%s:1,ipv4,password' %% listen_ip,
-  '-boot', 'menu=on',
+  '-boot', 'order=cd,menu=on',
   '-qmp', 'unix:%%s,server' %% socket_path,
   '-pidfile', pid_file_path,
 ]
 
+rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
+for numa in numa_list:
+  if rgx.match(numa):
+    numa_parameter.extend(['-numa', numa])
+kvm_argument_list += numa_parameter
+
+if tap_network_parameter == [] and nat_network_parameter == []:
+  print 'Warning : No network interface defined.'
+else:
+  kvm_argument_list += nat_network_parameter + tap_network_parameter
+
+for disk in additional_disk_list:
+  kvm_argument_list.extend([
+            '-drive', 'file=%%s,if=%%s' %% (disk, disk_type)])
 # Try to connect to NBD server (and second nbd if defined).
 # If not available, don't even specify it in qemu command line parameters.
 # Reason: if qemu starts with unavailable NBD drive, it will just crash.
@@ -124,4 +266,5 @@ else:
       '-drive', 'file=%%s,media=cdrom' %% default_disk_image
   ])
 
+print 'Starting KVM: \n %%s' %% ' '.join(kvm_argument_list)
 os.execv(qemu_path, kvm_argument_list)

slapos/recipe/logrotate.py

@@ -65,6 +65,7 @@ class Part(GenericBaseRecipe):
       'dateext',
       'rotate 3650',
       'compress',
+      'delaycompress',
       'notifempty',
       'sharedscripts',
       'create',

slapos/recipe/mkdirectory.py

@@ -36,9 +36,7 @@ class Recipe(GenericBaseRecipe):
     self.mode = int(self.directory.pop('mode', '700'), 8)
 
   def install(self):
-    for path in sorted(self.directory.values()):
-      if not os.path.exists(path):
+    for path in sorted(self.directory.itervalues()):
+      if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
-      elif not os.path.isdir(path):
-        raise OSError("%s path exits, but it's not a directory.")
     return []

slapos/recipe/neoppod.py

@@ -89,7 +89,12 @@ class Admin(NeoBaseRecipe):
 class Master(NeoBaseRecipe):
   def _getOptionList(self):
     options = self.options
-    return [
+    r = [
       '-p', options['partitions'],
       '-r', options['replicas'],
     ]
+    for x in (('-C', options['upstream-cluster']),
+              ('-M', options['upstream-masters'])):
+      if x[1]:
+        r += x
+    return r

slapos/recipe/re6stnet/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import subprocess
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+import socket
+import struct
+import os, stat
+import string, random
+import json
+import traceback
+from slapos import slap
+
+class Recipe(GenericBaseRecipe):
+  
+  def __init__(self, buildout, name, options):
+    """Default initialisation"""
+    self.slap = slap.slap()
+
+    # SLAP related information
+    slap_connection = buildout['slap-connection']
+    self.computer_id = slap_connection['computer-id']
+    self.computer_partition_id = slap_connection['partition-id']
+    self.server_url = slap_connection['server-url']
+    self.software_release_url = slap_connection['software-release-url']
+    self.key_file = slap_connection.get('key-file')
+    self.cert_file = slap_connection.get('cert-file')
+    self.slave_list = json.loads(options['slave-instance-list'])
+
+    options['slave-amount'] = '%s' % len(self.slave_list)
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def getSerialFromIpv6(self, ipv6):
+    prefix = ipv6.split('/')[0].lower()
+    hi, lo = struct.unpack('!QQ', socket.inet_pton(socket.AF_INET6, prefix))
+    ipv6_int = (hi << 64) | lo
+    serial = '0x1%x' % ipv6_int
+
+    # delete non significant part
+    for part in prefix.split(':')[::-1]:
+      if part:
+        for i in ['0']*(4 - len(part)):
+          part = i + part
+        serial = serial.split(part)[0] + part
+        break
+
+    return serial
+
+  def generateCertificate(self):
+    key_file = self.options['key-file'].strip()
+    cert_file = self.options['cert-file'].strip()
+    dh_file = self.options['dh-file'].strip()
+    if not os.path.exists(dh_file):
+      dh_command = [self.options['openssl-bin'], 'dhparam', '-out',
+                            '%s' % dh_file, self.options['key-size']]
+      try:
+        subprocess.check_call(dh_command)
+      except Exception:
+        if os.path.exists(dh_file):
+          os.unlink(dh_file)
+        raise
+
+    if not os.path.exists(cert_file):
+      serial = self.getSerialFromIpv6(self.options['ipv6-prefix'].strip())
+      key_command = [self.options['openssl-bin'], 'genrsa', '-out',
+                            '%s' % key_file, self.options['key-size']]
+
+      #'-config', openssl_configuration
+      cert_command = [self.options['openssl-bin'], 'req', '-nodes', '-new',
+                  '-x509', '-batch', '-key', '%s' % key_file, '-set_serial',
+                  '%s' % serial, '-days', '3650', '-out', '%s' % cert_file]
+
+      try:
+        subprocess.check_call(key_command)
+      except Exception:
+        if os.path.exists(key_file):
+          os.unlink(key_file)
+        raise
+
+      try:
+        subprocess.check_call(cert_command)
+      except Exception:
+        if os.path.exists(cert_file):
+          os.unlink(cert_file)
+        raise
+
+  def generateSlaveTokenList(self, slave_instance_list, token_file):
+    to_remove_dict = {}
+    to_add_dict = {}
+    token_dict = self.loadJsonFile(token_file)
+
+    reference_list = [slave_instance.get('slave_reference') for slave_instance
+                        in slave_instance_list]
+    for reference in reference_list:
+      if not reference in token_dict:
+        # we generate new token
+        number = reference.split('-')[1]
+        new_token = number + ''.join(random.sample(string.ascii_lowercase, 20))
+        token_dict[reference] = new_token
+        to_add_dict[reference] = new_token
+
+    for reference in token_dict.keys():
+      if not reference in reference_list:
+        # This slave instance is destroyed ?
+        to_remove_dict[reference] = token_dict.pop(reference)
+
+    return token_dict, to_add_dict, to_remove_dict
+
+  def loadJsonFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return json.loads(content)
+    else:
+      return {}
+
+  def writeFile(self, path, data):
+    with open(path, 'w') as f:
+      f.write(data)
+    return path
+
+  def readFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return content
+    return ''
+
+  def genHash(self, length):
+    hash_path = os.path.join(self.options['conf-dir'], '%s-hash' % length)
+    if not os.path.exists(hash_path):
+      pool = string.letters + string.digits
+      hash_string = ''.join(random.choice(pool) for i in xrange(length))
+      self.writeFile(hash_path, hash_string)
+    else:
+      hash_string = self.readFile(hash_path)
+
+    return hash_string
+
+  def install(self):
+    path_list = []
+    token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
+    token_list_path = self.options['token-dir']
+
+    self.generateCertificate()
+
+    wrapper = self.createFile(self.options['wrapper'], self.substituteTemplate(
+      self.getTemplateFilename('registry-run.in'), dict(
+          parameter='@%s' % self.options['config-file'],
+          pid_file=self.options['pid-file'],
+          command=self.options['command']
+        )
+      )
+    )
+    os.chmod(self.options['wrapper'], stat.S_IRWXU)
+
+    path_list.append(wrapper)
+
+    registry_url = 'http://%s:%s/' % (self.options['ipv4'], self.options['port'])
+    token_dict, add_token_dict, rm_token_dict = self.generateSlaveTokenList(
+                                              self.slave_list, token_save_path)
+
+    # write request add token
+    for reference in add_token_dict:
+      path = os.path.join(token_list_path, '%s.add' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, add_token_dict[reference])
+
+    # write request remove token
+    for reference in rm_token_dict:
+      path = os.path.join(token_list_path, '%s.remove' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, rm_token_dict[reference])
+        # remove request add token if exists
+        add_path = os.path.join(token_list_path, '%s.add' % reference)
+        if os.path.exists(add_path):
+          os.unlink(add_path)
+
+    self.createFile(token_save_path, json.dumps(token_dict))
+
+    service_dict = dict(token_base_path=token_list_path,
+                        token_json=token_save_path,
+                        db=self.options['db-path'],
+                        partition_id=self.computer_partition_id,
+                        computer_id=self.computer_id,
+                        registry_url=registry_url)
+    service_dict['server_url'] = self.server_url
+    service_dict['cert_file'] = self.cert_file
+    service_dict['key_file'] = self.key_file
+
+    request_add = self.createPythonScript(
+        self.options['manager-wrapper'].strip(),
+        '%s.re6stnet.manage' % __name__, service_dict
+      )
+    path_list.append(request_add)
+
+    request_drop = self.createPythonScript(
+        self.options['drop-service-wrapper'].strip(),
+        '%s.re6stnet.requestRemoveToken' % __name__, service_dict
+      )
+    path_list.append(request_drop)
+
+    request_check = self.createPythonScript(
+        self.options['check-service-wrapper'].strip(),
+        '%s.re6stnet.checkService' % __name__, service_dict
+      )
+    path_list.append(request_check)
+
+    revoke_check = self.createPythonScript(
+        self.options['revoke-service-wrapper'].strip(),
+        '%s.re6stnet.requestRevoqueCertificate' % __name__, service_dict
+      )
+    path_list.append(revoke_check)
+
+    # Send connection parameters of slave instances
+    if token_dict:
+      self.slap.initializeConnection(self.server_url, self.key_file,
+        self.cert_file)
+      computer_partition = self.slap.registerComputerPartition(
+        self.computer_id,
+        self.computer_partition_id)
+
+      for slave_reference, token in token_dict.iteritems():
+        try:
+          status_file = os.path.join(token_list_path, '%s.status' % slave_reference)
+          status = self.readFile(status_file) or 'New token requested'
+          msg = status
+          if status == 'TOKEN_ADDED':
+            msg = 'Token is ready for use'
+          elif status == 'TOKEN_USED':
+            msg = 'Token not available, it has been used to generate re6stnet certificate.'
+
+          ipv6_file = os.path.join(token_list_path, '%s.ipv6' % slave_reference)
+          ipv6 = self.readFile(ipv6_file) or '::'
+
+          computer_partition.setConnectionDict(
+              {'token':token, '1_info':msg, 'ipv6': ipv6},
+              slave_reference)
+        except Exception:
+          self.logger.fatal("Error while sending slave %s informations: %s",
+             slave_reference, traceback.format_exc())
+
+    return path_list
+
+

slapos/recipe/re6stnet/re6stnet.py

+# -*- coding: utf-8 -*-
+import logging
+import json
+import os
+import time
+import sqlite3
+import slapos
+import traceback
+
+from re6st import registry, utils, x509
+from OpenSSL import crypto
+
+
+log = logging.getLogger('SLAPOS-RE6STNET')
+logging.basicConfig(level=logging.DEBUG)
+
+def loadJsonFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+      return json.loads(content)
+  else:
+    return {}
+
+def writeFile(path, data):
+  with open(path, 'w') as f:
+    f.write(data)
+
+def readFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+    return content
+  return ''
+
+def getDb(db_path):
+  db = sqlite3.connect(db_path, isolation_level=None,
+                                                  check_same_thread=False)
+  db.text_factory = str
+
+  return db.cursor()
+
+def bang(args):
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+
+  # Redeploy instance to update published information
+  slap.initializeConnection(args['server_url'], args['key_file'],
+                                                          args['cert_file'])
+  partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                   partition_id=partition_id)
+  partition.bang(message='Published parameters changed!')
+  log.info("Bang with message 'parameters changed'...")
+
+
+def requestAddToken(args, can_bang=True):
+
+  time.sleep(3)
+  registry_url = args['registry_url']
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
+
+  if not path_list:
+    log.info("No new token to add. Exiting...")
+    return
+
+  client = registry.RegistryClient(registry_url)
+  call_bang = False
+
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      # email is unique as reference is also unique
+      email = '%s@slapos' % reference.lower()
+      try:
+        result = client.requestAddToken(token, email)
+      except Exception:
+        log.debug('Request add token fail for %s... \n %s' % (request_file,
+                    traceback.format_exc()))
+        continue
+      if result and result == token:
+        # update information
+        log.info("New token added for slave instance %s. Updating file status..." %
+                            reference)
+        writeFile(os.path.join(base_token_path, '%s.status' % reference),
+                    'TOKEN_ADDED')
+        os.unlink(request_file)
+        call_bang = True
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+
+  if can_bang and call_bang:
+    bang(args)
+
+def requestRemoveToken(args):
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
+
+  if not path_list:
+    log.info("No token to delete. Exiting...")
+    return
+
+  client = registry.RegistryClient(args['registry_url'])
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      try:
+        result = client.requestDeleteToken(token)
+      except Exception:
+        log.debug('Request delete token fail for %s... \n %s' % (request_file,
+                    traceback.format_exc()))
+        continue
+      else:
+        # certificate is invalidated, it will be revoked
+        writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
+      if result == 'True':
+        # update information
+        log.info("Token deleted for slave instance %s. Clean up file status..." %
+                            reference)
+      if result in ['True', 'False']:
+        os.unlink(request_file)
+        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        if os.path.exists(status_file):
+          os.unlink(status_file)
+        ipv6_file = os.path.join(base_token_path, '%s.ipv6' % reference)
+        if os.path.exists(ipv6_file):
+          os.unlink(ipv6_file)
+
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+
+def requestRevoqueCertificate(args):
+
+  base_token_path = args['token_base_path']
+  db = getDb(args['db'])
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
+  client = registry.RegistryClient(args['registry_url'])
+
+  for reference_key in path_list:
+    reference = reference_key.split('.')[0]
+    # XXX - email is always unique
+    email = '%s@slapos' % reference.lower()
+    cert_string = ''
+    try:
+      cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
+          (email,)).next()
+    except StopIteration:
+      # Certificate was not generated yet !!!
+      pass
+
+    try:
+      if cert_string:
+        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
+        cn = x509.subnetFromCert(cert)
+        result = client.revoke(str(cn))
+        time.sleep(2)
+    except Exception:
+      log.debug('Request revoke certificate fail for %s... \n %s' % (reference,
+                  traceback.format_exc()))
+      continue
+    else:
+      os.unlink(os.path.join(base_token_path, reference_key))
+      log.info("Certificate revoked for slave instance %s." % reference)
+
+
+def dumpIPv6Network(slave_reference, db, network, ipv6_file):
+  email = '%s@slapos' % slave_reference.lower()
+
+  try:
+    cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
+        (email,)).next()
+  except StopIteration:
+    # Certificate was not generated yet !!!
+    pass
+
+  try:
+    if cert_string:
+      cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
+      cn = x509.subnetFromCert(cert)
+      subnet = network + utils.binFromSubnet(cn)
+      ipv6 = utils.ipFromBin(subnet)
+      writeFile(ipv6_file, ipv6)
+  except Exception:
+    log.debug('XXX for %s... \n %s' % (slave_reference,
+              traceback.format_exc()))
+
+def checkService(args, can_bang=True):
+  base_token_path = args['token_base_path']
+  token_dict = loadJsonFile(args['token_json'])
+
+  if not token_dict:
+    return
+
+  db = getDb(args['db'])
+  call_bang = False
+
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+  client = registry.RegistryClient(args['registry_url'])
+  ca = client.getCa()
+  network = x509.networkFromCa(crypto.load_certificate(crypto.FILETYPE_PEM, ca))
+
+  # Check token status
+  for slave_reference, token in token_dict.iteritems():
+    status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
+    ipv6_file = os.path.join(base_token_path, '%s.ipv6' % slave_reference)
+    if not os.path.exists(status_file):
+      # This token is not added yet!
+      continue
+
+    msg = readFile(status_file)
+    if msg == 'TOKEN_USED':
+      dumpIPv6Network(slave_reference, db, network, ipv6_file)
+      continue
+
+    # Check if token is not in the database
+    status = False
+    try:
+        token_found, = db.execute("SELECT token FROM token WHERE token = ?",
+            (token,)).next()
+        if token_found == token:
+          status = True
+    except StopIteration:
+        pass
+    if not status:
+      # Token is used to register client
+      call_bang = True
+      try:
+        writeFile(status_file, 'TOKEN_USED')
+        dumpIPv6Network(slave_reference, db, network, ipv6_file)
+        log.info("Token status of %s updated to 'used'." % slave_reference)
+      except IOError:
+        # XXX- this file should always exists
+        log.debug('Error when writing in file %s. Clould not update status of %s...' %
+                              (status_file, slave_reference))
+
+  if call_bang and can_bang:
+    bang(args)
+
+def manage(args):
+  # Request Add new tokens
+  requestAddToken(args)
+
+  # Request delete removed token
+  requestRemoveToken(args)
+
+  # check status of all token
+  checkService(args)
+

slapos/recipe/re6stnet/template/registry-run.in

+#!/bin/sh
+
+echo $$ > %(pid_file)s
+exec %(command)s \
+  %(parameter)s
\ No newline at end of file

slapos/recipe/request.py

@@ -29,9 +29,11 @@ from zc.buildout import UserError
 from slapos.recipe.librecipe import wrap, JSON_SERIALISED_MAGIC_KEY
 import json
 from slapos import slap as slapmodule
+from slapos.slap import SoftwareProductCollection
 import slapos.recipe.librecipe.generic as librecipe
 import traceback
 
+SOFTWARE_PRODUCT_NAMESPACE = "product."
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
 class Recipe(object):
@@ -130,6 +132,19 @@ class Recipe(object):
       options['computer-id'],
       options['partition-id'],
     ).request
+
+    if software_url is not None and \
+      software_url.startswith(SOFTWARE_PRODUCT_NAMESPACE):
+    
+      product = SoftwareProductCollection(self.logger, slap)
+      
+      try:
+        software_url = product.__getattr__(
+          software_url[len(SOFTWARE_PRODUCT_NAMESPACE):])
+      except AttributeError as e:
+        self.logger.warning('Error on get software release : %s ' % e.message)
+        
+
     self._raise_request_exception = None
     self._raise_request_exception_formatted = None
     self.instance = None

slapos/recipe/shellinabox.py

@@ -69,7 +69,6 @@ def shellinabox(args):
 
   command_line = [
     args['shellinabox'],
-    '-d',
     '-c', certificate_dir,
     '-s', service,
     '--ipv6', args['ipv6'],

slapos/recipe/simplehttpserver/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from slapos.recipe.librecipe import GenericBaseRecipe
+import string, random
+import os
+
+class Recipe(GenericBaseRecipe):
+
+  def __init__(self, buildout, name, options):
+
+    base_path = options['base-path']
+    pool = string.letters + string.digits
+    hash_string = ''.join(random.choice(pool) for i in xrange(64))
+    path = os.path.join(base_path, hash_string)
+
+    if os.path.exists(base_path):
+      path_list = os.listdir(base_path)
+      if len(path_list) == 1:
+        hash_string = path_list[0]
+        path = os.path.join(base_path, hash_string)
+      elif len(path_list) > 1:
+        raise ValueError("Folder %s should contain 0 or 1 element." % base_path)
+
+    options['root-dir'] = path
+    options['path'] = hash_string
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+
+  def install(self):
+
+    if not os.path.exists(self.options['root-dir']):
+      os.mkdir( self.options['root-dir'] )
+    parameters = {
+      'host': self.options['host'],
+      'port': int(self.options['port']),
+      'cwd': self.options['base-path'],
+      'log-file': self.options['log-file'],
+      'cert-file': self.options['cert-file'],
+      'key-file': self.options['key-file']
+    }
+
+    server = self.createPythonScript(
+        self.options['wrapper'].strip(),
+        '%s.simplehttpserver.run' % __name__, parameters
+      )
+
+    return [server]

slapos/recipe/simplehttpserver/simplehttpserver.py

+from SimpleHTTPServer import SimpleHTTPRequestHandler
+from BaseHTTPServer import HTTPServer
+import ssl
+import os
+import logging
+from netaddr import valid_ipv4, valid_ipv6
+import socket
+
+class ServerHandler(SimpleHTTPRequestHandler):
+
+  def respond(self, code=200, type='text/plain'):
+    self.send_response(code)
+    self.send_header("Content-type", type)
+    self.end_headers()
+
+  def do_GET(self):
+    logging.info('%s - GET: %s \n%s' % (self.client_address[0], self.path, self.headers))
+    if not self.path or self.path == '/':
+      # no access to root path
+      self.respond(403)
+      self.wfile.write("Forbidden")
+      return
+    SimpleHTTPRequestHandler.do_GET(self)
+
+class HTTPServerV6(HTTPServer):
+  address_family = socket.AF_INET6
+  
+
+def run(args):
+  
+  # minimal web server.  serves files relative to the
+  # current directory.
+  logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                            filename=args['log-file'] ,level=logging.INFO)
+  
+  port = args['port']
+  host = args['host']
+  os.chdir(args['cwd'])
+  
+  Handler = ServerHandler
+  
+  if valid_ipv6(host):
+    server = HTTPServerV6
+  else:
+    server = HTTPServer
+  
+  httpd = server((host, port), Handler)
+  if args.has_key('cert-file') and args.has_key('key-file'):
+    httpd.socket = ssl.wrap_socket (httpd.socket, 
+                                     server_side=True,
+                                     certfile=args['cert-file'],
+                                     keyfile=args['key-file'])
+
+  logging.info("Starting simple http server at https://%s:%s" % (host, port))
+  httpd.serve_forever()

slapos/recipe/slapconfiguration.py

@@ -32,6 +32,7 @@ import slapos.slap
 from slapos.recipe.librecipe import unwrap
 from ConfigParser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
+from slapos.util import mkdir_p
 
 class Recipe(object):
   """
@@ -61,6 +62,10 @@ class Recipe(object):
       Partition identifier.
       Example:
         ${slap-connection:partition-id}
+    storage-home
+      Path of folder configured for data storage
+      Example:
+        ${storage-configuration:storage-home}
 
   Output:
     slap-software-type
@@ -75,8 +80,20 @@ class Recipe(object):
       One of the IPv6 addresses.
     tap
       Set of TAP interfaces.
+    tap-network-information-dict
+      Dict of set of all TAP network information
+    tap-ipv4
+      ipv4 allowed for this TAP
+    tap-gateway
+      ipv4 of gateway interface of this TAP
+    tap-netmask
+      ipv4 netmask address of this TAP
+    tap-network
+      ipv4 network address of this TAP
     configuration
       Dict of all parameters.
+    storage-dict
+      Dict of partition data path when it is configured
     configuration.<key>
       One key per partition parameter.
       Partition parameter whose name cannot be represented unambiguously in
@@ -91,7 +108,8 @@ class Recipe(object):
   OPTCRE_match = RawConfigParser.OPTCRE.match
 
   def __init__(self, buildout, name, options):
-      parameter_dict = self.fetch_parameter_dict(options)
+      parameter_dict = self.fetch_parameter_dict(options,
+                                      buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
       for key, value in parameter_dict.iteritems():
@@ -99,7 +117,7 @@ class Recipe(object):
               continue
           options['configuration.' + key] = value
 
-  def fetch_parameter_dict(self, options):
+  def fetch_parameter_dict(self, options, instance_root):
       slap = slapos.slap.slap()
       slap.initializeConnection(
           options['url'],
@@ -134,6 +152,14 @@ class Recipe(object):
       v6_add = ipv6_set.add
       tap_set = set()
       tap_add = tap_set.add
+      route_gw_set = set()
+      route_gw_add = route_gw_set.add
+      route_mask_set = set()
+      route_mask_add = route_mask_set.add
+      route_ipv4_set = set()
+      route_v4_add = route_ipv4_set.add
+      route_network_set = set()
+      route_net_add = route_network_set.add
       for tap, ip in parameter_dict.pop('ip_list'):
           tap_add(tap)
           if valid_ipv4(ip):
@@ -141,6 +167,21 @@ class Recipe(object):
           elif valid_ipv6(ip):
               v6_add(ip)
           # XXX: emit warning on unknown address type ?
+
+      if 'full_ip_list' in parameter_dict:
+        for item in parameter_dict.pop('full_ip_list'):
+          if len(item) == 5:
+            tap, ip, gw, netmask, network = item
+            if  tap.startswith('route_'):
+              if valid_ipv4(gw):
+                route_gw_add(gw)
+              if valid_ipv4(netmask):
+                route_mask_add(netmask)
+              if valid_ipv4(ip):
+                route_v4_add(ip)
+              if valid_ipv4(network):
+                route_net_add(network)
+
       options['ipv4'] = ipv4_set
       options['ipv6'] = ipv6_set
 
@@ -149,6 +190,35 @@ class Recipe(object):
           options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
       if ipv6_set:
           options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+      if route_ipv4_set:
+        options['tap-ipv4'] = list(route_ipv4_set)[0].encode('UTF-8')
+        options['tap-network-information-dict'] = dict(ipv4=route_ipv4_set,
+                                    netmask=route_mask_set,
+                                    gateway=route_gw_set,
+                                    network=route_network_set)
+      else:
+        options['tap-network-information-dict'] = {}
+      if route_gw_set:
+        options['tap-gateway'] = list(route_gw_set)[0].encode('UTF-8')
+      if route_mask_set:
+        options['tap-netmask'] = list(route_mask_set)[0].encode('UTF-8')
+      if route_network_set:
+        options['tap-network'] = list(route_network_set)[0].encode('UTF-8')
+
+      storage_home = options.get('storage-home')
+      storage_dict = {}
+      if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+        for filename in os.listdir(storage_home):
+          storage_path = os.path.join(storage_home, filename, 
+                                    options['slap-computer-partition-id'])
+          if os.path.exists(storage_path) and os.path.isdir(storage_path):
+            storage_link = os.path.join(instance_root, 'DATA', filename)
+            mkdir_p(os.path.join(instance_root, 'DATA'))
+            if not os.path.lexists(storage_link):
+              os.symlink(storage_path, storage_link)
+            storage_dict[filename] = storage_link
+      options['storage-dict'] = storage_dict
 
       options['tap'] = tap_set
       return self._expandParameterDict(options, parameter_dict)

slapos/recipe/softwaretype.py

@@ -34,6 +34,7 @@ import subprocess
 import slapos.slap
 import netaddr
 import logging
+import errno
 
 import zc.buildout
 
@@ -54,6 +55,18 @@ class Recipe:
         return ip
     raise AttributeError
 
+  def _getTapIpAddressList(self, test_method):
+    """Internal helper method to fetch full ip address assigned for tap"""
+    if not 'full_ip_list' in self.parameter_dict:
+      return ()
+    for item in self.parameter_dict['full_ip_list']:
+      if len(item) == 5:
+        tap, ip, gw, mask, net = item
+        if tap.startswith('route_') and test_method(ip) and \
+                          test_method(gw) and test_method(mask):
+          return (ip, gw, mask, net)
+    return ()
+
   def getLocalIPv4Address(self):
     """Returns local IPv4 address available on partition"""
     # XXX: Lack checking for locality of address
@@ -64,6 +77,11 @@ class Recipe:
     # XXX: Lack checking for globality of address
     return self._getIpAddress(netaddr.valid_ipv6)
   
+  def getLocalTapIPv4AddressList(self):
+    """Returns global IPv6 address available for tap interface"""
+    # XXX: Lack checking for locality of address
+    return self._getTapIpAddressList(netaddr.valid_ipv4)
+
   def getNetworkInterface(self):
     """Returns the network interface available on partition"""
     if not 'ip_list' in self.parameter_dict:
@@ -73,6 +91,20 @@ class Recipe:
         return name
     raise AttributeError, "Not network interface found"
   
+  def mkdir_p(self, path, mode=0700):
+    """
+    Creates a directory and its parents, if needed.
+    NB: If the directory already exists, it does not change its permission.
+    """
+
+    try:
+        os.makedirs(path, mode)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
   def install(self):
     slap = slapos.slap.slap()
     slap_connection = self.buildout['slap_connection']
@@ -81,6 +113,15 @@ class Recipe:
     server_url = slap_connection['server_url']
     key_file = slap_connection.get('key_file')
     cert_file = slap_connection.get('cert_file')
+    instance_root = self.buildout['buildout']['directory']
+    storage_configuration_dict = self.buildout.get('storage-configuration')
+    network_dict = self.buildout.get('network-information')
+    storage_home = ''
+    global_ipv4_network = ''
+    if storage_configuration_dict:
+      storage_home = storage_configuration_dict.get('storage-home')
+    if network_dict:
+      global_ipv4_network = network_dict.get('global-ipv4-network')
     slap.initializeConnection(server_url, key_file, cert_file)
     self.computer_partition = slap.registerComputerPartition(
       computer_id,
@@ -128,6 +169,16 @@ class Recipe:
                  self.getGlobalIPv6Address())
     buildout.set('slap-network-information', 'network-interface',
                  self.getNetworkInterface())
+    tap_ip_list = self.getLocalTapIPv4AddressList()
+    tap_ipv4 = tap_gateway = tap_netmask = tap_network = ''
+    if tap_ip_list:
+      tap_ipv4, tap_gateway, tap_netmask, tap_network= tap_ip_list
+    buildout.set('slap-network-information', 'tap-ipv4', tap_ipv4)
+    buildout.set('slap-network-information', 'tap-gateway', tap_gateway)
+    buildout.set('slap-network-information', 'tap-netmask', tap_netmask)
+    buildout.set('slap-network-information', 'tap-network', tap_network)
+    buildout.set('slap-network-information', 'global-ipv4-network',
+                                                          global_ipv4_network)
 
     # Copy/paste slap_connection
     buildout.add_section('slap-connection')
@@ -137,6 +188,27 @@ class Recipe:
     # XXX: Needed for lxc. Use non standard API
     buildout.set('slap-connection', 'requested', self.computer_partition._requested_state)
 
+    # setup storage directory
+    buildout.add_section('storage-configuration')
+    buildout.set('storage-configuration', 'storage-home', storage_home)
+    if storage_home and os.path.exists(storage_home) and \
+                                  os.path.isdir(storage_home):
+      # Create folder instance_root/DATA/ if not exist
+      data_home = os.path.join(instance_root, 'DATA')
+      self.mkdir_p(data_home)
+      for filename in os.listdir(storage_home):
+        storage_path = os.path.join(storage_home, filename, computer_partition_id)
+        if os.path.exists(storage_path) and os.path.isdir(storage_path):
+          storage_link = os.path.join(data_home, filename)
+          if os.path.lexists(storage_link):
+            if not os.path.islink(storage_link):
+              raise zc.buildout.UserError(
+                  'Target %r already exists but is not a link' % storage_link)
+              #os.unlink(storage_link)
+          else:
+            os.symlink(storage_path, storage_link)
+          buildout.set('storage-configuration', filename, storage_link)
+
     work_directory = os.path.abspath(self.buildout['buildout'][
       'directory'])
     buildout_filename = os.path.join(work_directory,

slapos/recipe/tidstorage/__init__.py

@@ -33,14 +33,15 @@ class Recipe(GenericBaseRecipe):
       self.options['configuration-path'],
       self.substituteTemplate(
         self.getTemplateFilename('tidstorage.py.in'), self.options))
+    r = [configuration_file]
 
-    tidstorage_wrapper = self.createPythonScript(
-      self.options['tidstorage-wrapper'],
+    wrapper = self.options.get('tidstorage-wrapper')
+    wrapper and r.append(self.createPythonScript(wrapper,
       'slapos.recipe.librecipe.execute.execute',
       [self.options['tidstoraged-binary'],  '--nofork', '--config',
-        configuration_file])
+        configuration_file]))
 
-    repozo_wrapper = self.createPythonScript(
+    r.append(self.createPythonScript(
       self.options['repozo-wrapper'],
       'slapos.recipe.librecipe.execute.execute',
       [self.options['tidstorage-repozo-binary'],
@@ -48,6 +49,6 @@ class Recipe(GenericBaseRecipe):
         '--repozo', self.options['repozo-binary'],
         '--gzip',
         '--quick',
-      ])
+      ]))
 
-    return [configuration_file, tidstorage_wrapper, repozo_wrapper]
+    return r

slapos/recipe/wrapper.py

@@ -53,9 +53,7 @@ class Recipe(GenericBaseRecipe):
                           if filename.strip()]
         if environment is not None:
             environment = dict((k.strip(), v.strip()) for k, v in [
-                             line.split('=')
-                             for line in environment.split('\n')
-                           ])
+              line.split('=') for line in environment.splitlines() if line.strip() ])
         return [self.createPythonScript(
             wrapper_path,
             'slapos.recipe.librecipe.execute.generic_exec',

slapos/test/recipe/test_dcron.py

@@ -13,12 +13,13 @@ class TestDcron(unittest.TestCase):
     _("10-15", "0 0 15 10 *")
     _("monday *-12-* 17:00", "00 17 * 12 1")
     _("12,14,13,12:20,10,30", "20,10,30 12,14,13,12 * * *") # TODO: sort
-    _("*-1/2-1,3 *:30", "30 * 1,3 1/2 *")
+    _("*-1/2-1,3 *:30", "30 * 1,3 1-11/2 *")
     _("03-05 08:05", "05 08 05 03 *")
     _("08:05:00", "05 08 * * *")
     _("05:40", "40 05 * * *")
     _("Sat,Sun 12-* 08:05", "05 08 * 12 0,6")
     _("Sat,Sun 08:05", "05 08 * * 0,6")
+    _("*:25/20", "25-45/20 * * * *")
 
     def _(systemd):
       self.assertRaises(Exception, systemd_to_cron, systemd)
@@ -31,7 +32,7 @@ class TestDcron(unittest.TestCase):
     _("08:05:40")
     _("2003-03-05")
 
-    _("0-1"); _("13-1"); _("6/4-1"); _("5/8-1")
-    _("1-0"); _("1-32"); _("1-4/3"); _("1-14/18")
-    _("24:0");_("9/9:0"); _("8/16:0")
-    _("0:60"); _("0:22/22"); _("0:15/45")
+    _("0-1"); _("13-1"); _("8/5-1")
+    _("1-0"); _("1-32"); _("1-14/18")
+    _("24:0"); _("8/16:0")
+    _("0:60"); _("0:15/45")

slapos/test/recipe/test_re6stnet.py

+
+import os, time
+import shutil
+import sys
+import tempfile
+import unittest
+from slapos.slap.slap import NotFoundError, ConnectionError
+
+from slapos.recipe import re6stnet
+
+
+class Re6stnetTest(unittest.TestCase):
+
+  def setUp(self):
+    self.ssl_dir = tempfile.mkdtemp()
+    self.conf_dir = tempfile.mkdtemp()
+    self.base_dir = tempfile.mkdtemp()
+    self.token_dir = tempfile.mkdtemp()
+    self.dir_list = [self.ssl_dir, self.conf_dir, self.base_dir, self.token_dir]
+    config_file = os.path.join(self.base_dir, 'config')
+    with open(config_file, 'w') as f:
+      f.write('port  9201')
+    self.options = options = {
+                'openssl-bin': '/usr/bin/openssl',
+                'key-file': os.path.join(self.ssl_dir, 'cert.key'),
+                'cert-file': os.path.join(self.ssl_dir, 'cert.crt'),
+                'dh-file': os.path.join(self.ssl_dir, 'dh.pem'),
+                'key-size': '2048',
+                'conf-dir': self.conf_dir,
+                'token-dir': self.token_dir,
+                'wrapper': os.path.join(self.base_dir, 'wrapper'),
+                'config-file': config_file,
+                'ipv4': '127.0.0.1',
+                'port': '9201',
+                'pid-file': '/path/to/pid/file',
+                'db-path': '/path/to/db',
+                'command': '/path/to/command',
+                'manager-wrapper': os.path.join(self.base_dir, 'manager_wrapper'),
+                'drop-service-wrapper': os.path.join(self.base_dir, 'drop_wrapper'),
+                'check-service-wrapper': os.path.join(self.base_dir, 'check_wrapper'),
+                'revoke-service-wrapper': os.path.join(self.base_dir, 'revoke_wrapper'),
+                'slave-instance-list': '{}'
+                }
+    
+  def tearDown(self):
+    for path in self.dir_list:
+      if os.path.exists(path):
+        shutil.rmtree(path)
+    
+  def new_recipe(self):
+      buildout = {
+              'buildout': {
+                  'bin-directory': '',
+                  'find-links': '',
+                  'allow-hosts': '',
+                  'develop-eggs-directory': '',
+                  'eggs-directory': '',
+                  'python': 'testpython',
+                  },
+               'testpython': {
+                   'executable': sys.executable,
+                   },
+               'slap-connection': {
+                   'computer-id': 'comp-test',
+                   'partition-id': 'slappart0',
+                   'server-url': 'http://server.com',
+                   'software-release-url': 'http://software.com',
+                   'key-file': '/path/to/key',
+                   'cert-file': '/path/to/cert'
+                   }
+              }
+
+      options = self.options
+
+      return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options)
+
+  def checkWrapper(self, path):
+    self.assertTrue(os.path.exists(path))
+    content = ""
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    with open(path, 'r') as f:
+        content = f.read()
+    self.assertIn("'token_json': '%s'" % token_file, content)
+    self.assertIn("'partition_id': 'slappart0'", content)
+    self.assertIn("'computer_id': 'comp-test'", content)
+    self.assertIn("'key_file': '/path/to/key'", content)
+    self.assertIn("'cert_file': '/path/to/cert'", content)
+    self.assertIn("'server_url': 'http://server.com'", content)
+    self.assertIn("'db': '%s'" % self.options['db-path'], content)
+    self.assertIn("'token_base_path': '%s'" % self.token_dir, content)
+    self.assertIn("'registry_url': 'http://%s:%s/'" % (self.options['ipv4'],
+                                                self.options['port']), content)
+  
+  def checkRegistryWrapper(self):
+    path = os.path.join(self.base_dir, 'wrapper')
+    self.assertTrue(os.path.exists(path))
+    content = ""
+    config_file = os.path.join(self.base_dir, 'config')
+    with open(path, 'r') as f:
+        content = f.read()
+    self.assertIn("@%s" % config_file, content)
+    self.assertIn("/path/to/pid/file", content)
+    self.assertIn("/path/to/command", content)
+
+  def fake_generateCertificates(self):
+    return
+
+  def test_generateCertificates(self):
+    
+    self.options['ipv6-prefix'] = '2001:db8:24::/48'
+    self.options['key-size'] = '2048'
+    
+    recipe = self.new_recipe()
+    
+    recipe.generateCertificate()
+    
+    self.assertItemsEqual(os.listdir(self.ssl_dir),
+                          ['cert.key', 'cert.crt', 'dh.pem'])
+    
+    last_time = time.ctime(os.stat(self.options['key-file'])[7])
+
+    recipe.generateCertificate()
+
+    self.assertTrue(os.path.exists(self.options['key-file']))
+    this_time = time.ctime(os.stat(self.options['key-file'])[7])
+
+    self.assertEqual(last_time, this_time)
+  
+  def test_getSerialFromIpv6(self):
+
+    ipv6 = 'be28:db8:fe6a:d85:4fe:54a:ae:aea/64'
+
+    recipe = self.new_recipe()
+    serial = recipe.getSerialFromIpv6(ipv6)
+
+    self.assertEqual(serial, '0x1be280db8fe6a0d8504fe054a00ae0aea')
+
+    ipv6 = '2001:db8:24::/48'
+    serial = recipe.getSerialFromIpv6(ipv6)
+
+    self.assertEqual(serial, '0x120010db80024')
+
+  def test_install(self):
+    self.options.update({
+        'ipv6-prefix': '2001:db8:24::/48',
+        'slave-instance-list': '''[
+            {"slave_reference":"SOFTINST-58770"},
+            {"slave_reference":"SOFTINST-58778"}
+            ]
+            '''
+        })
+
+    recipe = self.new_recipe()
+    recipe.generateCertificate = self.fake_generateCertificates
+
+    try:
+      recipe.install()
+    except (NotFoundError, ConnectionError):
+      # Recipe will raise not found error when trying to publish slave informations
+      pass
+    
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    self.assertTrue(os.path.exists(token_file))
+    
+    # token file must contain 2 elements
+    token_content = recipe.readFile(token_file)
+    self.assertIn('SOFTINST-58770', token_content)
+    self.assertIn('SOFTINST-58778', token_content)
+    
+    token_dict = recipe.loadJsonFile(token_file)
+    self.assertEqual(len(token_dict), 2)
+    self.assertTrue(token_dict.has_key('SOFTINST-58770'))
+    self.assertTrue(token_dict.has_key('SOFTINST-58778'))
+        
+    self.assertItemsEqual(os.listdir(self.token_dir),
+                          ['SOFTINST-58770.add', 'SOFTINST-58778.add'])
+
+    first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add'))
+    self.assertEqual(token_dict['SOFTINST-58770'], first_add)
+    
+    second_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.add'))
+    self.assertEqual(token_dict['SOFTINST-58778'], second_add)
+    
+    self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'revoke_wrapper'))
+    self.checkRegistryWrapper()
+    
+    # Remove one element
+    self.options.update({
+        "slave-instance-list": """[{"slave_reference":"SOFTINST-58770"}]"""
+        })
+    recipe = self.new_recipe()
+    recipe.generateCertificate = self.fake_generateCertificates
+
+    try:
+      recipe.install()
+    except (NotFoundError, ConnectionError):
+      # Recipe will raise not found error when trying to publish slave informations
+      pass
+    token_dict = recipe.loadJsonFile(token_file)
+    
+    self.assertEqual(len(token_dict), 1)
+    self.assertEqual(token_dict['SOFTINST-58770'], first_add)
+    self.assertItemsEqual(os.listdir(self.token_dir),
+                          ['SOFTINST-58770.add', 'SOFTINST-58778.remove'])
+    
+    second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove'))
+    self.assertEqual(second_add, second_remove)
+  
+  def test_install_empty_slave(self):
+    self.options.update({
+        'ipv6-prefix': '2001:db8:24::/48'
+        })
+    recipe = self.new_recipe()
+    recipe.generateCertificate = self.fake_generateCertificates
+
+    recipe.install()
+
+    token_file = os.path.join(self.options['conf-dir'], 'token.json')
+    self.assertTrue(os.path.exists(token_file))
+    
+    token_content = recipe.readFile(token_file)
+    self.assertEqual(token_content, '{}')
+    self.assertItemsEqual(os.listdir(self.options['token-dir']), [])
+
+    self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'revoke_wrapper'))
+    self.checkRegistryWrapper()
+

software/apache-frontend/common.cfg

@@ -5,7 +5,6 @@ extends =
 
   ../../stack/slapos.cfg
   ../../component/dash/buildout.cfg
-  ../../component/binutils/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/apache/buildout.cfg
   ../../component/gzip/buildout.cfg
@@ -25,7 +24,6 @@ parts +=
   template
   template-apache-frontend
   template-apache-replicate
-  binutils
   apache-2.2
   apache-antiloris-apache-2.2
 
@@ -80,7 +78,7 @@ mode = 0644
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = bae669cdc917c68186a387903478a53d
+md5sum = 1fe76dde85c488e94baf8510775ebcaf
 mode = 640
 
 [template-slave-configuration]
@@ -98,7 +96,7 @@ mode = 640
 [template-apache-frontend-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache.conf.in
-md5sum = ce88924c53f09c9a3ef12ec4d8a8ad16
+md5sum = 6c72015a9af4f1edab63712f5c6aec99
 mode = 640
 
 [template-apache-cached-configuration]
@@ -129,13 +127,13 @@ mode = 640
 [template-default-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/000.conf.in
-md5sum = b40ffdab93a80b40046e3bbb2f7a58bc
+md5sum = ed1b680e31e30596bf051682ec0270b4
 mode = 640
 
 [template-default-slave-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
-md5sum = 9bd3eda3c2aad5061f6cd6985e6f18d0
+md5sum = 5463dd67f1b1bea0bee57a421e371dd0
 mode = 640
 
 [template-log-access]

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -15,12 +15,20 @@
       "type": "string",
       "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
     },
+
+    "server-alias": {
+      "title": "Server Alias",
+      "description": "Server Alias List separated by space",
+      "type": "string",
+      "default": ""
+    },
+
     "type": {
       "title": "Backend Type",
-      "description": "Type of the backend",
+      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster",
       "type": "string",
       "default": "",
-      "enum": ["", "zope"]
+      "enum": ["", "zope", "redirect"]
     },
 
     "path": {
@@ -71,6 +79,30 @@
       "default": "false",
       "enum": ["false", "true"]
     },
+
+    "disable-no-cache-request": {
+      "title": "Disable 'no-cache' requests",
+      "description": "If set to true, no-cache control headers will be disabled",
+      "type": "string",
+      "default": "false",
+      "enum": ["false", "true"]
+    },
+
+    "prefer-gzip-encoding-to-backend": {
+      "title": "Prefer gzip Encoding for Backend",
+      "description": "If set to true, if a request is made with accept encoding 'gzip', only that one will be transferred to the backend",
+      "type": "string",
+      "default": "false",
+      "enum": ["false", "true"]
+    },
+
+    "disabled-cookie-list": {
+      "title": "Disabled Cookies",
+      "description": "List of Cookies separated by space that will not be sent to the backend",
+      "type": "string",
+      "default": ""
+    },
+
     "apache_custom_http": {
       "title": "HTTP configuration",
       "description": "Raw http configuration in python template format. Your site will be rejected if you use it without notification and approval of the frontend adminastrator",

software/apache-frontend/software.cfg

@@ -12,7 +12,7 @@ gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7
 slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
@@ -30,4 +30,4 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2

software/apache-frontend/templates/000.conf.in

@@ -2,7 +2,8 @@
   ServerName www.example.org
   SSLEngine on
   SSLProxyEngine on
-  SSLProtocol ALL -SSLv2
+  SSLProtocol ALL -SSLv2 -SSLv3
+  SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
 
   # Rewrite part
   ProxyVia On

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -163,7 +163,7 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
 
 # The slave use cache
 # Next line is forbidden and people who copy it will be hanged short
-{%     set enable_cache = ('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES -%}
+{%   set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_instance.get('type', '') != 'redirect') -%}
 {%     if enable_cache -%}
 {%       do cached_server_dict.__setitem__(slave_instance.get('custom_domain'), slave_instance.get('url')) -%}
 {%       do slave_instance.__setitem__('url', cache_access) -%}

software/apache-frontend/templates/apache.conf.in

@@ -41,6 +41,10 @@ CustomLog "{{ access_log }}" combined
 <Directory {{ document_root }}>
   Order Allow,Deny
   Allow from All
+  Options -Indexes
+  ErrorDocument 404 /notfound.html
+  RewriteEngine on
+  RewriteRule ^/?$ notfound.html [R=404,L]
 </Directory>
 
 # List of modules
@@ -115,7 +119,8 @@ SSLSessionCache shmcb:/{{ httpd_mod_ssl_cache_directory }}/ssl_scache(512000)
 SSLSessionCacheTimeout  300
 SSLRandomSeed startup /dev/urandom 256
 SSLRandomSeed connect builtin
-SSLProtocol ALL -SSLv2
+SSLProtocol all -SSLv2 -SSLv3
+SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
 <FilesMatch "\.(cgi|shtml|phtml|php)$">
       SSLOptions +StdEnvVars
 </FilesMatch>
@@ -128,3 +133,6 @@ include {{frontend_configuration.get('log-access-configuration')}}
 NameVirtualHost *:{{ http_port }}
 NameVirtualHost *:{{ https_port }}
 include {{ slave_configuration_directory }}/*.conf
+
+ErrorDocument 404 /notfound.html
+RewriteRule (.*) /notfound.html [R=404,L]

software/apache-frontend/templates/default-virtualhost.conf.in

 {% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
+{% set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES -%}
+{%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES -%}
  
 <VirtualHost *:{{ https_port }}>
   ServerName {{ slave_parameter.get('custom_domain') }}
   ServerAlias {{ slave_parameter.get('custom_domain') }}
 
+{%- if 'server-alias' in slave_parameter -%}
+  {% set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
+  {%- for server_alias in server_alias_list %}
+  ServerAlias {{ server_alias }}
+  {% endfor %}
+{%- endif %}
+
   SSLEngine on
   SSLProxyEngine on
-  SSLProtocol -ALL +SSLv3 +TLSv1
-  SSLHonorCipherOrder On
-  SSLCipherSuite RC4-SHA:HIGH:!ADH
+  SSLProtocol all -SSLv2 -SSLv3
+  SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4
+
 
 {% set ssl_configuration_list = [('SSLCertificateFile', 'path_to_ssl_crt'),
        			      	 ('SSLCertificateKeyFile', 'path_to_ssl_key'),
@@ -34,6 +43,22 @@
   ProxyTimeout 600
   RewriteEngine On
 
+{% if disable_no_cache_header %}
+  RequestHeader unset Cache-Control
+  RequestHeader unset Pragma
+{% endif -%}
+
+{% if 'disabled-cookie-list' in slave_parameter -%}
+  {% set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
+  {%- for disabled_cookie in disabled_cookie_list %}
+{{'  RequestHeader edit Cookie "(^%(disabled_cookie)s=[^;]*; |; %(disabled_cookie)s=[^;]*|^%(disabled_cookie)s=[^;]*$)" ""' % dict(disabled_cookie=disabled_cookie)  }}
+  {% endfor -%}
+{% endif %}
+
+{%- if prefer_gzip %}
+  RequestHeader edit Accept-Encoding "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" "gzip"
+{% endif %}
+
 {% if slave_parameter.get('type', '') ==  'zope' -%}
   {% if 'default-path' in slave_parameter %}
   RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
@@ -42,6 +67,8 @@
   # If so, let's use Virtual Host Daemon rewrite
   # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
   RewriteRule ^/(.*)$ {{ slave_parameter.get('url', '') }}/VirtualHostBase/https/{{ slave_parameter.get('custom_domain', '') }}:443/{{ slave_parameter.get('path', '') }}/VirtualHostRoot/$1 [L,P]
+{% elif slave_parameter.get('type', '') ==  'redirect' -%}
+  RewriteRule     (.*)  {{slave_parameter.get('url', '')}}$1 [R,L]
 {% else -%}
   {% if 'default-path' in slave_parameter %}
   RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]
@@ -53,6 +80,14 @@
 <VirtualHost *:{{ http_port }}>
   ServerName {{ slave_parameter.get('custom_domain') }}
   ServerAlias {{ slave_parameter.get('custom_domain') }}
+
+{%- if 'server-alias' in slave_parameter %}
+  {% set server_alias_list =  slave_parameter.get('server-alias', '').split() %}
+  {%- for server_alias in server_alias_list %}
+  ServerAlias {{ server_alias }}
+  {% endfor -%}
+{% endif %}
+
   SSLProxyEngine on
   # Rewrite part
   ProxyVia On
@@ -69,6 +104,22 @@
   # Remove "Secure" from cookies, as backend may be https
   Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
 
+{% if disable_no_cache_header %}
+  RequestHeader unset Cache-Control
+  RequestHeader unset Pragma
+{% endif -%}
+
+{% if 'disabled-cookie-list' in slave_parameter -%}
+  {% set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
+  {%- for disabled_cookie in disabled_cookie_list %}
+{{'  RequestHeader edit Cookie "(^%(disabled_cookie)s=[^;]*; |; %(disabled_cookie)s=[^;]*|^%(disabled_cookie)s=[^;]*$)" ""' % dict(disabled_cookie=disabled_cookie)  }}
+  {% endfor -%}
+{% endif %}
+
+{%- if prefer_gzip %}
+  RequestHeader edit Accept-Encoding "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)" "gzip"
+{% endif %}
+
 # Next line is forbidden and people who copy it will be hanged short
 {% set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES -%}
 {% if https_only -%}
@@ -77,6 +128,8 @@
   # on standard port (443).
   RewriteCond     %{SERVER_PORT}  !^{{ https_port }}$
   RewriteRule     ^/(.*)          https://%{SERVER_NAME}/$1 [NC,R,L]
+{% elif slave_parameter.get('type', '') ==  'redirect' -%}
+  RewriteRule     (.*)  {{slave_parameter.get('url', '')}}$1 [R,L]
 {% elif slave_parameter.get('type', '') ==  'zope' -%}
   {% if 'default-path' in slave_parameter %}
   RewriteRule ^/?$ {{ slave_parameter.get('default-path') }} [R=301,L]

software/backupserver/software.cfg

@@ -67,7 +67,7 @@ mode = 0644
 [template-backup-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/template-backup-script.sh.in
-md5sum = 0d8fb8ea80966af7d67ad50d50547dce
+md5sum = 47b20031db3b575651d8515d5add23e6
 output = ${buildout:directory}/template-backup-script.sh.in
 mode = 0644
 

software/backupserver/template-backup-script.sh.in

@@ -29,7 +29,7 @@ if [ $RESULT -eq 0 ]
 then
   ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup success" >> $${:status_log}
   ${coreutils-output:rm} -f $${:statistic_log}
-  ${findutils-output:find} rdiff-backup-data/ -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average >> $${:statistic_log}
+  ${findutils-output:find} rdiff-backup-data/ -maxdepth 1 -name "session_statistic*" | ${coreutils-output:sort} | ${coreutils-output:tail} -n 1 | ${findutils-output:xargs} ${rdiff-backup-output:rdiff-backup} --calculate-average >> $${:statistic_log}
 else
   ${coreutils-output:echo} "`${coreutils-output:date} -u`,$${:statistic_log},$${:hostname} backup failed" >> $${:status_log}
 fi

software/cdn-me/instance-cdn-request.cfg

+[buildout]
+parts =
+  request-re6stnet-token-slave
+  request-frontend-token-slave
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+# Create all needed directories
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+etc = $${:home}/etc/
+var = $${:home}/var/
+srv = $${:home}/srv/
+bin = $${:home}/bin/
+tmp = $${:home}/tmp/
+
+[request-frontend-token-slave]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = WebSite Frontend
+# XXX We have hardcoded SR URL here.
+software-url = product.frontend
+slave = true
+config-url = http://$${request-re6stnet-token-slave:connection-ipv6}/
+config-domain = $${slap-parameter:frontend-domain}
+return = site_url domain
+
+[request-re6stnet-token-slave]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Re6st token Frontend
+# XXX We have hardcoded SR URL here.
+software-url = product.re6st
+slave = true
+return = token info_1 ipv6
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+url =  https://$${request-frontend-token-slave:connection-domain}
+token = $${request-re6stnet-token-slave:connection-token}
+ipv6 = $${request-re6stnet-token-slave:connection-ipv6}
+info_1 = $${request-re6stnet-token-slave:info_1}
\ No newline at end of file

software/cdn-me/instance.cfg

+[buildout]
+parts =
+  switch_softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[switch_softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${instance-base-runner:rendered}
+
+[instance-base-runner]
+recipe = slapos.recipe.template:jinja2
+template = ${template-cdn-request:output}
+rendered = $${buildout:directory}/template-cdn.cfg
+extensions = jinja2.ext.do
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+mode = 0644
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}

software/cdn-me/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+
+# stacks are listed from most generic to most specific,
+# to avoid versioning issues
+
+parts =
+  slapos-cookbook-develop
+  template
+  eggs
+  template-cdn-request
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+md5sum = a91fe7c80720d57c2acbf606a6a0d84d
+mode = 0644
+
+[template-cdn-request]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-cdn-request.cfg
+output = ${buildout:directory}/template-cdn-request.cfg
+md5sum = 4c5ad2a5e9c4364588e1e4212ed8d1aa
+mode = 0644
+
+[slapos.cookbook-repository]
+branch = request.product
+
+[eggs]
+recipe = z3c.recipe.scripts
+eggs =
+  collective.recipe.environment
+  cns.recipe.symlink
+  erp5.util
+  lock-file
+  plone.recipe.command
+  slapos.recipe.build
+  ${slapos-cookbook:eggs}
\ No newline at end of file

software/cloudooo/software.cfg

@@ -21,6 +21,7 @@ cert = ${slap-connection:cert-file}
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended
@@ -79,7 +80,3 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
 md5sum = 4bede3be20dbc2ecfdb5d49b3184742e
 mode = 640
-
-[versions]
-# use newest version of slapos.cookbook
-slapos.cookbook =

software/dream/instance.cfg

@@ -2,12 +2,16 @@
 parts =
   dream_simulation
   dream_platform
+  dream_test_suite
+  dream_interpreter
+  grunt_watch
   publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+# parameters
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap_connection:computer_id}
@@ -15,22 +19,42 @@ partition = $${slap_connection:partition_id}
 url = $${slap_connection:server_url}
 key = $${slap_connection:key_file}
 cert = $${slap_connection:cert_file}
+[dream_platform_parameter]
+port = 8080
+host = $${instance-parameter:ipv6-random}
 
-port = 5000
-host = $${slap-network-information:local-ipv4}
+# interpreter
+[dream_interpreter]
+recipe = slapos.cookbook:wrapper
+command-line = ${buildout:bin-directory}/dream_interpreter
+wrapper-path = $${buildout:bin-directory}/dream_interpreter
+parameters-extra = true
 
+# service
 [dream_platform]
 recipe = slapos.cookbook:wrapper
-command-line = ${buildout:bin-directory}/dream_platform --host $${instance-parameter:host} --port $${instance-parameter:port}
+command-line = ${buildout:bin-directory}/dream_platform --debug --host $${dream_platform_parameter:host} --port $${dream_platform_parameter:port} --log $${directory:log}/dream_platform.log
 wrapper-path = $${directory:service}/dream_platform
 parameters-extra = true
 
+[grunt_watch]
+recipe = slapos.cookbook:wrapper
+command-line = bash -c 'cd ${dream-repository.git:location}; PATH=${nodejs:location}/bin/:$PATH ${dream-repository.git:location}/node_modules/grunt-cli/bin/grunt watch -f > $${directory:log}/grunt.log'
+wrapper-path = $${directory:service}/dream_grunt_watch
+
+# CLI
 [dream_simulation]
 recipe = slapos.cookbook:wrapper
 command-line = ${buildout:bin-directory}/dream_simulation
 wrapper-path = $${directory:script}/dream_simulation
 parameters-extra = true
 
+[dream_test_suite]
+recipe = slapos.cookbook:wrapper
+command-line = ${dream_testrunner:script}
+wrapper-path = $${directory:script}/dream_test_suite
+parameters-extra = true
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 home = $${buildout:directory}
@@ -43,8 +67,5 @@ log = $${:var}/log
 
 [publish-connection-parameter]
 recipe = slapos.cookbook:publishurl
-url = http://$${instance-parameter:host}:$${instance-parameter:port}
+url = http://[$${dream_platform_parameter:host}]:$${dream_platform_parameter:port}
 
-; FIXME: how to enable this without "switch" swoftare type ?
-[slap-network-information]
-local-ipv4 = 0.0.0.0

software/dream/software.cfg

 [buildout]
+versions = versions
 extends =
   ../../stack/slapos.cfg
-  ../../component/numpy/buildout.cfg
-  ../../component/r-language/buildout.cfg
-
-parts +=
-  rpy2
+  ../../stack/nodejs.cfg
+  ../../component/manpy/buildout.cfg
+parts =
   slapos-cookbook
-  dream-build
-  dream
+  manpy
+  dream_testrunner
+  dream_interpreter
+  npm_install
   instance
 
-[rpy2_env]
-PATH = ${r-language:location}/bin/:%(PATH)s
-
-[rpy2]
-recipe = zc.recipe.egg:custom
-environment = rpy2_env
-egg = rpy2>=2.3,<2.4
-
-[dream-repository.git]
-recipe = slapos.recipe.build:gitclone
-repository = http://git.erp5.org/repos/dream.git
-branch = master
-git-executable = ${git:location}/bin/git
-
-[dream-build]
-; use a develop egg to easily develop and because dream already exist on pypi
-; https://pypi.python.org/pypi/dream
-recipe = zc.recipe.egg:develop
-dependency = ${scipy-build:recipe}
-setup = ${dream-repository.git:location}
-
-[dream]
-recipe = zc.recipe.egg:scripts
-eggs = dream
-initialization =
-  # rpy2 needs R in $PATH
-  import os
-  os.environ['PATH'] = '${r-language:location}/bin' + os.pathsep + os.environ['PATH']
-
 [instance]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/instance.cfg
 
+[dream_testrunner]
+recipe = zc.recipe.testrunner
+eggs = dream
+script = dream_testrunner
+initialization =
+  ${manpy:initialization}
+
+[dream_interpreter]
+recipe = z3c.recipe.scripts
+eggs = ${manpy:eggs}
+interpreter = dream_interpreter
+initialization =
+  ${manpy:initialization}
+
+[npm_install]
+recipe = plone.recipe.command
+stop-on-error = true
+command = cd ${dream-repository.git:location} && PATH=${git:location}/bin/:$PATH ${nodejs:location}/bin/npm install .
+update_command = ${:command}
+
+[versions]
+rpy2 = 2.4.0
+pydot = 1.0.28
+xlrd = 0.9.3
+xlwt = 0.7.5
+pyparsing = 2.0.3
+numpy = 1.9.1
+scipy = 0.13.3
+simpy = 3.0.5
+zope.dottedname = 4.1.0
+tablib = 0.10.0
+MySQL-python = 1.2.5
+
+# indirect dependancies
+collective.recipe.template = 1.11
+cp.recipe.cmd = 0.5
+plone.recipe.command = 1.1
+slapos.recipe.template = 2.7
+zope.exceptions = 4.0.7
+zope.testing = 4.1.3
+zc.recipe.testrunner = 2.0.0
+zope.testrunner = 4.4.6
+z3c.recipe.scripts = 1.0.1

software/dream/software_testnode.cfg

+[buildout]
+extends =
+  software.cfg
+
+# nodejs installation script does not support too deep directory structure like
+# we can have when installing testnode in a webrunner. Since we do not need
+# nodejs for the simulation executor, we can simply disable this section
+[nodejs]
+recipe =
+location =
+
+[npm_install]
+recipe =

software/erp5testnode/software.cfg

@@ -9,14 +9,8 @@ extends =
   ../../component/pwgen/buildout.cfg
   ../../component/apache/buildout.cfg
 
-# Local development
-develop =
-  ${:parts-directory}/slapos.cookbook-repository
 parts =
-# Local development
   slapos-cookbook
-  slapos.cookbook-repository
-  check-recipe
   template
   lxml-python
   eggs
@@ -24,20 +18,6 @@ parts =
   git
   apache
 
-# Local development
-[slapos.cookbook-repository]
-recipe = plone.recipe.command
-stop-on-error = true
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = "${git:location}/bin/git" clone --branch erp5testnode --quiet http://git.erp5.org/repos/slapos.git "${:location}"
-update-command = cd "${:location}" && "${git:location}/bin/git" fetch --quiet && "${git:location}/bin/git" reset --hard @{upstream}
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -79,7 +59,5 @@ md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.41
-plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
-jsonschema = 2.4.0
+erp5.util = 0.4.42
+slapos.recipe.template = 2.7

software/gateone/instance-gateone.cfg.in

+
+{% set python_bin = parameter_dict['python-executable'] -%}
+{% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+tmp = ${buildout:directory}/tmp
+log = ${:var}/log
+services = ${:etc}/service
+script = ${:etc}/run
+promises = ${:etc}/promise
+run = ${:var}/run
+ca-dir = ${:etc}/ssl
+requests = ${:ca-dir}/requests/
+private = ${:ca-dir}/private/
+certs = ${:ca-dir}/certs/
+newcerts = ${:ca-dir}/newcerts/
+crl = ${:ca-dir}/crl/
+
+[gateone-dir]
+gateone = ${directory:srv}/gateone
+log-prefix = ${directory:log}/gateone
+cache = ${:gateone}/cache
+sessions = ${:gateone}/sessions
+users = ${:gateone}/users
+conf = ${directory:etc}/gateone
+ssl = ${:conf}/ssl
+
+[gateone-configure]
+recipe = slapos.cookbook:wrapper
+port = 10443
+ipv6 = {{ ipv6 }}
+ipv4 = {{ ipv4 }}
+settings-dir = ${gateone-dir:conf}
+wrapper-path = ${directory:bin}/gateone-configure
+# XXX- issue with dtach=true, we set --dtach=false
+command = 
+    {{ parameter_dict['gateone-bin'] }} --address=${:ipv4} --port=${:port}  --certificate=${gateone-dir:ssl}/certificate.pem --keyfile=${gateone-dir:ssl}/keyfile.pem --cache_dir=${gateone-dir:cache} --pid_file=${directory:run}/gateone.pid --session_dir=${gateone-dir:sessions} --settings_dir=${:settings-dir} --user_dir=${gateone-dir:users} --log_file_prefix=${gateone-dir:log-prefix} --origins=${:ipv4} --logging=info --dtach=false
+#--gid=1012 --uid=987
+command-line = 
+  ${:command} --configure
+environment = 
+  PATH={{ bin_directory }}:{{ openssl_location }}/bin:{{ parameter_dict['dtach-location'] }}/bin:/usr/local/bin:/usr/bin:/bin
+  LD_LIBRARY_PATH={{ parameter_dict['readline-location'] }}/lib
+
+[gateone-run]
+recipe = slapos.cookbook:wrapper
+port = 10443
+ipv6 = {{ ipv6 }}
+ipv4 = {{ ipv4 }}
+wrapper-path = ${directory:services}/gateone
+command-line = ${gateone-configure:command}
+environment = 
+  PATH={{ bin_directory }}:{{ openssl_location }}/bin:{{ parameter_dict['dtach-location'] }}/bin:/usr/local/bin:/usr/bin:/bin
+  LD_LIBRARY_PATH={{ parameter_dict['readline-location'] }}/lib
+  HOME=${buildout:directory}
+  PWD=${gateone-dir:gateone}
+
+# --cookie_secret 45-characters
+# --uid needs to be set for slapos
+# --origins=${:ipv4};${:ipv6}
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_location }}/bin/openssl
+ca-dir = ${directory:ca-dir}
+requests-directory = ${directory:requests}
+wrapper = ${directory:services}/certificate_authority
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+
+[ca-nginx]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = ${directory:certs}/nginx.key
+cert-file = ${directory:certs}/nginx.crt
+executable = ${nginx-run:wrapper-path}
+wrapper = ${directory:services}/nginx-proxy
+
+[tempdirectory]
+recipe = slapos.cookbook:mkdirectory
+client_body_temp_path = ${directory:tmp}/client_body_temp_path
+proxy_temp_path = ${directory:tmp}/proxy_temp_path
+fastcgi_temp_path = ${directory:tmp}/fastcgi_temp_path
+uwsgi_temp_path = ${directory:tmp}/uwsgi_temp_path
+scgi_temp_path = ${directory:tmp}/scgi_temp_path
+
+[nginx-config-dict]
+nb_workers = 2
+ipv6 = {{ ipv6 }}
+ipv4 = {{ ipv4 }}
+port = 10443
+backend-url = https://${gateone-configure:ipv4}:${gateone-configure:port}
+ssl-certificate = ${ca-nginx:cert-file}
+ssl-key = ${ca-nginx:key-file}
+pid = ${directory:run}/nginx.pid
+log = ${directory:log}/nginx.log
+access-log = ${directory:log}/nginx.access.log
+error-log = ${directory:log}/nginx.error.log
+tmp = $${directory:tmp}/
+
+[nginx-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-ngnix-conf'] }}
+rendered = ${directory:etc}/nginx.conf
+context =
+    section parameter_dict nginx-config-dict
+    section param_tempdir tempdirectory
+
+[nginx-run]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/nginx_proxy
+command-line = {{ parameter_dict['nginx-location'] }}/sbin/nginx -p ${buildout:directory} -c ${nginx-conf:rendered}
+
+[logrotate-apache]
+< = logrotate-entry-base
+name = apache
+log = ${apache-conf:error-log} ${apache-conf:access-log}
+post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
+
+[logrotate-entry-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+
+[publish]
+recipe = slapos.cookbook:publish
+url = https://[${nginx-config-dict:ipv6}]:${gateone-run:port}
+
+[buildout]
+extends =
+  {{ logrotate_cfg }}
+
+parts = 
+  certificate-authority
+  ca-nginx
+  publish
+
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-parameter]
+

software/gateone/instance-logrotate-base.cfg.in

+[buildout]
+parts =
+  cron-entry-logrotate
+
+[cron]
+recipe = slapos.cookbook:cron
+cron-entries = ${logrotate-directory:cron-entries}
+dcrond-binary = {{ dcron_location }}/sbin/crond
+crontabs = ${logrotate-directory:crontabs}
+cronstamps = ${logrotate-directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${logrotate-directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${logrotate-directory:bin}/cron_simplelogger
+log = ${logrotate-directory:log}/cron.log
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+logrotate-entries = ${logrotate-directory:logrotate-entries}
+backup = ${logrotate-directory:logrotate-backup}
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+wrapper = ${logrotate-directory:bin}/logrotate
+conf = ${logrotate-directory:etc}/logrotate.conf
+state-file = ${logrotate-directory:srv}/logrotate.status
+
+[cron-entry-logrotate]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate-directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+crontabs = ${:etc}/crontabs
+logrotate-backup = ${:backup}/logrotate
+logrotate-entries = ${:etc}/logrotate.d
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+backup = ${:srv}/backup
+etc = ${buildout:directory}/etc
+services = ${:etc}/service
+log = ${buildout:directory}/var/log
+

software/gateone/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+extra-context =
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key ipv6_set slap-configuration:ipv6
+    key ipv4_set slap-configuration:ipv4
+    key slapparameter_dict slap-configuration:configuration
+    key computer_id slap-configuration:computer
+    raw logrotate_cfg {{ template_logrotate_base }}
+    raw dash_binary {{ dash_location }}/bin/dash
+    raw bin_directory {{ bin_directory }}
+    raw openssl_location {{ openssl_location }}
+    ${:extra-context}
+
+[dynamic-template-gateone-parameters]
+bin-directory = {{ bin_directory }}
+#python-executable  = 
+readline-location = {{ readline_location }}
+dtach-location = {{ dtach_location }}
+gateone-location = {{ gateone_location}}
+gateone-bin = {{ gateone_bin }}
+nginx-location = {{ nginx_location}}
+template-ngnix-conf = {{ template_nginx_conf }}
+
+[dynamic-template-gateone]
+< = jinja2-template-base
+template = {{ template_gateone }}
+filename = instance-gateone.cfg
+extensions = jinja2.ext.do
+extra-context =
+    section parameter_dict dynamic-template-gateone-parameters
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+
+default = ${dynamic-template-gateone:rendered}
+gateone = ${:default}

software/gateone/software.cfg

+[buildout]
+
+extends =
+  ../../component/dash/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/kerberos/buildout.cfg
+  ../../component/python-kerberos/buildout.cfg
+  ../../component/gateone/buildout.cfg
+  ../../component/dtach/buildout.cfg
+  ../../component/python-2.7/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../stack/slapos.cfg
+
+parts = 
+  slapos-cookbook
+  kerberos
+  eggs
+  gateone-develop
+  gateone
+  dash
+  check-recipe
+  template
+  
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  ${python-cryptography:egg}
+  ${python-kerberos:egg}
+  tornado
+scripts =
+  slapos-kill
+
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = python
+eggs =
+  ${lxml-python:egg}
+  ${python-kerberos:egg}
+  tornado
+  gateone
+  setuptools
+  pyOpenSSL
+  futures
+
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = re6st-master
+git-executable = ${git:location}/bin/git
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 644
+
+[template-jinja2-base]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:directory}/${:filename}
+# XXX: extra-context is needed because we cannot append to a key of an extended
+# section.
+extra-context =
+context =
+    key bin_directory buildout:bin-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    ${:extra-context}
+
+
+[template]
+< = template-jinja2-base
+filename = template.cfg
+template = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 428669a609aca3e0a7cae1387d332a75
+extra-context =
+    key dash_location dash:location
+    key dtach_location dtach:location
+    key gateone_location gateone-repository:location
+    key logrotate_location logrotate:location
+    key nginx_location nginx:location
+    key openssl_location openssl:location
+    key readline_location readline:location
+    key template_nginx_conf template-nginx-conf:target
+    key template_logrotate_base template-logrotate-base:rendered
+    key template_gateone template-gateone:target
+    raw gateone_bin ${buildout:bin-directory}/gateone
+    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+
+[template-gateone]
+< = download-base
+filename = instance-gateone.cfg.in
+md5sum = e7096a17c36c3bd27a011de57b7abfc1
+
+[template-logrotate-base]
+< = template-jinja2-base
+filename = instance-logrotate-base.cfg
+md5sum = f28fbd310944f321ccb34b2a34c82005
+extra-context =
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+
+[template-nginx-conf]
+< = download-base
+url = ${:_profile_base_location_}/templates/${:filename}.in
+filename = nginx.conf
+md5sum = 72f4cc110f618b317793e21124f45121
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/gateone.egg-link
+
+[versions]
\ No newline at end of file

software/gateone/templates/nginx.conf.in

+worker_processes {{ parameter_dict['nb_workers'] }};
+
+pid {{ parameter_dict['pid'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+daemon off;
+
+events {
+	worker_connections 1024;
+	accept_mutex off;
+}
+
+
+http {
+     default_type application/octet-stream;
+     access_log {{ parameter_dict['access-log'] }} combined;
+     client_max_body_size 10M;
+     map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+     }
+     server {
+        listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }} ssl;
+        server_name _;
+        ssl_certificate     {{ parameter_dict['ssl-certificate'] }};
+        ssl_certificate_key {{ parameter_dict['ssl-key'] }};
+        ssl_protocols       SSLv3 TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+        keepalive_timeout 90s;
+        client_body_temp_path {{ param_tempdir['client_body_temp_path'] }};
+        proxy_temp_path {{ param_tempdir['proxy_temp_path'] }};
+        fastcgi_temp_path {{ param_tempdir['fastcgi_temp_path'] }};
+        uwsgi_temp_path {{ param_tempdir['uwsgi_temp_path'] }};
+        scgi_temp_path {{ param_tempdir['scgi_temp_path'] }};
+	      error_page 401 /login;
+        location / {
+            proxy_pass_header Server;
+        		proxy_set_header Host $http_host;
+        		proxy_redirect off;
+        		proxy_set_header X-Real-IP $remote_addr;
+        		proxy_set_header X-Scheme $scheme;
+        		
+        		proxy_pass {{ parameter_dict['backend-url'] }};
+        		
+        		proxy_http_version 1.1;
+        		proxy_set_header Upgrade $http_upgrade;
+        		proxy_set_header Connection "upgrade";
+        }
+    }
+}
\ No newline at end of file

software/helloworld/hello-web.in

+#!{{ python_executable }}
+"""Simple web-server that says "Hello World" for every path
+
+hello-web [--logfile <logfile>] <bind-ip> <bind-port> ...
+"""
+
+import sys
+import time
+import argparse
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from socket import AF_INET6
+
+
+class WebHello(BaseHTTPRequestHandler):
+
+    def do_GET(self):
+        self.send_response(200) # ok
+        self.send_header("Content-type", "text/plain")
+        self.end_headers()
+
+        print >>self.wfile, \
+            "Hello %s at `%s`  ; %s" % (
+                ' '.join(self.server.webhello_argv) or 'world',
+                self.path, time.asctime())
+
+
+class HTTPServerV6(HTTPServer):
+    address_family = AF_INET6
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--logfile', dest='logfile')
+    parser.add_argument('bind_ip')
+    parser.add_argument('bind_port', type=int)
+    parser.add_argument('argv_extra', metavar='...', nargs=argparse.REMAINDER)
+
+    args = parser.parse_args()
+
+    # HTTPServer logs to sys.stderr - override it if we have --logfile
+    if args.logfile:
+        f = open(args.logfile, 'a', buffering=1)
+        sys.stderr = f
+
+    print >>sys.stderr, '* %s Hello-Web starting at %s' % (
+        time.asctime(), (args.bind_ip, args.bind_port))
+
+    # TODO autodetect ipv6/ipv4
+    httpd = HTTPServerV6( (args.bind_ip, args.bind_port), WebHello)
+    httpd.webhello_argv = args.argv_extra
+    httpd.serve_forever()
+
+if __name__ == '__main__':
+    main()

software/helloworld/instance.cfg.in

@@ -7,6 +7,7 @@
 parts =
   directory
   hello-world
+  hello-world-promise
   publish-connection-parameter
 
 # Define egg directories to be the one from Software Release
@@ -24,7 +25,7 @@ offline = true
 # It will then authenticate to SlapOS Master and fetch the instance parameters.
 # The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
 # Always the same. Just copy/paste.
-# See docstring of slapos.cookbook:slapconfiguration for more informations.
+# See docstring of slapos.cookbook:slapconfiguration for more information.
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap_connection:computer_id}
 partition = $${slap_connection:partition_id}
@@ -58,24 +59,48 @@ promise = $${:etc}/promise/
 # Path of the log directory used by our service (see [hello-world])
 log = $${:var}/log
 
-# Create a simple shell script that will only output your name if you
-# specified it as instance parameter.
-# Usually, of course, we use more useful commands, like web servers.
+
+# Create a simple web server that says "hello <configuration.name>" to the web.
 [hello-world]
+# helloworld service is listening on:
+# - global IPv6 address, and
+# - fixed port
+#
+# NOTE because every computer partition is allocated its own global IPv6
+# address, it is ok to fix the port - different hello-world instances will have
+# different IPv6 addresses and they all will be accessible at the same time.
+ipv6 = $${instance-parameter:ipv6-random}
+port = 7777
+# full URL - for convenience
+url = http://[$${:ipv6}]:$${:port}
+
+# the service will log here
+logfile = $${directory:log}/hello-world.log
+
+# Actual script that starts the service:
 # This recipe will try to "exec" the command-line after separating parameters.
 recipe = slapos.cookbook:wrapper
-# Notice that there is only one $ at ${dash:location}, it is because it comes from the Software Release buildout profile.
-command-line = ${dash:location}/bin/dash -c 'echo "Hello $${instance-parameter:configuration.name}, it is $(date)." > $${directory:log}/log.log; sleep 1000000;'
-# Put this shell script in the "etc/service" directory. Every executable of this
-# repository will be started and monitored by supervisord. If one service
-# exits/crashes, it will trigger a "bang" and cause run of slapgrid for the
-# instance.
+command-line =
+    ${hello-web-bin:rendered} --logfile $${hello-world:logfile}
+        $${:ipv6} $${:port} $${instance-parameter:configuration.name}
+# Put this shell script in the "etc/service" directory. Each executable of this
+# repository will be started and monitored by supervisord. If a service
+# exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
 wrapper-path = $${directory:service}/hello-world
 
 
+# promise, that checks that hello-world service is alive
+[hello-world-promise]
+recipe = slapos.cookbook:check_port_listening
+path = $${directory:promise}/hello-world
+hostname= $${hello-world:ipv6}
+port = $${hello-world:port}
+
+
 # Publish all the parameters needed for the user to connect to the instance.
 # It can be anything: URL(s), password(s), or arbitrary parameters.
 # Here we'll just echo back the entered name as instance parameter
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
 name = Hello $${instance-parameter:configuration.name}!
+url = $${hello-world:url}

software/helloworld/software.cfg

@@ -6,9 +6,10 @@ extends =
   ../../stack/slapos.cfg
 # Extend here component profiles, like openssl, apache, mariadb, curl...
 # Or/and extend a stack (lamp, tomcat) that does most of the work for you
-# In this example we only need the dash binary to run a simple "hello world"
-# shell script.
-  ../../component/dash/buildout.cfg
+# In this example we don't need anything more than python which is provided by
+# above stack/slapos.cfg
+# ../../component/component1/buildout.cfg
+# ../../component/component2/buildout.cfg
 
 parts =
 # Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
@@ -18,6 +19,10 @@ parts =
 # instance
   instance-profile
 
+# "build" python program (install + correct shebang for our python)
+  hello-web-bin
+
+
 # Download instance.cfg.in (buildout profile used to deployment of instance),
 # replace all ${foo:bar} parameters by real values, and change $${foo:bar} to
 # ${foo:bar}
@@ -26,5 +31,23 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
 # MD5 checksum can be skipped for development (easier to develop), but must be filled for production
-md5sum = ed94ac99ae1e596c0da5350da6ab6f52
+md5sum = 968bea0fc81dc604a874c53648b7d13f
 mode = 0644
+
+
+# install hello-web with correct python_executable
+[hello-web-bin]
+recipe = slapos.recipe.template:jinja2
+filename = hello-web
+md5sum = da4a93ff679d40c6682859476dcf4ce0
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:bin-directory}/${:filename}
+mode = 0755
+# XXX python_executable should be ${${buildout:python}:executable}
+#     but buildout cannot support such indirection.
+#
+# in real-cases, python software is usually installed with zc.recipe.egg
+# which cares about correctly specifiing python interpreter for
+# entry-points automatically.
+context =
+    raw python_executable ${buildout:executable}

software/html5as/software.cfg

@@ -92,4 +92,4 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2

software/html5ide/software.cfg

@@ -48,4 +48,4 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/ipython_notebook/instance.cfg

+[buildout]
+parts =
+  instance
+  publish-connection-parameter
+## Monitoring part XXX whe should not have to specify all parts like this
+## Parts to add for monitoring
+  certificate-authority
+  cron
+  cron-entry-monitor
+  cron-entry-rss
+  deploy-index
+  deploy-settings-cgi
+  deploy-status-cgi
+  deploy-status-history-cgi
+  setup-static-files
+  certificate-authority
+  zero-parameters
+  public-symlink
+  cgi-httpd-wrapper
+  cgi-httpd-graceful-wrapper
+  monitor-promise
+  monitor-instance-log-access
+## Monitor for ipython
+  monitor-current-log-access
+  monitor-deploy-set-password-cgi
+
+extends = ${monitor-template:output}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
+
+[instance-parameter]
+port = 8888
+host = $${slapconfiguration:ipv6-random}
+cert_file = $${generate-certificate:cert_file}
+key_file = $${generate-certificate:key_file}
+logfile = $${directory:log}/ipython_notebook.log
+notebook_dir = $${directory:notebook_dir}
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e $${instance-parameter:key_file} ]
+  then
+    ${openssl-output:openssl} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout $${instance-parameter:key_file} \
+      -out $${instance-parameter:cert_file}
+  fi
+update-command = $${:command}
+cert_file = $${directory:etc}/ipython_notebook_cert.crt
+key_file = $${directory:etc}/ipython_notebook_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  ${buildout:bin-directory}/ipython notebook
+  --no-browser
+  --matplotlib=inline
+  --ip=$${instance-parameter:host}
+  --port=$${instance-parameter:port}
+  --port-retries=0
+  --certfile=$${instance-parameter:cert_file}
+  --keyfile=$${instance-parameter:key_file}
+  --notebook-dir=$${instance-parameter:notebook_dir}
+  --logfile=$${instance-parameter:logfile}
+  --config=$${ipython_notebook_config:rendered}
+wrapper-path = $${directory:service}/ipython_notebook
+parameters-extra = true
+
+[ipython_notebook_config]
+recipe = slapos.recipe.template:jinja2
+template = ${ipython_notebook_config:location}/${ipython_notebook_config:filename}
+rendered = $${directory:etc}/ipython_notebook_config.py
+mode = 0744
+context =
+  raw config_cfg $${buildout:directory}/knowledge0.cfg
+
+[monitor-current-log-access]
+< = monitor-directory-access
+source = $${instance-parameter:logfile}
+
+[monitor-deploy-set-password-cgi]
+recipe = slapos.recipe.template:jinja2
+template = ${ipython_notebook_set_password:location}/${ipython_notebook_set_password:filename}
+rendered = $${monitor-directory:knowledge0-cgi}/$${:filename}
+filename = ipython-notebook-password.cgi
+mode = 0744
+context =
+  raw config_cfg $${buildout:directory}/knowledge0.cfg
+  raw python_executable ${buildout:bin-directory}/ipython
+  key pwd monitor-directory:knowledge0-cgi
+  key this_file :filename
+  key httpd_graceful cgi-httpd-graceful-wrapper:rendered
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = $${buildout:directory}
+etc = $${:home}/etc
+var = $${:home}/var
+script = $${:etc}/run/
+service = $${:etc}/service
+promise = $${:etc}/promise/
+log = $${:var}/log
+notebook_dir = $${:var}/notebooks
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = https://[$${instance-parameter:host}]:$${instance-parameter:port}
+monitor_url = $${monitor-parameters:url}

software/ipython_notebook/software.cfg

+[buildout]
+versions = versions
+extends = 
+  ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/ipython/buildout.cfg
+  ../../component/scipy/buildout.cfg
+  ../../component/scikit-learn/buildout.cfg
+  ../../component/pandas/buildout.cfg
+  ../../component/openssl/buildout.cfg
+parts =
+  monitor-eggs
+  slapos-cookbook
+  ipython_notebook
+  ipython_notebook_set_password
+  instance
+
+[ipython_notebook]
+; In the ipython notebook software, we use more eggs than in the minimal
+; ipython notebook component
+eggs +=
+  ${scipy:egg}
+  ${pandas:egg}
+  ${scikit-learn:egg}
+
+[ipython_notebook_config]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/${:filename}
+download-only = true
+md5sum = a5bc4ee8539109d1de7ab33b4c2c97ea
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = ipython_notebook_config.jinja
+mode = 0644
+
+[ipython_notebook_set_password]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/${:filename}
+download-only = true
+md5sum = d7d4a7e19d55bf14007819258bf42100
+destination = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = ipython_set_password.jinja
+mode = 0644
+
+[instance]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/instance.cfg
+
+
+
+
+[versions]
+PyRSS2Gen = 1.1
+Pygments = 2.0.2
+cns.recipe.symlink = 0.2.3
+ipython = 3.1.0
+matplotlib = 1.4.3
+mistune = 0.5.1
+nose = 1.3.7
+pandas = 0.16.1
+plone.recipe.command = 1.1
+pyzmq = 14.6.0
+scikit-learn = 0.16.1
+scipy = 0.15.1
+simpy = 3.0.7
+slapos.recipe.template = 2.7
+terminado = 0.5
+tornado = 4.2
+
+# Required by:
+# dream==0.0.1
+MySQL-python = 1.2.5
+
+# Required by:
+# tornado==4.2
+backports.ssl-match-hostname = 3.4.0.2
+
+# Required by:
+# tornado==4.2
+certifi = 2015.4.28
+
+# Required by:
+# matplotlib==1.4.3
+mock = 1.0.1
+
+# Required by:
+# dream==0.0.1
+numpy = 1.9.2
+
+# Required by:
+# terminado==0.5
+ptyprocess = 0.5
+
+# Required by:
+# dream==0.0.1
+pydot = 1.0.28
+
+# Required by:
+# matplotlib==1.4.3
+# pandas==0.16.1
+python-dateutil = 2.4.2
+
+# Required by:
+# dream==0.0.1
+rpy2 = 2.6.0
+
+# Required by:
+# rpy2==2.6.0
+singledispatch = 3.4.0.3
+
+# Required by:
+# dream==0.0.1
+tablib = 0.10.0
+
+# Required by:
+# dream==0.0.1
+xlrd = 0.9.3
+
+# Required by:
+# dream==0.0.1
+xlwt = 1.0.0
+
+# Required by:
+# dream==0.0.1
+zope.dottedname = 4.1.0

software/ipython_notebook/template/ipython_notebook_config.jinja

+
+import ConfigParser
+
+knowledge_0 = '{{ config_cfg }}'
+
+c = get_config()
+
+parser = ConfigParser.ConfigParser()
+parser.read(knowledge_0)
+
+if parser.has_option("ipython_notebook", "password"):
+  c.NotebookApp.password = parser.get("ipython_notebook", "password")
+

software/ipython_notebook/template/ipython_set_password.jinja

+#!{{ python_executable }}
+
+import cgi
+import cgitb
+import ConfigParser
+import os
+import re
+import subprocess
+
+from IPython.lib import passwd
+
+#cgitb.enable(display=0, logdir="/tmp/cgi.log")
+cgitb.enable()
+form = cgi.FieldStorage()
+
+config_file = "{{ config_cfg }}"
+
+
+if not os.path.exists(config_file):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+
+parser = ConfigParser.ConfigParser()
+parser.read(config_file)
+if not parser.has_section("ipython_notebook"):
+  parser.add_section("ipython_notebook")
+
+if not parser.has_option("ipython_notebook", "password"):
+  parser.set("ipython_notebook", "password", "")
+
+if "password" in form:
+  parser.set("ipython_notebook", "password", passwd(form["password"].value))
+  # subprocess.call('{{ httpd_graceful }}')
+  # TODO: we should restart ipython
+
+with open(config_file, 'w') as file:
+  parser.write(file)
+
+# TODO cleanup
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"static/style.css\">"
+print "</head><body>"
+print "<h1>IPython Notebook Password :</h1>"
+print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
+print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
+
+print """<div class="pure-control-group">
+<label for="password">Password*:</label>
+<input placeholder="Set your password" type="password" name="password" id="password"></br>
+</div><div class="pure-control-group">
+<label for="password">Verify Password*:</label>
+<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
+</div><p id="validate-status" style="color:red"></p>
+<div class="pure-controls">
+<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
+</form>
+<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
+<script type="text/javascript" src="static/monitor-register.js"></script>
+</body></html>
+"""
+

software/kvm/common.cfg

@@ -11,6 +11,7 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../component/netcat/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../stack/resilient/buildout.cfg
@@ -49,6 +50,7 @@ eggs =
   erp5.util
   cns.recipe.symlink
   collective.recipe.template
+  plone.recipe.command
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -85,7 +87,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 4c8f07da2217e54163c265fe6fe3d41d
+md5sum = cf67212d3155767d0d0d8a6d75d2d8ad
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -93,7 +95,15 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = ac69266206830226185e576fb6e4935a
+md5sum = 3e3354844b2052609e3c49eca03b607e
+download-only = true
+on-update = true
+
+[template-kvm-cluster]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
+mode = 644
+md5sum = cc72d7b89d8b474d7b4f2c9319b385d5
 download-only = true
 on-update = true
 
@@ -108,7 +118,7 @@ on-update = true
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = 3af2cdbaa7773e639cfdf17db60d2aed
+md5sum = e3d275621420f0b8c081228caeb571f9
 mode = 0644
 download-only = true
 on-update = true
@@ -157,3 +167,33 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
+
+[template-apache-conf]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/apache.conf.in
+mode = 644
+filename = apache.conf.in
+md5sum = 91f05377aff35ffbac7f2687e90b5dcc
+download-only = true
+on-update = true
+
+[template-content]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/template-content.in
+mode = 644
+filename = template-content.in
+md5sum = 47d492dafe5cb314bdc49bf013d21ead
+download-only = true
+on-update = true
+
+[template-httpd]
+recipe = slapos.recipe.template:jinja2
+filename = template-httpd.cfg
+template = ${:_profile_base_location_}/instance-kvm-http.cfg.in
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/instance-kvm-http.cfg
+md5sum = 84b96dfc78e8d2611bf7210b8b6bb9c5
+context =
+    key apache_location apache:location
+    raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
+
+

software/kvm/development.cfg

@@ -6,30 +6,38 @@ extends =
 parts +=
   slapos.cookbook-repository
   slapos.toolbox-repository
+  slapos.core-repository
   erp5.util-repository
   check-recipe
 
 develop =
   ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.core-repository
   ${:parts-directory}/slapos.toolbox-repository
   ${:parts-directory}/erp5.util-repository
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = kvm
+branch = kvm-cluster
+git-executable = ${git:location}/bin/git
+
+[slapos.core-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.core.git
+branch = master
 git-executable = ${git:location}/bin/git
 
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = kvmresiliency
+branch = master
 git-executable = ${git:location}/bin/git
 
 [erp5.util-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
-branch = scalability-master2
+branch = master
 git-executable = ${git:location}/bin/git
 
 [check-recipe]
@@ -38,5 +46,12 @@ stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+
+[versions]
+slapos.cookbook =
+slapos.core =
+slapos.toolbox =
+erp5.util =
\ No newline at end of file

software/kvm/instance-for-erp5testnode.cfg.in

@@ -22,6 +22,13 @@ test = $${dynamic-template-kvm-resilient-test:rendered}
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
+# XXX - If this configuration is not generated by slapgrid, use empty values
+[storage-configuration]
+storage-home = 
+
+[network-information]
+global-ipv4-network = 
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -29,6 +36,7 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
@@ -38,16 +46,22 @@ extensions = jinja2.ext.do
 context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
+    key global_ipv4_prefix network-information:global-ipv4-network
     key slapparameter_dict slap-configuration:configuration
+    key storage_dict slap-configuration:storage-dict
+    key tap_network_dict slap-configuration:tap-network-information-dict
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
     raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
     raw novnc_location ${noVNC:location}
+    raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
     raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
     raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+    raw template_httpd_cfg ${template-httpd:rendered}
+    raw template_content ${template-content:location}/${template-content:filename}
     raw websockify_executable_location ${buildout:directory}/bin/websockify
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}

software/kvm/instance-kvm-cluster-input-schema.json

+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Input Parameters",
+  "properties": {
+    "frontend": {
+      "title": "KVM frontend (web socket)",
+      "description": "Front end used to provide VNC.",
+      "properties": {
+        "frontend-instance-guid": {
+          "title": "Frontend Instance ID",
+          "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+          "type": "string",
+          "default": ""
+        },
+        "frontend-software-type": {
+          "title": "Frontend Software Type",
+          "description": "Type of the frontend instance, like \"frontend\".",
+          "type": "string",
+          "default": "frontend"
+        },
+        "frontend-software-url": {
+          "title": "Frontend Software URL",
+          "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+          "type": "string",
+          "format": "uri",
+          "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg"
+        }
+      },
+      "type": "object"
+    },
+    "slave-frontend": {
+      "title": "Web frontend",
+      "description": "Front end used to provide web access for internal services at the kvm.",
+      "properties": {
+        "instance-guid": {
+          "title": "Main Frontend Instance ID",
+          "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+          "type": "string",
+          "default": ""
+        },
+        "frontend-software-type": {
+          "title": "Frontend Software Type",
+          "description": "Type of the frontend instance, like \"frontend\".",
+          "type": "string",
+          "default": "custom-personal"
+        },
+        "frontend-software-url": {
+          "title": "Frontend Software URL",
+          "description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+          "type": "string",
+          "format": "uri",
+          "default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+        },
+        "slave-frontend-dict": {
+          "title": "Slave Front end definition",
+          "description": "Slave Front end definition",
+          "patternProperties": {
+            ".*": {
+              "properties": {
+                "domain": {
+                  "title": "Name of the domain to be used",
+                  "description": "Name of the domain to be used (example: mydomain.com). Subdomains of this domain will be used for the slave instances (example: instance12345.mydomain.com).",
+                  "type": "string",
+                  "default": ""
+                },
+                "url": {
+                  "title": "URL of backend to use.",
+                  "description": "URL of backend to use. This URL will be used to request frontend if parameter 'kvm-partition-name' is not set.",
+                  "format": "uri",
+                  "type": "string",
+                  "default": ""
+                },
+                "enable-cache": {
+                  "title": "Use cache for this slave frontend.",
+                  "description": "Specify if slave frontend should use a squid to connect to backend.",
+                  "type": "boolean",
+                  "default": false
+                },
+                "type": {
+                  "title": "Specify if slave frontend will redirect to a zope backend.",
+                  "description": "If specified, Apache RewriteRule will use Zope's Virtual Host Daemon. Possible values: 'zope', 'default'.",
+                  "type": "string",
+                  "default": "default"
+                },
+                "zope-path": {
+                  "title": "Path to the VirtualHostRoot of the zope.",
+                  "description": "Only used if type is 'zope'. Will append the specified path to the VirtualHostRoot of the zope's VirtualHostMonster.",
+                  "type": "string",
+                  "default": ""
+                },
+                "https-only": {
+                  "title": "Access website with https url only.",
+                  "description": "Specify if website should be accessed using https only. If so, the frontend will redirect the user to https if accessed from http.",
+                  "type": "boolean",
+                  "default": false
+                },
+                "kvm-partition-name": {
+                  "title": "Reference name in kvm instance definition to get url from.",
+                  "description": "Compose url from kvm definition. Only work if 'use-nat' is true and 'service-port' is set. This will allow to get URL from defined nat-rules. Play the same as url, but help if you don't know kvm ipv6 yet.",
+                  "type": "string",
+                  "default": ""
+                },
+                "service-port": {
+                  "title": "Port of service into the VM (require: kvm-name).",
+                  "description": "This will allow to get URL from defined nat-rules. The port should exist in nat-rules of KVM you have referenced by 'kvm-partition-name'",
+                  "type": "integer"
+                },
+                "url-scheme": {
+                  "title": "Scheme of HTTP service into the VM (require: kvm-name).",
+                  "description": "Say If HTTP service to run/or running into the Virtual Machine will use http or https. Possible values: http, https.",
+                  "type": "string",
+                  "enum": ["http", "https"],
+                  "default": "http"
+                }
+              },
+              "type": "object"
+            }
+          },
+         "type": "object"
+        }
+      },
+      "type": "object"
+    },
+    "authorized-key": {
+      "title": "Public keys for virtual machines.",
+      "description": "Set the list of public keys to add in your virtual machine. The public key file will be available in the VM via url http://10.0.2.100/authorized_keys if you keep the NAT interface enabled",
+      "type": "array"
+    },
+    "kvm-partition-dict": {
+      "title": "kvm instances definition",
+      "description": "kvm instances definition",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "computer-guid": {
+              "title": "ID of the computer where to deploy this VM.",
+              "description": "Unique identifier of the computer, like \"COMP-1234\". By default, let Master choose a computer.",
+              "type": "string",
+              "default": ""
+            },
+            "ram-size": {
+              "title": "RAM size",
+              "description": "RAM size, in MB.",
+              "type": "integer",
+              "default": 1024,
+              "minimum": 128,
+              "multipleOf": 128,
+              "maximum": 16384
+            },
+            "disk-size": {
+              "title": "Disk size",
+              "description": "Disk size, in GB.",
+              "type": "integer",
+              "default": 10,
+              "minimum": 1,
+              "maximum": 1000
+            },
+            "disk-type": {
+              "title": "Disk type",
+              "description": "Type of QEMU disk drive.",
+              "type": "string",
+              "default": "virtio",
+              "enum": [
+                "ide",
+                "scsi",
+                "sd",
+                "mtd",
+                "floppy",
+                "pflash",
+                "virtio"
+              ]
+            },
+            "cpu-count": {
+              "title": "CPU count",
+              "description": "Number of CPU cores.",
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 8
+            },
+            "cpu-options": {
+              "title": "CPU Additional options: cores, threads, sockets, maxcpus.",
+              "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
+              "type": "string"
+            },
+            "numa": {
+              "title": "Simulate a multi node NUMA system.",
+              "description": "Simulate a multi node NUMA system. If mem and cpus are omitted, resources are split equally. Each numa option are separated by space: node,nodeid=4,cpus=40-49,mem=64g node,nodeid=1,cpus=10-19,mem=128g. Set this option if you know what you're doing.",
+              "type": "string"
+            },
+            "nbd-host": {
+              "title": "NBD hostname or IP",
+              "description": "hostname (or IP) of the NBD server containing the boot image.",
+              "type": "string",
+              "format": [
+                "host-name",
+                "ip-address",
+                "ipv6"
+              ],
+              "default": "debian.nbd.vifib.net"
+            },
+            "nbd-port": {
+              "title": "NBD port",
+              "description": "Port of the NBD server containing the boot image.",
+              "type": "integer",
+              "default": 1024,
+              "minimum": 1,
+              "maximum": 65535
+            },
+            "nbd2-host": {
+              "title": "Second NBD hostname or IP",
+              "description": "hostname (or IP) of the second NBD server (containing drivers for example).",
+              "type": "string",
+              "format": [
+                "host-name",
+                "ip-address",
+                "ipv6"
+              ]
+            },
+            "nbd2-port": {
+              "title": "Second NBD port",
+              "description": "Port of the second NBD server containing the boot image.",
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 65535
+            },
+            "virtual-hard-drive-url": {
+              "title": "Existing disk image URL",
+              "description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
+              "format": "uri",
+              "type": "string"
+            },
+            "virtual-hard-drive-md5sum": {
+              "title": "Checksum of virtual hard drive",
+              "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+              "type": "string"
+            },
+            "virtual-hard-drive-gzipped": {
+              "title": "Virtual hard drive to download is gzipped",
+              "description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
+              "type": "boolean",
+              "default": false
+            },
+            "external-disk-number": {
+              "title": "Number of additional disk to create for virtual machine",
+              "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+              "type": "integer",
+              "minimum": 0,
+              "default": 0
+            },
+            "external-disk-size": {
+              "title": "Size of additional disk to create for virtual machine, in Gigabytes",
+              "description": "Specify the size of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+              "type": "integer",
+              "minimum": 10,
+              "maximum": 1000,
+              "default": 20
+            },
+            "external-disk-format": {
+              "title": "Type of external disk drive to create by QEMU.",
+              "description": "Type of QEMU disk drive, to create.",
+              "type": "string",
+              "default": "qcow2",
+              "enum": ["qcow2", "raw", "vdi", "vmdk", "cloop"]
+            },
+            "use-tap": {
+              "title": "Use QEMU TAP network interface",
+              "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
+              "type": "boolean",
+              "default": true
+            },
+            "use-nat": {
+              "title": "Use QEMU USER Mode networking",
+              "description": "Use QEMU user-mode network stack (NAT).",
+              "type": "boolean",
+              "default": true
+            },
+            "nat-rules": {
+              "title": "List of rules for NAT of QEMU user mode network stack.",
+              "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\".",
+              "type": "array",
+              "default": [
+                22,
+                80,
+                443
+              ]
+            }
+          },
+          "type": "object"
+        }
+      },
+      "type": "object"
+    }
+  }
+}

software/kvm/instance-kvm-cluster.cfg.jinja2.in

+{% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set ipv6 = (ipv6 | list)[0] -%}
+{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
+{% set slave_frontend_dict = slapparameter_dict.get('slave-frontend', {}) -%}
+{% set slave_frontend_sr = slave_frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
+{% set slave_frontend_stype = slave_frontend_dict.get('software-type', 'custom-personal') -%}
+{% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
+{% set kvm_instance_dict = {} -%}
+{% set kvm_hostname_list = [] -%}
+
+[request-common]
+recipe = slapos.cookbook:request
+software-url = ${slap-connection:software-release-url}
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
+
+# Request kvm instances
+{% for instance_name, kvm_parameter_dict in slapparameter_dict.get('kvm-partition-dict', {'kvm-default': {}}).items() -%}
+{% set section = 'request-' ~ instance_name -%}
+{% set use_nat = kvm_parameter_dict.get('use-nat', True) -%}
+[{{ section }}]
+<= request-common
+software-type = kvm
+name = {{ instance_name }}
+sla-computer_guid = {{ dumps(kvm_parameter_dict.get('computer-guid', '')) }}
+
+config-frontend-instance-name = {{ instance_name ~ ' VNC Frontend' }}
+config-frontend-software-type = {{ dumps(frontend_dict.get('software-type', 'frontend')) }}
+config-frontend-software-url = {{ dumps(frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg')) }}
+config-frontend-instance-guid = {{ dumps(frontend_dict.get('instance-guid', '')) }}
+config-name = {{ instance_name }}
+{% if slapparameter_dict.get('authorized-key', []) -%}
+config-authorized-key = {{ slapparameter_dict.get('authorized-key') | join('##') }}
+{% endif -%}
+config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
+config-nbd-host = {{ dumps(kvm_parameter_dict.get('nbd-host', '')) }}
+config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
+config-nbd2-host = {{ dumps(kvm_parameter_dict.get('nbd-host2', '')) }}
+config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
+config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
+config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
+config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
+config-cpu-options = {{ dumps(kvm_parameter_dict.get('cpu-options', '')) }}
+config-numa = {{ dumps(kvm_parameter_dict.get('numa', '')) }}
+
+{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', [22, 80, 443]) -%}
+config-nat-rules = {{ nat_rules_list | join(' ') }}
+config-publish-nat-url = True
+config-use-nat = {{ use_nat }}
+config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
+config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
+config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
+config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
+config-external-disk-number = {{ dumps(kvm_parameter_dict.get('external-disk-number', 0)) }}
+config-external-disk-size = {{ dumps(kvm_parameter_dict.get('external-disk-size', 20)) }}
+config-external-disk-format = {{ dumps(kvm_parameter_dict.get('external-disk-format', 'qcow2')) }}
+config-enable-http-server = {{ dumps(kvm_parameter_dict.get('enable-http-server', True)) }}
+config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
+
+# Enable simple http server on ipv6 so all VMs will access it
+config-document-host = ${http-server:host}
+config-document-port = ${http-server:port}
+config-document-path = ${http-server:path}
+
+return = 
+  backend-url
+  url
+{% if use_nat -%}
+{%   for port in nat_rules_list -%}
+{{ '  ' }}nat-rule-url-{{ port }}
+{%   endfor -%}
+{% endif -%}
+{% if kvm_parameter_dict.get('use-tap', True) -%}
+{{ '  ' }}tap-ipv4
+
+{% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
+{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
+{% endif -%}
+{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
+{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
+{% do kvm_instance_dict.__setitem__(instance_name, (kvm_parameter_dict.get('use-nat', True), nat_rules_list)) -%}
+{% endfor %}
+
+
+#request custom kvm frontend
+{% for frontend_name, frontend_parameter_dict in slave_frontend_dict.get('slave-frontend-dict', {}).items() -%}
+{%   set name = frontend_name -%}
+{%   set url = frontend_parameter_dict.get('url', '') -%}
+{%   set error = '' -%}
+{%   if frontend_parameter_dict.get('kvm-partition-name', '') != '' -%}
+{%     set kvm_name = frontend_parameter_dict['kvm-partition-name'] -%}
+{%     set service_port = frontend_parameter_dict['service-port'] -%}
+{%     if kvm_name in kvm_instance_dict.keys()  and not kvm_instance_dict[kvm_name][0] -%}
+{%       set error = "You should set parameter use-nat to 'true' for '" ~ kvm_name ~ "', or provide url to use for frontend." -%}
+{%     elif kvm_name in kvm_instance_dict.keys() and service_port in kvm_instance_dict[kvm_name][1] -%}
+{%       set url = '${request-' ~ kvm_name ~ ':connection-nat-rule-url-' ~ service_port ~ '}' -%}
+{%       set url = frontend_parameter_dict.get('url-scheme', 'http') ~ '://' ~ url -%}
+{%     else -%}
+{%       set error = kvm_name ~ " and/or port " ~ service_port ~ " doesn't match any KVM name and/or related nat-rules in your request parameters." -%}
+{%     endif -%}
+{%   endif -%}
+{%   set section = 'request-' ~ name ~ '-slave-frontend' -%}
+[{{ section }}]
+<= request-common
+recipe = slapos.cookbook:request
+software-url = {{ slave_frontend_sr }}
+name = Frontend {{ name }}
+software-type = {{ slave_frontend_stype }}
+slave = true
+config-url = {{ url }}
+config-custom_domain = {{ dumps(frontend_parameter_dict.get('domain', '')) }}
+config-enable_cache = {{ dumps(frontend_parameter_dict.get('enable-cache', False)) }}
+config-https-only = {{ dumps(frontend_parameter_dict.get('https-only', False)) }}
+{%  if frontend_parameter_dict.get('type', '') -%}
+config-type = {{ dumps(frontend_parameter_dict['type']) }}
+{%    if frontend_parameter_dict.get('path', '') -%}
+config-path = {{ dumps(frontend_parameter_dict['path']) }}
+{%    endif -%}
+{%  endif -%}
+return = 
+  site_url
+sla-instance_guid = {{ slave_frontend_iguid }}
+{%   do publish_dict.__setitem__(name ~ '-url', '${' ~ section ~ ':connection-site_url}') -%}
+{%   if error != '' -%}
+{%     do publish_dict.__setitem__('1_error', error) -%}
+{%   endif -%}
+{% endfor %}
+
+# Enable simple http server on ipv6 so all VMs will access it
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+document = ${:srv}/document
+ssl = ${:etc}/ssl
+
+[http-ssl]
+recipe = plone.recipe.command
+command = "{{ openssl_executable_location }}" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
+key = ${directory:ssl}/key
+cert = ${directory:ssl}/cert
+update-command = 
+stop-on-error = true
+
+[http-server]
+recipe = slapos.cookbook:simplehttpserver
+host = {{ ipv6 }}
+port = 9002
+base-path = ${directory:document}
+wrapper = ${directory:services}/simple-http-server
+log-file = ${directory:log}/http.log
+cert-file = ${http-ssl:cert}
+key-file = ${http-ssl:key}
+
+[write-vm-hostname]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_content }}
+filename = hosts
+rendered = ${http-server:root-dir}/${:filename}
+context =
+    raw content_list {{ kvm_hostname_list | join('#') }}
+    raw sep #
+
+[publish]
+recipe = slapos.cookbook:publish
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
+{% endfor %}
+[buildout]
+parts = 
+  http-server
+  write-vm-hostname
+  publish
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+ 
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
\ No newline at end of file

software/kvm/instance-kvm-http.cfg.in

+[buildout]
+
+parts = 
+  httpd
+  httpd-promise
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+public = ${:srv}/public/
+log = ${:var}/log
+services = ${:etc}/service
+promises = ${:etc}/promise
+run = ${:var}/run
+
+[apache-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_apache_conf }}
+rendered = ${directory:etc}/apache.conf
+#ipv6 = ${slap-network-information:global-ipv6}
+ipv4 = ${slap-network-information:local-ipv4}
+port = ${slap-parameter:httpd-port}
+error-log = ${directory:log}/apache-error.log
+access-log = ${directory:log}/apache-access.log
+pid-file = ${directory:run}/apache.pid
+index = ${directory:public}
+context = 
+  key port :port
+  key ip :ipv4
+  key access_log :access-log
+  key error_log :error-log
+  key pid_file :pid-file
+  key index_folder :index
+
+[httpd]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:services}/httpd
+command-line = "{{ apache_location }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
+
+[httpd-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/apache-httpd
+hostname = ${apache-conf:ipv4}
+port = ${apache-conf:port}
\ No newline at end of file

software/kvm/instance-kvm-input-schema.json

@@ -19,7 +19,7 @@
       "type": "integer",
       "default": 10,
       "minimum": 1,
-      "maximum": 80
+      "maximum": 1000
     },
     "disk-type": {
       "title": "Disk type",
@@ -36,6 +36,16 @@
       "minimum": 1,
       "maximum": 8
     },
+    "cpu-options": {
+      "title": "Additional options (cores, threads, sockets, maxcpus) to use with cpu-count.",
+      "description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Only set this option if you really know what you're doing.",
+      "type": "string"
+    },
+    "numa": {
+      "title": "Simulate a multi node NUMA system. If mem and cpus are omitted, resources are split equally.",
+      "description": "Simulate a multi node NUMA system. If mem and cpus are omitted, resources are split equally. Each numa option are separated by space: node,nodeid=4,cpus=40-49,mem=64g node,nodeid=1,cpus=10-19,mem=128g. Only set this option if you really know what you're doing.",
+      "type": "string"
+    },
 
     "nbd-host": {
       "title": "NBD hostname",
@@ -78,18 +88,58 @@
       "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
       "type": "string"
     },
+    "virtual-hard-drive-gzipped": {
+      "title": "Define if virtual hard drive to download is gzipped",
+      "description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
+      "type": "boolean",
+      "default": false
+    },
+
+    "external-disk-number": {
+      "title": "Number of additional disk to create for virtual machine",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 0,
+      "default": 0
+    },
+    "external-disk-size": {
+      "title": "Number of additional disk to create for virtual machine, in Gigabytes",
+      "description": "Specify the number of additional disk to create for virtual machine in data folder of SlapOS Node. Requires instance_storage_home to be configured on SlapOS Node.",
+      "type": "integer",
+      "minimum": 5,
+      "maximum": 1000,
+      "default": 20
+    },
+    "external-disk-format": {
+      "title": "Type of external disk drive to create by QEMU.",
+      "description": "Type of QEMU disk drive, to create.",
+      "type": "string",
+      "default": "qcow2",
+      "enum": ["qcow2", "raw", "vdi", "vmdk", "cloop"]
+    },
 
     "use-tap": {
       "title": "Use QEMU TAP network interface",
-      "description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
+      "description": "Use QEMU TAP network interface, might require a bridge on SlapOS Node.",
       "type": "boolean",
       "default": false
     },
+    "use-nat": {
+      "title": "Use QEMU USER Mode networking",
+      "description": "Use QEMU user-mode network stack (NAT).",
+      "type": "boolean",
+      "default": true
+    },
     "nat-rules": {
       "title": "List of rules for NAT of QEMU user mode network stack.",
       "description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
       "type": "string"
     },
+    "authorized-key": {
+      "title": "Public keys to get from all virtual machines.",
+      "description": "Set the public keys to add in your virtual machine. Keys are separated with '##'. The public key file will be available in the VM via url http://10.0.2.100/authorized_keys if you keep the NAT interface enabled",
+      "type": "string"
+    },
     "frontend-instance-guid": {
       "title": "Frontend Instance ID",
       "description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",

software/kvm/instance-kvm-resilient-input-schema.json

 {
     "type": "object",
     "$schema": "http://json-schema.org/draft-04/schema",
-    "items": {
     "allOf": [
         {
-                "$ref": "instance-kvm-input-schema.json"
-            }
-        ],
-        "title": "Input Parameters",
+            "$ref": "instance-kvm-input-schema.json#/"
+        },
+        {
             "properties": {
                 "-sla-0-computer_guid": {
                     "title": "Target computer for main instance",
                     "description": "Target computer GUID for main instance.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "-sla-1-computer_guid": {
                     "title": "Target computer for first clone",
                     "description": "Target computer for first clone and PBS.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "-sla-2-computer_guid": {
                     "title": "Target computer for second clone",
                     "description": "Target computer for second clone and PBS.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "resiliency-backup-periodicity": {
                     "title": "Periodicity of backup",
                     "description": "Periodicity of backup, in cron format.",
-                "type": "string"
+                    "type": "string",
+                    "optional": true
                 },
                 "remove-backup-older-than": {
                     "title": "Remove backups older than...",
                     "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                     "type": "string",
-                "default": "3B"
+                    "default": "3B",
+                    "optional": true
                 },
                 "resilient-clone-number": {
                     "title": "Amount of backup(s) to create",
                     "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                     "type": "integer",
-                "default": 2
+                    "default": 2,
+                    "optional": true
                 },
                 "ignore-known-hosts-file": {
                     "title": "Ignore known_hosts file",
                     "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, webrunner).",
                     "type": "boolean",
-                "default": false
+                    "default": false,
+                    "optional": true
                 }
             }
         }
+    ]
 }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -66,5 +66,5 @@ return = ipv6
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]
-virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/fbd4ad.php?dl=true
+virtual-hard-drive-url = http://www.nexedi.org/static/slapos/kvm_resiliency_test/virtual.qcow2
 virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b

software/kvm/instance-kvm.cfg.jinja2

+{% set enable_http = slapparameter_dict.get('enable-http-server', 'False') -%}
 #############################
 #
 # Instanciate kvm
@@ -15,6 +16,17 @@ parts =
   cron
 #  cron-entry-monitor
   frontend-promise
+{% if slapparameter_dict.get('document-host', '') %}
+  cluster-url-path
+{% endif -%}
+{% if slapparameter_dict.get('enable-http-server', 'False') == 'True' %}
+  httpd
+  httpd-promise
+  publish-host-config
+  
+extends = 
+  {{ template_httpd_cfg }}
+{% endif -%}
 
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
@@ -33,6 +45,7 @@ promises = ${:etc}/promise
 novnc-conf = ${:etc}/novnc
 run = ${:var}/run
 ca-dir = ${:srv}/ssl
+public = ${:srv}/public/
 cron-entries = ${:etc}/cron.d
 crontabs = ${:etc}/crontabs
 cronstamps = ${:etc}/cronstamps
@@ -41,6 +54,10 @@ cronstamps = ${:etc}/cronstamps
 recipe = slapos.cookbook:generate.mac
 storage-path = ${directory:srv}/mac
 
+[create-tap-mac]
+recipe = slapos.cookbook:generate.mac
+storage-path = ${directory:srv}/tap_mac
+
 [gen-passwd]
 recipe = slapos.cookbook:generate.password
 storage-path = ${directory:srv}/passwd
@@ -77,25 +94,54 @@ socket-path = ${directory:var}/qmp_socket
 pid-file-path = ${directory:run}/pid_file
 
 smp-count = ${slap-parameter:cpu-count}
+smp-options = ${slap-parameter:cpu-options}
 ram-size = ${slap-parameter:ram-size}
+numa = ${slap-parameter:numa}
 mac-address = ${create-mac:mac-address}
+tap-mac-address = ${create-tap-mac:mac-address}
 
 # XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
 runner-path = ${directory:services}/kvm
 controller-path = ${directory:scripts}/kvm_controller
 
 use-tap = ${slap-parameter:use-tap}
+use-nat = ${slap-parameter:use-nat}
 nat-rules = ${slap-parameter:nat-rules}
 6tunnel-wrapper-path = ${directory:services}/6tunnel
 
 virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
+virtual-hard-drive-gzipped = ${slap-parameter:virtual-hard-drive-gzipped}
 
 shell-path = {{ dash_executable_location }}
 qemu-path =  {{ qemu_executable_location }}
 qemu-img-path = {{ qemu_img_executable_location }}
 6tunnel-path = {{ sixtunnel_executable_location }}
 
+etc-directory = ${directory:etc}
+disk-storage-list = 
+{% for key, path in storage_dict.items() -%}
+{{ '  ' ~ key ~ ' ' ~ path }}
+{% endfor -%}
+external-disk-number = ${slap-parameter:external-disk-number}
+external-disk-size = ${slap-parameter:external-disk-size}
+external-disk-format = ${slap-parameter:external-disk-format}
+
+{% if slapparameter_dict.get('enable-http-server', 'False') == 'True' or ( slapparameter_dict.get('use-tap', 'False') == 'True' and tap_network_dict.has_key('ipv4') ) -%}
+httpd-port = ${slap-parameter:httpd-port}
+{% else -%}
+httpd-port = 0
+{% endif -%}
+
+# Main instance document server info
+{% if slapparameter_dict.get('document-host', '') and slapparameter_dict.get('document-port', '') -%}
+cluster-doc-host = ${tunnel-cluster-url:ipv4}
+cluster-doc-port = ${tunnel-cluster-url:ipv4-port}
+{% else -%}
+cluster-doc-host = 
+cluster-doc-port = 0
+{% endif -%}
+netcat-binary = {{ netcat_bin }}
 
 [kvm-vnc-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -107,7 +153,16 @@ port = ${kvm-instance:vnc-port}
 # Check that disk image is not corrupted
 recipe = collective.recipe.template
 input = inline:#!/bin/sh
+  # Return code 0 is "OK"
+  # Return code 3 is "found leaks, but image is OK"
+  # http://git.qemu.org/?p=qemu.git;a=blob;f=qemu-img.c;h=4e9a7f5741c9cb863d978225829e68fefcae3947;hb=HEAD#l702
   ${kvm-instance:qemu-img-path} check ${kvm-instance:disk-path}
+  RETURN_CODE=$?
+  if [ $RETURN_CODE -eq 0 ] || [ $RETURN_CODE -eq 3 ]; then
+    exit 0
+  else
+    exit 1
+  fi
 output = ${directory:promises}/kvm-disk-image-corruption
 mode = 700
 
@@ -193,7 +248,7 @@ key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
-name = VNC Frontend
+name = ${slap-parameter:frontend-instance-name}
 software-type = ${slap-parameter:frontend-software-type}
 slave = true
 config-host = ${novnc-instance:ip}
@@ -211,22 +266,127 @@ curl_path = {{ curl_executable_location }}
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish
+ipv6 = ${slap-network-information:global-ipv6}
 backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc_auto.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1&password=${kvm-instance:vnc-passwd}
 url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slave-frontend:connection-domainname}&port=${request-slave-frontend:connection-port}&encrypt=1&path=${request-slave-frontend:connection-resource}&password=${kvm-instance:vnc-passwd}
+{% set disk_number = len(storage_dict) -%}
+maximum-extra-disk-amount = {{ disk_number }}
+{% set iface = 'eth0' -%}
+{% if slapparameter_dict.get('use-nat', 'True') == 'True' -%}
+{%   set iface = 'eth1' -%}
 # Publish NAT port mapping status
 # XXX: hardcoded value from [slap-parameter]
 {%   set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') %}
 {%   for port in nat_rule_list.split(' ') -%}
-  {% set external_port = 10000 + port|int() -%}
-  nat-rule-port-{{port}} = ${slap-network-information:global-ipv6} : {{external_port}}
+{%     set external_port = 10000 + port|int() -%}
+nat-rule-port-{{port}} = ${slap-network-information:global-ipv6} : {{external_port}}
+{%     if slapparameter_dict.get('publish-nat-url', False) -%}
+nat-rule-url-{{port}} = [${slap-network-information:global-ipv6}]:{{external_port}}
+{%     endif -%}
 {%   endfor -%}
-
+{% endif -%}
+{% if slapparameter_dict.get('use-tap', 'False') == 'True' -%}
+tap-ipv4 = ${slap-network-information:tap-ipv4}
+{% endif -%}
+
+{% set kvm_http = 'http://${slap-network-information:local-ipv4}:' ~ slapparameter_dict.get('httpd-port', 8081) -%}
+{% if enable_http == 'True' %}
+{%   if slapparameter_dict.get('use-nat', 'True') == 'True' -%}
+{%     set kvm_http = 'http://10.0.2.100' -%}
+{%   endif %}
+{%   if slapparameter_dict.get('authorized-key', '') -%}
+7_info = Get the publick key file in your VM with the command: wget {{ kvm_http }}/${get-authorized-key:filename}
+{%   endif %}
+{% endif %}
+
+{% if slapparameter_dict.get('use-tap', 'False') == 'True' and tap_network_dict.has_key('ipv4') -%}
+1_info = Use these configurations below to configure interface {{ iface }} in your VM.
+2_info = ${network-config:ifconfig}
+3_info = ${network-config:route-iface}
+4_info = ${network-config:route-network}
+5_info = ${network-config:route-default}
+{% if enable_http == 'True' %}
+6_info = Or run in your VM the command: wget -O- {{ kvm_http }}/netconfig.sh | /bin/sh -
+{% endif %}
+
+[network-config]
+recipe = plone.recipe.command
+path = ${directory:public}/netconfig.sh
+ifconfig = ifconfig {{ iface }} ${slap-network-information:tap-ipv4} netmask ${slap-network-information:tap-netmask}
+route-iface = route add ${slap-network-information:tap-gateway} dev {{ iface }}
+route-network = route add -net ${slap-network-information:tap-network} netmask ${slap-network-information:tap-netmask} gw ${slap-network-information:tap-gateway}
+{%   if iface == 'eth0' -%}
+route-default = route add default gw ${slap-network-information:tap-gateway}
+{%   elif global_ipv4_prefix -%}
+route-default = ip route add {{ global_ipv4_prefix }} via ${slap-network-information:tap-gateway} dev {{ iface }} src ${slap-network-information:tap-ipv4}
+{% else -%}
+route-default =
+{%   endif -%}
+command = 
+  echo "#!/bin/sh" > ${:path}
+  echo "" >> ${:path}
+  echo "${:ifconfig}" >> ${:path}
+  echo "${:route-iface}" >> ${:path}
+  echo "${:route-network}" >> ${:path}
+  echo "${:route-default}" >> ${:path}
+update-command = ${:command}
+{% endif -%}
+
+[get-authorized-key]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_content }}
+filename = authorized_keys
+rendered = ${directory:public}/${:filename}
+public-key = {{ slapparameter_dict.get('authorized-key', '') }}
+context =
+    key content_list :public-key
+    raw sep ##
+
+[publish-host-config]
+recipe = plone.recipe.command
+name = {{ slapparameter_dict.get('name', 'localhost') }}
+{% if slapparameter_dict.get('use-tap', 'False') == 'True' and tap_network_dict.has_key('ipv4') -%}
+local-ipv4 = ${slap-network-information:tap-ipv4}
+{% else -%}
+local-ipv4 = 127.0.0.1
+{% endif -%}
+path-host = ${directory:public}/hostname
+path-ip = ${directory:public}/ipv4
+command = 
+  rm -f ${:path-host}
+  rm -f ${:path-ip}
+  echo "${:name}" > ${:path-host}
+  echo "${:local-ipv4}" > ${:path-ip}
+update-command = ${:command}
+
+# To access documents of main instance (in case of kvm-cluster) through http
+[cluster-url-path]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_content }}
+filename = cluster.hash
+rendered = ${directory:public}/${:filename}
+hash-url = https://10.0.2.101:443/{{ slapparameter_dict.get('document-path', '') }}
+context =
+    key content_list :hash-url
+    raw sep #
+
+# This 6to4 tunnel help to access document url in ipv4
+[tunnel-cluster-url]
+recipe = slapos.cookbook:ipv4toipv6
+ipv6 = {{ slapparameter_dict.get('document-host', '') }}
+ipv4 = ${slap-network-information:local-ipv4}
+ipv6-port = {{ slapparameter_dict.get('document-port', '') }}
+ipv4-port = 16936
+shell-path = {{ dash_executable_location }}
+6tunnel-path = {{ sixtunnel_executable_location }}
+runner-path = ${directory:services}/6tunnel-cluster
 
 [slap-parameter]
 # Default values if not specified
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
 frontend-instance-guid =
+frontend-instance-name = VNC Frontend
 nbd-port = 1024
 nbd-host =
 nbd2-port = 1024
@@ -237,9 +397,25 @@ disk-size = 10
 disk-type = virtio
 
 cpu-count = 1
+# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
+cpu-options = 
+# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
+numa = 
 
 nat-rules = 22 80 443
+use-nat = True
 use-tap = False
 
 virtual-hard-drive-url =
 virtual-hard-drive-md5sum =
+virtual-hard-drive-gzipped = False
+
+external-disk-number = 0
+external-disk-size = 20
+external-disk-format = qcow2
+
+# Help to get some configuration files into the vm from http
+enable-http-server = False
+httpd-port = 8081
+# for auto config, the public key file will be available in the VM via url http://10.0.2.100/authorized_key if use-nat = True
+authorized-key = 

software/kvm/instance.cfg.in

@@ -8,6 +8,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${:kvm}
+kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
 kvm = $${dynamic-template-kvm:rendered}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
@@ -22,6 +23,13 @@ test = $${dynamic-template-kvm-resilient-test:rendered}
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
 
+# XXX - If this configuration is not generated by slapgrid, use empty values
+[storage-configuration]
+storage-home = 
+  
+[network-information]
+global-ipv4-network = 
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -29,6 +37,36 @@ partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
+storage-home = $${storage-configuration:storage-home}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = $${buildout:directory}/$${:filename}
+extensions = jinja2.ext.do
+mode = 0644
+extra-context =
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key ipv4 slap-configuration:ipv4
+    key ipv6 slap-configuration:ipv6
+    key global_ipv4_prefix network-information:global-ipv4-network
+    key tap_network_dict slap-configuration:tap-network-information-dict
+    key storage_dict slap-configuration:storage-dict
+    key slapparameter_dict slap-configuration:configuration
+    key computer_id slap-configuration:computer
+    raw openssl_executable_location ${openssl:location}/bin/openssl
+    $${:extra-context}
+
+[dynamic-template-kvm-cluster-parameters]
+
+[dynamic-template-kvm-cluster]
+<= jinja2-template-base
+template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
+filename = template-kvm-cluster.cfg
+extra-context =
+  section parameter_dict dynamic-template-kvm-cluster-parameters
+  raw template_content ${template-content:location}/${template-content:filename}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
@@ -38,16 +76,22 @@ extensions = jinja2.ext.do
 context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
+    key global_ipv4_prefix network-information:global-ipv4-network
     key slapparameter_dict slap-configuration:configuration
+    key storage_dict slap-configuration:storage-dict
+    key tap_network_dict slap-configuration:tap-network-information-dict
     raw curl_executable_location ${curl:location}/bin/curl
     raw dash_executable_location ${dash:location}/bin/dash
     raw dcron_executable_location ${dcron:location}/sbin/crond
     raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
     raw novnc_location ${noVNC:location}
+    raw netcat_bin ${netcat:location}/bin/netcat
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
     raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
     raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+    raw template_httpd_cfg ${template-httpd:rendered}
+    raw template_content ${template-content:location}/${template-content:filename}
     raw websockify_executable_location ${buildout:directory}/bin/websockify
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}

software/kvm/software-for-erp5testnode.cfg

@@ -5,6 +5,6 @@ extends = development.cfg
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in
-md5sum = e8afd5aa5b41df79238e9a84984a6aa5
+md5sum = e0af93ba7209cabd5db6d9afcb15c2aa
 output = ${buildout:directory}/template.cfg
 mode = 0644
\ No newline at end of file

software/kvm/software.cfg

@@ -2,46 +2,44 @@
 extends = common.cfg
 
 [versions]
+# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
+websockify = 0.5.1
+
 PyRSS2Gen = 1.1
-apache-libcloud = 0.15.1
-async = 0.6.1
+apache-libcloud = 0.16.0
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 ecdsa = 0.11
-erp5.util = 0.4.41
-gitdb = 0.5.4
+erp5.util = 0.4.42
+gitdb = 0.6.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.4
-smmap = 0.8.2
-websockify = 0.6.0
+slapos.recipe.template = 2.7
+slapos.toolbox = 0.46.1
+smmap = 0.9.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# websockify==0.6.0
-numpy = 1.8.2
-
-# Required by:
-# slapos.toolbox==0.40.4
-GitPython = 0.3.2.RC1
+# slapos.toolbox==0.46.1
+GitPython = 0.3.6
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.46.1
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.46.1
 feedparser = 5.1.3
 
 # Required by:
-# slapos.cookbook==0.87
-jsonschema = 2.4.0
+# slapos.toolbox==0.46.1
+lockfile = 0.10.2
 
 # Required by:
-# websockify==0.6.0
-numpy = 1.9.0
+# websockify==0.5.1
+numpy = 1.9.1
 
 # Required by:
-# slapos.toolbox==0.40.4
-paramiko = 1.15.1
+# slapos.toolbox==0.46.1
+paramiko = 1.15.2

software/kvm/software.cfg.json

+{
+    "name": "KVM",
+    "description": "KVM",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Standalone KVM",
+            "request": "instance-kvm-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 0
+        },
+        "kvm-resilient": {
+            "title": "Resilient",
+            "description": "Resilient KVM",
+            "request": "instance-kvm-resilient-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 1
+        },
+        "kvm-cluster": {
+            "title": "Cluster",
+            "description": "Cluster KVM",
+            "serialisation": "json-in-xml",
+            "request": "instance-kvm-cluster-input-schema.json",
+            "response": "instance-kvm-output-schema.json",
+            "index": 2
+        }
+    }
+}

software/kvm/template/apache.conf.in

+ServerLimit 2
+StartServers 1
+MaxClients 2
+
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule mime_module modules/mod_mime.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule autoindex_module modules/mod_autoindex.so
+
+Listen {{ ip }}:{{ port }}
+
+PidFile "{{ pid_file }}"
+ServerAdmin admin@
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+ServerTokens Prod
+ServerSignature Off
+TraceEnable Off
+
+ErrorLog "{{ error_log }}"
+# Default apache log format with request time in microsecond at the end
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+CustomLog "{{ access_log }}" combined
+SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+
+# Directory protection
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+    Require all denied
+</Directory>
+
+DocumentRoot {{ index_folder }}
+<Directory {{ index_folder }}>
+  Options Indexes FollowSymLinks
+  Require ip {{ ip }}
+  # Require env forwarded '{{ ip }}'
+  Require all denied
+</Directory>
\ No newline at end of file

software/kvm/template/template-content.in

+{% for content in content_list.split(sep) -%}
+{{ content }}
+{% endfor -%}
\ No newline at end of file

software/monitor/software.cfg

@@ -6,6 +6,7 @@ extends =
   ../../component/rsync/buildout.cfg
   ../../component/smartmontools/buildout.cfg
   ../../component/lmsensors/buildout.cfg
+  ../../component/fluentd/buildout.cfg
 
 parts = 
   slapos-cookbook
@@ -15,6 +16,7 @@ parts =
   rsync
   smartmontools
   lmsensors
+  fluentd
   
 [template]
 recipe = slapos.recipe.template 
@@ -76,7 +78,7 @@ eggs =
 PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7
 z3c.recipe.scripts = 1.0.1
 
 apache-libcloud = 0.16.0
@@ -100,4 +102,4 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.45.1
-paramiko = 1.15.1
+paramiko = 1.15.2

software/nginx-push-stream/software.cfg

@@ -43,4 +43,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/powerdns/software.cfg

@@ -64,4 +64,4 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/re6stnet/apache.conf.in

+LoadModule unixd_module modules/mod_unixd.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule mime_module modules/mod_mime.so
+#LoadModule dav_module modules/mod_dav.so
+#LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule negotiation_module modules/mod_negotiation.so
+LoadModule rewrite_module modules/mod_rewrite.so
+LoadModule headers_module modules/mod_headers.so
+
+PidFile "{{ pid_file }}"
+ServerAdmin admin@
+TypesConfig conf/mime.types
+AddType application/x-compress .Z
+AddType application/x-gzip .gz .tgz
+
+ServerTokens Prod
+ServerSignature Off
+TraceEnable Off
+
+ErrorLog "{{ error_log }}"
+# Default apache log format with request time in microsecond at the end
+LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+CustomLog "{{ access_log }}" combined
+
+{% if uri_scheme == 'https'  -%}
+# SSL Configuration
+SSLCertificateFile {{ certificate }}
+SSLCertificateKeyFile {{ key }}
+SSLRandomSeed startup builtin
+SSLRandomSeed connect builtin
+SSLProtocol ALL -SSLv2
+{% endif -%}
+
+<Directory />
+  Options FollowSymLinks
+  AllowOverride None
+  Allow from all
+</Directory>
+
+Listen {{ ipv6 }}:{{ apache_port }}
+<VirtualHost *:{{ apache_port }}>
+{% if uri_scheme == 'https'  -%}
+  SSLEngine On
+  SSLProxyEngine On
+{% endif -%}
+
+  ProxyPass / http://{{ re6st_ipv4 }}:{{ re6st_port }}/
+</VirtualHost>
\ No newline at end of file

software/re6stnet/instance-logrotate-base.cfg.in

+[buildout]
+parts =
+  cron-entry-logrotate
+
+[cron]
+recipe = slapos.cookbook:cron
+cron-entries = ${logrotate-directory:cron-entries}
+dcrond-binary = {{ dcron_location }}/sbin/crond
+crontabs = ${logrotate-directory:crontabs}
+cronstamps = ${logrotate-directory:cronstamps}
+catcher = ${cron-simplelogger:wrapper}
+binary = ${logrotate-directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = ${logrotate-directory:bin}/cron_simplelogger
+log = ${logrotate-directory:log}/cron.log
+
+[logrotate]
+recipe = slapos.cookbook:logrotate
+logrotate-entries = ${logrotate-directory:logrotate-entries}
+backup = ${logrotate-directory:logrotate-backup}
+logrotate-binary = {{ logrotate_location }}/usr/sbin/logrotate
+gzip-binary = {{ gzip_location }}/bin/gzip
+gunzip-binary = {{ gzip_location }}/bin/gunzip
+wrapper = ${logrotate-directory:bin}/logrotate
+conf = ${logrotate-directory:etc}/logrotate.conf
+state-file = ${logrotate-directory:srv}/logrotate.status
+
+[cron-entry-logrotate]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = logrotate
+frequency = 0 0 * * *
+command = ${logrotate:wrapper}
+
+[logrotate-directory]
+recipe = slapos.cookbook:mkdirectory
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+crontabs = ${:etc}/crontabs
+logrotate-backup = ${:backup}/logrotate
+logrotate-entries = ${:etc}/logrotate.d
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+backup = ${:srv}/backup
+etc = ${buildout:directory}/etc
+services = ${:etc}/service
+log = ${buildout:directory}/var/log
+

software/re6stnet/instance-re6stnet-input-schema.json

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "properties": {
+        "ipv6-prefix": {
+            "title": "Ipv6 prefix to use to setup the new re6st network",
+            "description": "Prefix ipv6 used by re6st to setup network. It is something like 2001:db8:42::/48",
+            "type": "string"
+        },
+        "key-size": {
+            "title": "Number of bit to use for certificate generation",
+            "description": "Specify the size of certificate generated by re6st. by default, generate 2048-bit key length",
+            "type": "integer",
+            "minimum": 1024,
+            "default": 2048
+        },
+        "prefix-length": {
+            "title": "Default length of allocated prefixes.",
+            "description": "Default length of allocated prefixes.",
+            "type": "integer",
+            "default": 16
+        },
+        "anonymous-prefix-length": {
+            "title": "Length of allocated anonymous prefixes.",
+            "description": "Length of allocated anonymous prefixes. 0 is unset.",
+            "type": "integer",
+            "default": 0
+        },
+        "mailhost": {
+            "title": "SMTP host to send confirmation emails.",
+            "description": "SMTP host to send confirmation emails. Not needed if when token is requested from slave instances.",
+            "type": "string",
+            "default": "127.0.0.1"
+        },
+        "ipv4-net": {
+            "title": "Enable ipv4 (ip/NET P_LENGTH).",
+            "description": "Enable ipv4 (ip/NET P_LENGTH). Each node is assigned a subnet of length PLEN, inside network IP/N. Ex: 10.42.0.0/16 8",
+            "type": "string",
+            "default": ""
+        },
+        "client-count": {
+            "title": "Number of client tunnels to set up.",
+            "description": "Number of client tunnels to set up.",
+            "type": "integer",
+            "default": 10
+        },
+        "tunnel-refresh": {
+            "title": "Interval in seconds between two tunnel refresh.",
+            "description": "Interval in seconds between two tunnel refresh: the worst tunnel is closed if the number of client tunnels has reached its maximum number (client-count).",
+            "type": "integer",
+            "default": 300
+        },
+        "max-clients": {
+            "title": "Maximum number of accepted clients per OpenVPN server.",
+            "description": "Maximum number of accepted clients per OpenVPN server. (if unset or 0: client-count * 2, which actually represents the average number of tunnels to other peers)",
+            "type": "integer",
+            "default": 0
+        },
+        "hello": {
+            "title": "Hello interval in seconds, for both wired and wireless connections.",
+            "description": "Hello interval in seconds, for both wired and wireless connections. OpenVPN ping-exit option is set to 4 times the hello interval. It takes between 3 and 4 times the hello interval for Babel to re-establish connection with a node for which the direct connection has been cut.",
+            "type": "integer",
+            "default": 15
+        },
+        "min-protocol": {
+            "title": "Reject nodes that are too old.",
+            "description": "Reject nodes that are too old. Default is unset.",
+            "type": "integer",
+            "default": -1
+        },
+        "encrypt": {
+            "title": "Specify that tunnels should be encrypted.",
+            "description": "Specify that tunnels should be encrypted.",
+            "type": "boolean",
+            "default": false
+        }
+    }
+}
\ No newline at end of file

software/re6stnet/instance-re6stnet-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Re6st Master instanciation",
+  "properties": {
+    "re6stry-url": {
+      "description": "ipv6 url to access your re6st registry service",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}
\ No newline at end of file

software/re6stnet/instance-re6stnet.cfg.in

+
+{% set python_bin = parameter_dict['python-executable'] -%}
+{% set re6st_registry = parameter_dict['re6st-registry'] -%}
+{% set re6stnet = parameter_dict['re6stnet'] -%}
+{% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set ipv6 = (ipv6_set | list)[0] -%}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set uri_scheme = slapparameter_dict.get('uri-scheme', 'http') -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+services = ${:etc}/service
+script = ${:etc}/run
+promises = ${:etc}/promise
+run = ${:var}/run
+ca-dir = ${:etc}/ssl
+requests = ${:ca-dir}/requests
+private = ${:ca-dir}/private
+certs = ${:ca-dir}/certs
+newcerts = ${:ca-dir}/newcerts
+crl = ${:ca-dir}/crl
+re6st = ${:srv}/res6stnet
+
+[re6stnet-dirs]
+recipe = slapos.cookbook:mkdirectory
+registry = ${directory:re6st}/registry
+log = ${directory:log}/re6stnet
+conf = ${directory:etc}/re6stnet
+ssl = ${:conf}/ssl
+token = ${:conf}/token
+run = ${directory:run}/re6stnet
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = {{ openssl_bin }}/openssl
+ca-dir = ${directory:ca-dir}
+requests-directory = ${directory:requests}
+wrapper = ${directory:services}/certificate_authority
+ca-private = ${directory:private}
+ca-certs = ${directory:certs}
+ca-newcerts = ${directory:newcerts}
+ca-crl = ${directory:crl}
+
+
+[apache-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-apache-conf'] }}
+rendered = ${directory:etc}/apache.conf
+ipv6 = {{ ipv6 }}
+port = 9026
+error-log = ${directory:log}/apache-error.log
+access-log = ${directory:log}/apache-access.log
+pid-file = ${directory:run}/apache.pid
+context = 
+  key apache_port :port
+  key re6st_ipv4 re6st-registry:ipv4
+  key re6st_port re6st-registry:port
+  key access_log :access-log
+  key error_log :error-log
+  key pid_file :pid-file
+  raw certificate ${directory:certs}/apache.crt
+  raw key ${directory:private}/apache.key
+  raw ipv6 {{ ipv6 }}
+  raw uri_scheme {{ uri_scheme }}
+
+{% set apache_wrapper = '${directory:services}/httpd' -%}
+{% if uri_scheme == 'https' -%}
+{% set apache_wrapper = '${directory:bin}/httpd_raw' -%}
+{% endif -%}
+[apache-httpd]
+recipe = slapos.cookbook:wrapper
+wrapper-path = {{ apache_wrapper }}
+command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
+
+{% if uri_scheme == 'https' %}
+[{{ section('apache-ca') }}]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+executable = ${apache-httpd:wrapper-path}
+wrapper = ${directory:services}/httpd
+key-file = ${certificate-authority:ca-private}/apache.key
+cert-file = ${certificate-authority:ca-certs}/apache.crt
+{% endif %}
+
+[apache-httpd-graceful]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-wrapper'] }}
+rendered = ${directory:script}/httpd-graceful
+mode = 0700
+context =
+  raw content {{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}; if [ $? -eq 0 ]; then kill -USR1 $(cat ${apache-conf:pid-file}); fi
+  raw dash {{ dash_binary }}
+
+[logrotate-apache]
+< = logrotate-entry-base
+name = apache
+log = ${apache-conf:error-log} ${apache-conf:access-log}
+post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
+
+[logrotate-entry-base]
+recipe = slapos.cookbook:logrotate.d
+logrotate-entries = ${logrotate:logrotate-entries}
+backup = ${logrotate:backup}
+
+[re6st-registry-conf-dict]
+port = 9201
+ipv4 = {{ ipv4 }}
+ipv6 = {{ ipv6 }}
+db = ${re6stnet-dirs:registry}/registry.db
+ca = ${re6stnet-dirs:ssl}/re6stnet.crt
+key = ${re6stnet-dirs:ssl}/re6stnet.key
+dh = ${re6stnet-dirs:ssl}/dh.pem
+verbose = 2
+mailhost = {{ slapparameter_dict.get('mailhost', '127.0.0.1') }}
+prefix-length = {{ slapparameter_dict.get('prefix-length', 16) }}
+anonymous-prefix-length = {{ slapparameter_dict.get('anonymous-prefix-length', 0) }}
+logfile = ${re6stnet-dirs:log}/registry.log
+run-dir = ${re6stnet-dirs:run}
+ipv4-net = {{ slapparameter_dict.get('ipv4-net', '') }}
+client-count = {{ slapparameter_dict.get('client-count', 10) }}
+tunnel-refresh = {{ slapparameter_dict.get('tunnel-refresh', 300) }}
+max-clients = {{ slapparameter_dict.get('max-clients', 0) }}
+hello = {{ slapparameter_dict.get('hello', 15) }}
+min-protocol = {{ slapparameter_dict.get('min-protocol', -1) }}
+encrypt = {{ slapparameter_dict.get('encrypt', 'False') }}
+
+[re6st-registry-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['template-re6st-registry-conf'] }}
+rendered = ${directory:etc}/re6st-registry.conf
+context = section parameter_dict re6st-registry-conf-dict
+
+[re6st-registry]
+recipe = slapos.cookbook:re6stnet.registry
+command = {{ re6st_registry }}
+wrapper = ${directory:services}/re6st-registry
+pid-file = ${directory:run}/registry.pid
+manager-wrapper = ${directory:bin}/re6stManageToken
+check-service-wrapper = ${directory:bin}/re6stCheckService
+drop-service-wrapper = ${directory:bin}/re6stManageDeleteToken
+revoke-service-wrapper = ${directory:bin}/re6stRevokeCertificate
+openssl-bin = {{ openssl_bin }}/openssl
+python-bin = {{ python_bin }}
+ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }}
+key-size = {{ slapparameter_dict.get('key-size', 2048) }}
+conf-dir = ${re6stnet-dirs:conf}
+token-dir = ${re6stnet-dirs:token}
+
+#Re6st config
+config-file = ${re6st-registry-conf:rendered}
+port = ${re6st-registry-conf-dict:port}
+ipv4 = ${re6st-registry-conf-dict:ipv4}
+db-path = ${re6st-registry-conf-dict:db}
+key-file = ${re6st-registry-conf-dict:key}
+cert-file = ${re6st-registry-conf-dict:ca}
+dh-file = ${re6st-registry-conf-dict:dh}
+
+slave-instance-list = ${slap-parameter:slave_instance_list}
+
+environment = 
+  PATH={{ openssl_bin }}
+
+[re6stnet-manage]
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:script}/re6st-token-manager
+command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
+
+[cron-entry-re6st-check]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-check-token
+frequency = */5 * * * *
+command = {{ python_bin }} ${re6st-registry:check-service-wrapper}
+
+[cron-entry-re6st-revoke]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-revoke-cert
+frequency = */5 * * * *
+command = {{ python_bin }} ${re6st-registry:revoke-service-wrapper}
+
+[cron-entry-re6st-drop]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-drop-token
+frequency = */5 * * * *
+command = {{ python_bin }} ${re6st-registry:drop-service-wrapper}
+
+[logrotate-entry-re6stnet]
+< = logrotate-entry-base
+name = re6stnet
+log = ${re6st-registry-conf-dict:logfile}
+post = {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${re6st-registry:pid-file} -s USR1
+
+[re6st-registry-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/re6st-registry
+hostname = ${re6st-registry:ipv4}
+port = ${re6st-registry:port}
+
+[apache-registry-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promises}/apache-re6st-registry
+hostname = ${apache-conf:ipv6}
+port = ${apache-conf:port}
+
+{% do publish_dict.__setitem__('re6stry-url', uri_scheme ~ '://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
+{% do publish_dict.__setitem__('re6stry-local-url',  'http://${re6st-registry:ipv4}:${re6st-registry:port}/') -%}
+{% do publish_dict.__setitem__('slave-amount',  '${re6st-registry:slave-amount}') -%}
+[publish]
+recipe = slapos.cookbook:publish
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
+{% endfor -%}
+
+[buildout]
+extends =
+  {{ logrotate_cfg }}
+
+parts = 
+  certificate-authority
+  logrotate-apache
+  logrotate-entry-re6stnet
+  re6stnet-manage
+  cron-entry-logrotate
+  cron-entry-re6st-check
+  cron-entry-re6st-drop
+  cron-entry-re6st-revoke
+  apache-httpd
+  apache-httpd-graceful
+  publish
+  
+  re6st-registry-promise
+  apache-registry-promise
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slap-parameter]
+slave_instance_list = {}
+

software/re6stnet/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+extra-context =
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key ipv6_set slap-configuration:ipv6
+    key ipv4_set slap-configuration:ipv4
+    key slapparameter_dict slap-configuration:configuration
+    key computer_id slap-configuration:computer
+    raw logrotate_cfg {{ template_logrotate_base }}
+    raw dash_binary {{ dash_location }}/bin/dash
+    raw openssl_bin {{ openssl_location}}/bin
+    ${:extra-context}
+
+[dynamic-template-re6stnet-parameters]
+bin-directory = {{ bin_directory }}
+python-executable  = {{ python_with_eggs }}
+re6st-registry = {{ bin_directory }}/re6st-registry
+re6stnet = {{ bin_directory }}/re6stnet
+template-apache-conf = {{ template_apache_conf }}
+template-wrapper = {{ template_wrapper }}
+apache-location = {{ apache_location }}
+template-re6st-registry-conf = {{ template_re6st_registry_conf }}
+
+[dynamic-template-re6stnet]
+< = jinja2-template-base
+template = {{ template_re6stnet }}
+filename = instance-re6stnet.cfg
+extensions = jinja2.ext.do
+extra-context =
+    section parameter_dict dynamic-template-re6stnet-parameters
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+
+default = ${dynamic-template-re6stnet:rendered}
+registry = ${:default}

software/re6stnet/re6st-registry.conf.in

+port  {{ parameter_dict['port'] }}
+4  {{ parameter_dict['ipv4'] }}
+# 6  {{ parameter_dict['ipv6'] }}
+db  {{ parameter_dict['db'] }}
+ca  {{ parameter_dict['ca'] }}
+key  {{ parameter_dict['key'] }}
+dh   {{ parameter_dict['dh'] }}
+{% if parameter_dict.get('ipv4-net', '') -%}
+ipv4 {{ parameter_dict['ipv4-net'] }}
+{% endif -%}
+client-count {{ parameter_dict['client-count'] }}
+tunnel-refresh {{ parameter_dict['tunnel-refresh'] }}
+hello {{ parameter_dict['hello'] }}
+mailhost {{ parameter_dict['mailhost'] }}
+prefix-length  {{ parameter_dict['prefix-length'] }}
+anonymous-prefix-length  {{ parameter_dict['anonymous-prefix-length'] }}
+logfile  {{ parameter_dict['logfile'] }}
+verbose  {{ parameter_dict['verbose'] }}
+run {{ parameter_dict['run-dir'] }}
+{% if parameter_dict.get('min-protocol', '-1') != '-1' -%}
+min-protocol {{ parameter_dict['min-protocol'] }}
+{% endif -%}
+{% if parameter_dict.get('encrypt', 'False') in ('true', 'True', '1') -%}
+encrypt
+{% endif -%}
+{% if parameter_dict.get('max-clients') != '0' -%}
+max-clients {{ parameter_dict['max-clients'] }}
+{% endif -%}
\ No newline at end of file

software/re6stnet/software.cfg

+[buildout]
+
+extends =
+  ../../component/re6stnet/buildout.cfg
+  ../../component/dash/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/openssl/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../stack/slapos.cfg
+
+develop = 
+  ${:parts-directory}/re6stnet-repository
+
+parts = 
+  slapos-cookbook
+  eggs
+  dash
+  babeld
+  re6stnet-develop
+  re6stnet
+  template
+  slapos.cookbook-repository
+  check-recipe
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.toolbox
+scripts =
+  slapos-kill
+
+[extra-eggs]
+recipe = zc.recipe.egg
+interpreter = pythonwitheggs
+eggs =
+  ${lxml-python:egg}
+  ${python-cffi:egg}
+  ${python-cryptography:egg}
+  pyOpenSSL
+  miniupnpc
+  re6stnet
+
+[re6stnet-repository]
+repository = http://git.erp5.org/repos/re6stnet.git
+branch = re6st-slapos
+revision = 8130c7380cbf3f13bdc4e5ecb8e792fb7d2b7b2e
+
+
+[slapos.cookbook-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.git
+branch = re6st-master
+git-executable = ${git:location}/bin/git
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 644
+
+[template-jinja2-base]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}.in
+rendered = ${buildout:directory}/${:filename}
+# XXX: extra-context is needed because we cannot append to a key of an extended
+# section.
+extra-context =
+context =
+    key bin_directory buildout:bin-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    ${:extra-context}
+
+
+[template]
+< = template-jinja2-base
+filename = template.cfg
+template = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 1f39ec8d7d91c0ba2b4ad56f27ebac39
+extra-context =
+    key apache_location apache:location
+    key dash_location dash:location
+    key logrotate_location logrotate:location
+    key openssl_location openssl:location
+    key template_apache_conf template-apache-conf:target
+    key template_re6stnet template-re6stnet:target
+    key template_re6st_registry_conf template-re6st-registry-conf:target
+    key template_logrotate_base template-logrotate-base:rendered
+    key template_wrapper template-wrapper:target
+    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+
+[template-re6stnet]
+< = download-base
+filename = instance-re6stnet.cfg.in
+md5sum = a5000a513877bdab10f160ac0aaac95f
+
+[template-logrotate-base]
+< = template-jinja2-base
+filename = instance-logrotate-base.cfg
+md5sum = f28fbd310944f321ccb34b2a34c82005
+extra-context =
+    key dcron_location dcron:location
+    key gzip_location gzip:location
+    key logrotate_location logrotate:location
+
+[template-apache-conf]
+< = download-base
+filename = apache.conf.in
+md5sum = 6fcf417f6b9651b1ed442f00c094f50c
+
+[template-re6st-registry-conf]
+< = download-base
+filename = re6st-registry.conf.in
+md5sum = 5dc218f887faeffc466e41c7d6191e49
+
+[template-wrapper]
+< = download-base
+filename = wrapper.in
+md5sum = 69e63cb58267335e21da772bd867657e
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command =
+  grep parts ${buildout:develop-eggs-directory}/re6stnet.egg-link
+
+[versions]
+apache-libcloud = 0.17.0
+ecdsa = 0.13
+gitdb = 0.6.4
+plone.recipe.command = 1.1
+pycrypto = 2.6.1
+slapos.recipe.template = 2.7
+slapos.toolbox = 0.47.3
+smmap = 0.9.0
+
+# Required by:
+# slapos.toolbox==0.47.3
+GitPython = 0.3.6
+
+# Required by:
+# slapos.toolbox==0.47.3
+atomize = 0.2.0
+
+# Required by:
+# apache-libcloud==0.17.0
+backports.ssl-match-hostname = 3.4.0.2
+
+# Required by:
+# slapos.toolbox==0.47.3
+feedparser = 5.1.3
+
+# Required by:
+# slapos.toolbox==0.47.3
+lockfile = 0.10.2
+
+# Required by:
+# re6stnet===0-413.gbec6b3c.dirty
+miniupnpc = 1.9
+
+# Required by:
+# slapos.toolbox==0.47.3
+paramiko = 1.15.2
+
+# Required by:
+# slapos.toolbox==0.47.3
+rpdb = 0.1.5
+

software/re6stnet/software.cfg.json

+{
+    "name": "RE6STNET",
+    "description": "Master instance of re6st (Resilient, Scalable, IPv6 Network application)",
+    "serialisation": "xml",
+    "software-type": {
+        "default": {
+            "title": "Default",
+            "description": "Re6st registry",
+            "request": "instance-re6stnet-input-schema.json",
+            "response": "instance-re6stnet-output-schema.json",
+            "index": 0
+        },
+        "registry": {
+            "title": "registry",
+            "description": "Re6st registry",
+            "request": "instance-re6stnet-resilient-input-schema.json",
+            "response": "instance-re6stnet-output-schema.json",
+            "index": 1
+        }
+    }
+}
\ No newline at end of file

software/re6stnet/wrapper.in

+#!{{ dash }}
+
+{{ content }}
\ No newline at end of file

software/seleniumrunner/software.cfg

@@ -47,4 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/slapos-in-partition/software.cfg

@@ -12,4 +12,4 @@ Pygments = 1.6
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.10
 slapos.recipe.download = 1.0.dev-r4053
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/slapos-testing/software.cfg

@@ -83,4 +83,4 @@ mode = 640
 Pygments = 1.6
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7

software/slaprunner/common.cfg

@@ -29,7 +29,6 @@ parts =
   eggs
   instance-runner-import
   instance-runner-export
-  slapos-cookbook
   template-slapos-cfg
 # XXX: we have to manually add this for resilience
   rdiff-backup
@@ -39,21 +38,21 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = 8a47421ac6158b4ee476acab212c67d9
+md5sum = bb7e0bf9959c4437ff1e23e645315ccf
 mode = 0644
 
 [template-runner]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = 64a2a9748829f7ecbf89dd0cb455c4c7
+md5sum = 67fd0af225c6c4f93f539f82dd23a2db
 mode = 0644
 
 [template-runner-import-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
 download-only = true
-md5sum = c0d05a26b06ce172efaad03c52ef92ca
+md5sum = d7f88b58b2508ce5af42ea7d7241626e
 filename = runner-import.sh.jinja2
 mode = 0644
 
@@ -61,14 +60,14 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = a85d054b3e2ae9243d8d188c897dc121
+md5sum = a442695f6875ae0b65439c88db1644fb
 mode = 0644
 
 [template-runner-export-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = 0f290b46c0b89ff84aee5c10477e07ca
+md5sum = 9dc934fe5015ff53869830a833266192
 filename = runner-export.sh.jinja2
 mode = 0644
 
@@ -76,7 +75,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 521bad4c571b5b2dc3eee6090802de95
+md5sum = 53be4bd07b2af826d9c80eea4185f7cb
 mode = 0644
 
 [template-resilient]
@@ -90,7 +89,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = b95b230bcdd7d4eaf9452f73156b27e8
+md5sum = fb8c45e5c35548331fb06c4633ec592a
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -106,7 +105,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/httpd_conf.in
 download-only = true
-md5sum = 94bc5d6f1afc4e075d530fed496f47c2
+md5sum = b5d095f54f714d17dff12c0c5fe4afb7
 filename = httpd_conf.in
 mode = 0644
 
@@ -121,7 +120,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
 [template-slapos-cfg]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = ca57a8bf1d4da20344fab602a69b807f
+md5sum = c490563b831fc959a09b4b233263a6c6
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = slapos.cfg.in
 download-only = true
@@ -172,45 +171,27 @@ destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi
 filename = cors-domain.jinja
 mode = 0644
 
+[monitor-check-webrunner-internal-instance]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/${:filename}
+download-only = true
+#md5sum = 4c44d617d5bfd1db8695200e896480c0
+destination = ${buildout:parts-directory}/${:filename}
+filename = monitor-check-webrunner-internal-instances.py
+mode = 0644
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs =
-  ${lxml-python:egg}
   collective.recipe.environment
   cns.recipe.symlink
   erp5.util
-  hexagonit.recipe.download
-  inotifyx
   lock-file
-  lxml
-  netaddr
   plone.recipe.command
-  pytz
-  slapos.cookbook
-  slapos.libnetworkcache
   slapos.recipe.build
   slapos.toolbox[flask_auth]
-  slapos.core
-  supervisor
-  xml_marshaller
   Gunicorn
-
-# Add slapos.libnetworkcache to path of slapos.core so that slaprunner can build SRs using cache
-[slapos-cookbook]
-eggs =
-  ${lxml-python:egg}
-  slapos.cookbook
-  cliff
-  hexagonit.recipe.download
-  inotifyx
-  netaddr
-  netifaces
-  requests
-  slapos.core
-  supervisor
-  xml_marshaller
-  pytz
-  slapos.libnetworkcache
+  ${slapos-cookbook:eggs}
 
 [extra-eggs]
 recipe = zc.recipe.egg

software/slaprunner/development.cfg

@@ -69,3 +69,7 @@ command =
   grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link
 #  grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
 
+[versions]
+slapos.cookbook =
+slapos.core =
+slapos.toolbox =

software/slaprunner/httpd_conf.in

@@ -90,7 +90,7 @@ RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
 RewriteCond %{REQUEST_URI} /git-receive-pack$
 
 <LocationMatch "^/git/">
-        SetEnv GIT_PROJECT_ROOT:{{- parameters.project_private_folder -}}
+        SetEnv GIT_PROJECT_ROOT:{{ parameters.project_private_folder }}
         Order Deny,Allow
         Deny from env=AUTHREQUIRED
 
@@ -101,7 +101,7 @@ RewriteCond %{REQUEST_URI} /git-receive-pack$
 </LocationMatch>
 
 <LocationMatch "^/git-public/">
-        SetEnv GIT_PROJECT_ROOT:{{- parameters.project_public_folder -}}
+        SetEnv GIT_PROJECT_ROOT:{{ parameters.project_public_folder }}
         Order Deny,Allow
         Deny from env=AUTHREQUIRED
 

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -54,7 +54,7 @@ name = Resilient Instance (Root Instance)
 {% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
-config-resiliency-backup-periodicity = */6 * * * *
+config-resiliency-backup-periodicity = */10 * * * *
 config-resilient-clone-number = 1
 config-ignore-known-hosts-file = true
 # XXX hardcoded

software/slaprunner/instance-runner-export.cfg.in

@@ -16,13 +16,14 @@ parts +=
   publish-connection-informations
   slaprunner-promise
   slaprunner-frontend-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-backup
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -49,7 +50,9 @@ parts +=
   bash-profile
 ## Monitor for runner
   monitor-current-log-access
+  monitor-deploy-cors-domain-cgi
   monitor-check-resilient-feed-file
+  monitor-check-webrunner-internal-instance
 
 
 [exporter]

software/slaprunner/instance-runner-import.cfg.in

@@ -14,18 +14,20 @@ parts +=
   dropbear-server-add-authorized-key
   sshkeys-authority
   slaprunner-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   shellinabox
   symlinks
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software-type
   bash-profile
   supervisord-wrapper
+  importer-consistency-promise
 # have to repeat the next one, as it's not inherited from pbsready-import
   import-on-notification
 ## Monitoring part
@@ -52,6 +54,14 @@ parts +=
 ## Monitor for import runner
   monitor-latest-restored-backup
 
+# For the needs of importer, we run the full slaprunner
+# In case both exporter and importer (aka main instance and clone instance)
+# run with the same IP (usually for testing purposes),
+# run slaprunner using different ports.
+[slaprunner]
+proxy_port = 50000
+runner_port = 50005
+
 [importer]
 recipe = slapos.recipe.template:jinja2
 template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
@@ -59,12 +69,30 @@ rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
 # backward compatibility for resilient stack
 wrapper = $${:rendered}
 mode = 700
+restore-exit-code-file=$${directory:srv}/importer-exit-code-file
 context =
   key backend_url slaprunner:access-url
+  key proxy_host slaprunner:ipv4
   section directory directory
   raw  shell_binary ${dash:location}/bin/dash
   raw  rsync_binary ${rsync:location}/bin/rsync
-  raw  curl_binary ${curl:location}/bin/curl
+  raw  restore_exit_code_file $${:restore-exit-code-file}
+
+[importer-consistency-promise]
+# Test that the importer script and "after-import" subscripts:
+# 1/ Have been run in the last 24 hours
+# 2/ Have succeeded
+recipe = collective.recipe.template
+input = inline: #!/bin/sh
+  EXIT_CODE_FILE=$(find "$${importer:restore-exit-code-file}" -mtime -1)
+  if [ -z "$EXIT_CODE_FILE" ]; then
+     echo "Consistency check is too old."
+     exit 1
+   fi
+  EXIT_CODE=$(cat $EXIT_CODE_FILE)
+  exit $EXIT_CODE
+output = $${directory:promises}/importer-consistency-promise
+mode = 755
 
 [slap-parameter]
 auto-deploy-instance = false

software/slaprunner/instance-runner-input-schema.json

@@ -92,6 +92,7 @@
             "exclusiveMinimum": true
         },
         "cpu-usage-ratio": {
+            "title": "CPU Usage Ratio",
             "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
             "type": "integer",
             "default" : 4

software/slaprunner/instance-runner-resilient-input-schema.json

+{
+    "type": "object",
+    "$schema": "http://json-schema.org/draft-04/schema",
+    "allOf": [
+        {
+            "$ref": "instance-runner-input-schema.json#/"
+        },
+        {
+            "properties": {
+                "-sla-0-computer_guid": {
+                    "title": "Target computer for main instance",
+                    "description": "Target computer GUID for main instance.",
+                    "type": "string",
+                    "optional": true
+                },
+                "-sla-1-computer_guid": {
+                    "title": "Target computer for first clone",
+                    "description": "Target computer for first clone and PBS.",
+                    "type": "string",
+                    "optional": true
+                },
+                "-sla-2-computer_guid": {
+                    "title": "Target computer for second clone",
+                    "description": "Target computer for second clone and PBS.",
+                    "type": "string",
+                    "optional": true
+                },
+                "resiliency-backup-periodicity": {
+                    "title": "Periodicity of backup",
+                    "description": "Periodicity of backup, in cron format.",
+                    "type": "string",
+                    "optional": true
+                },
+                "remove-backup-older-than": {
+                    "title": "Remove backups older than...",
+                    "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+                    "type": "string",
+                    "default": "3B",
+                    "optional": true
+                },
+                "resilient-clone-number": {
+                    "title": "Amount of backup(s) to create",
+                    "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
+                    "type": "integer",
+                    "default": 2,
+                    "optional": true
+                },
+                "ignore-known-hosts-file": {
+                    "title": "Ignore known_hosts file",
+                    "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, webrunner).",
+                    "type": "boolean",
+                    "default": false,
+                    "optional": true
+                }
+            }
+        }
+    ]
+}
\ No newline at end of file

software/slaprunner/instance-runner.cfg

@@ -13,15 +13,17 @@ parts =
   publish-connection-informations
   slaprunner-promise
   slaprunner-frontend-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
+  instance-software
   instance-software-type
   minishell-cwd
   bash-profile
@@ -51,6 +53,7 @@ parts =
 ## Monitor for runner
   monitor-current-log-access
   monitor-deploy-cors-domain-cgi
+  monitor-check-webrunner-internal-instance
 
 extends = ${monitor-template:output}
 
@@ -77,7 +80,7 @@ cert-file = $${slap-connection:cert-file}
 computer-id = $${slap-connection:computer-id}
 partition-id = $${slap-connection:partition-id}
 
-{%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
+{% if slapparameter_dict.get('custom-frontend-instance-guid') -%}
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 {% endif -%}
 
@@ -105,11 +108,12 @@ custom-frontend-url = https://$${request-custom-frontend:connection-domain}
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+home = $${buildout:directory}
+etc = $${:home}/etc/
+var = $${:home}/var/
+srv = $${:home}/srv/
+bin = $${:home}/bin/
+tmp = $${:home}/tmp/
 
 sshkeys = $${:srv}/sshkeys
 services = $${:etc}/service/
@@ -120,7 +124,7 @@ run = $${:var}/run/
 backup = $${:srv}/backup/
 promises = $${:etc}/promise/
 test = $${:etc}/test/
-nginx-data = $${directory:srv}/nginx
+nginx-data = $${:srv}/nginx
 ca-dir = $${:srv}/ssl
 project = $${:srv}/runner/project
 
@@ -189,6 +193,17 @@ software_info_json = $${runnerdirectory:home}/software_info.json
 instance_info_json = $${runnerdirectory:home}/instance_info.json
 path = $${shell:path}
 
+
+#---------------------------
+#--
+#-- supervisord managing slaprunner instance processes
+[slaprunner-supervisord-wrapper]
+recipe = slapos.cookbook:wrapper
+# XXX hardcoded locations
+command-line = $${buildout:directory}/bin/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
+wrapper-path = $${directory:services}/slaprunner-supervisord
+
+
 [test-runner]
 <= slaprunner
 slapos.cfg = $${directory:etc}/slapos-test.cfg
@@ -204,7 +219,7 @@ auto_deploy = True
 [runtestsuite]
 recipe = slapos.cookbook:wrapper
 command-line = ${buildout:directory}/bin/slaprunnertest
-wrapper-path = $${directory:bin}/runStandaloneTestSuite
+wrapper-path = $${directory:bin}/runTestSuite
 environment = RUNNER_CONFIG=$${slapos-cfg:rendered}
 
 # Deploy dropbear (minimalist SSH server)
@@ -262,7 +277,6 @@ scgi_temp_path = $${directory:tmp}/scgi_temp_path
 nb_workers = 2
 # Network
 local-ip = $${slap-network-information:local-ipv4}
-port = 30001
 global-ip = $${slap-network-information:global-ipv6}
 global-port = $${slaprunner:runner_port}
 # Backend
@@ -355,7 +369,7 @@ path_pid = $${directory:run}/gunicorn.pid
 
 [gunicorn-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = $${gunicorn:bin_gunicorn} slapos.runner:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload
+command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload
 error-log-file = gunicorn-error.log
 wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${environ:PATH}:${git:location}/bin/
@@ -558,12 +572,7 @@ repository = $${slap-parameter:slapos-repository}
 git-executable = ${git:location}/bin/git
 develop = true
 location = $${directory:project}/slapos
-
-[slapos-repo-config]
-recipe = plone.recipe.command
-stop-on-error = true
-command = cd $${slapos-repo:location} && ${git:location}/bin/git checkout $${slap-parameter:slapos-reference} && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
-update-command = true
+branch = $${slap-parameter:slapos-reference}
 
 [prepare-software]
 recipe = slapos.cookbook:wrapper
@@ -599,6 +608,11 @@ stop-on-error = true
 software-type-path = $${directory:etc}/.software_type.xml
 command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi
 
+[instance-software]
+recipe = plone.recipe.command
+stop-on-error = true
+command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -628,6 +642,11 @@ context =
     raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin
     key workdir runnerdirectory:home
 
+
+#---------------------------
+#--
+#-- supervisord managing slaprunner automation features
+
 [supervisord]
 autorestart = false
 autostart = false
@@ -696,6 +715,7 @@ path = $${directory:promises}/supervisord
 hostname = $${slaprunner:ipv4}
 port = $${supervisord:port}
 
+
 [monitor-current-log-access]
 < = monitor-directory-access
 source = $${directory:log}
@@ -715,6 +735,13 @@ context =
   key this_file :filename
   key httpd_graceful cgi-httpd-graceful-wrapper:rendered
 
+[monitor-check-webrunner-internal-instance]
+recipe = slapos.recipe.template:jinja2
+template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/$${:filename}
+filename = monitor-check-webrunner-internal-instance.py
+mode = 0744
+
 [monitor-httpd-cors]
 recipe = plone.recipe.command
 command = if [ ! -f $${:location} ]; then touch $${:location}; fi

software/slaprunner/instance.cfg

@@ -59,7 +59,7 @@ context =
 mode = 0644
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration.serialised
+recipe = slapos.cookbook:slapconfiguration
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}

software/slaprunner/software.cfg

@@ -9,39 +9,46 @@ extends = common.cfg
 [versions]
 Flask-Auth = 0.85
 PyRSS2Gen = 1.1
-apache-libcloud = 0.15.1
-async = 0.6.1
+apache-libcloud = 0.17.0
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.11
-ecdsa = 0.11
-erp5.util = 0.4.41
-gitdb = 0.5.4
-gunicorn = 19.1.1
+ecdsa = 0.13
+erp5.util = 0.4.42
+gitdb = 0.6.4
+gunicorn = 19.3.0
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-slapos.recipe.build = 0.12
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.4
-smmap = 0.8.2
+slapos.recipe.template = 2.7
+slapos.toolbox = 0.48
+smmap = 0.9.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.4
-GitPython = 0.3.2.RC1
+# slapos.toolbox==0.48
+GitPython = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.48
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.4
-feedparser = 5.1.3
+# apache-libcloud==0.17.0
+backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.cookbook==0.87
-jsonschema = 2.4.0
+# slapos.toolbox==0.48
+feedparser = 5.2.0.post1
 
 # Required by:
-# slapos.toolbox==0.40.4
-paramiko = 1.15.1
+# slapos.toolbox==0.48
+lockfile = 0.10.2
+
+# Required by:
+# slapos.toolbox==0.48
+paramiko = 1.15.2
+
+# Required by:
+# slapos.toolbox==0.48
+rpdb = 0.1.5

software/slaprunner/template/monitor-check-webrunner-internal-instances.py

+#!/usr/bin/python
+
+import os
+import subprocess
+import sys
+
+def runPromise(promise_path):
+  promise_relative_path = promise_path.replace(os.path.expanduser('~'), '~')
+  print 'Running promise %s...' % promise_relative_path
+  promise_process = subprocess.Popen(promise_path, stderr=subprocess.PIPE)
+  stdout, stderr = promise_process.communicate()
+  return_code = promise_process.returncode
+  if return_code == 0:
+    print 'Success.'
+    return True
+  else:
+    sys.stderr.write('Failure while running promise %s. %s\n' % (promise_relative_path, stderr))
+
+def getPromisePathListFromPartitionPath(partition_path):
+  promise_directory_path = os.path.join(partition_path, 'etc/promise')
+  try:
+    promise_name_list = os.listdir(promise_directory_path)
+    return [os.path.join(promise_directory_path, promise_name) for promise_name in promise_name_list]
+  except OSError:
+    return []
+
+def main():
+  # XXX hardcoded
+  partition_root_path = os.path.expanduser('~/srv/runner/instance')
+  success = True
+  for partition_name in os.listdir(partition_root_path):
+    partition_path = os.path.join(partition_root_path, partition_name)
+    for promise_path in getPromisePathListFromPartitionPath(partition_path):
+      result = runPromise(promise_path)
+      if not result:
+        success = False
+  if not success:
+    sys.exit(1)
+
+if __name__ == '__main__':
+  main()
+

software/slaprunner/template/runner-export.sh.jinja2

@@ -2,20 +2,45 @@
 LC_ALL=C
 export LC_ALL
 umask 077
+
+srv_directory={{ directory['srv'] }}
+
 sync_element () {
   path=$1
   backup_path=$2
   shift 2
   element_list=$*
+
+  # Concatenate the exclude file of each partition of webrunner
+  # to create a global exclude file.
+  exclude_content="instance/supervisord.socket"
+  for partition in $srv_directory/runner/instance/slappart*
+  do
+    exclude_file="$partition/srv/exporter.exclude"
+    if [ -e "$exclude_file" ]; then
+      partition_exclude_content_relative=$(cat "$exclude_file")
+      # For every line of the local exclude file, add the absolute path
+      for line in "$partition_exclude_content_relative"
+      do
+        if [ ! -z "$line" ]; then
+          exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
+       fi
+      done
+    fi
+  done
+  echo "$exclude_content" > $srv_directory/exporter.exclude
+
   for element in $element_list
   do
+    echo "Changing current directory to $path."
     cd $path;
     if [ -f $element ] || [ -d $element ]; then
-       {{ rsync_binary }} -rlptgov --safe-links --delete $element  $backup_path;
+       echo "Running {{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path"
+       {{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path;
     fi
   done
 }
-sync_element {{ directory['srv'] }}/runner  {{ directory['backup'] }}/runner/ instance project  proxy.db
+sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
 # We sync .* appart
 date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
 cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/

software/slaprunner/template/runner-import.sh.jinja2

 #!{{ shell_binary }}
+set -e
 LC_ALL=C
 export LC_ALL
 umask 077
+srv_directory={{ directory['srv'] }}
 restore_element () {
   backup_path=$1
   restore_path=$2
@@ -11,21 +13,92 @@ restore_element () {
   do
     cd $backup_path;
     if [ -f $element ] || [ -d $element ]; then
-       {{ rsync_binary }} -av --delete $backup_path/$element $restore_path;
+       {{ rsync_binary }} -av --delete --exclude *.sock --exclude *.pid --exclude .installed.cfg --exclude .installed-switch-softwaretype.cfg $backup_path/$element $restore_path;
     fi
   done
 }
 
 write_backup_proof () {
   cd {{ directory['backup'] }}
-  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
-  diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
+  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > $srv_directory/proof.signature
+  diff -ruw {{ directory['backup'] }} $srv_directory/proof.signature > $srv_directory/backup.diff || true # diff exits with code 1 when files are different
 }
 
 # For now we just make the diff before
 write_backup_proof
 
-restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner  instance project  proxy.db
+restore_element {{ directory['backup'] }}/runner/ $srv_directory/runner  instance project  proxy.db
 restore_element  {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
 cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
-{{ curl_binary }} --insecure -vg6L --max-time 5  {{ backend_url }}/isSRReady;
+
+
+# Invoke arbitrary script to perform specific restoration
+# procedure.
+RESTORE_EXIT_CODE=0
+runner_import_restore=$srv_directory/runner-import-restore
+if [ ! -e "$runner_import_restore" ]; then
+  touch $runner_import_restore
+  chmod +x $runner_import_restore
+fi
+echo "Running $runner_import_restore script..."
+$srv_directory/runner-import-restore || RESTORE_EXIT_CODE=$?
+
+echo "Updating slapproxy database, software release and instances..."
+HOME="{{ directory['home'] }}"
+# XXX Hardcoded
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export MAKEFLAGS=-j4
+SLAPOS="{{ directory['bin'] }}/slapos"
+# XXX hardcoded
+SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
+DATABASE="$HOME/srv/runner/proxy.db"
+# Change slapproxy database to point instances to new software release
+# XXX hardcoded
+PARTITION=$(basename $HOME)
+OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
+SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9]\?/\1'"$PARTITION"'/')
+$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
+$SQLITE3 $DATABASE "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || $SQLITE3 $DATABASE "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
+# Change slapproxy database to have all instances stopped
+$SQLITE3 $DATABASE "update partition11 set slap_state='stopped';"
+
+set -x
+# Run slapproxy on different port (in case of running inside of erp5testnode
+# with only one IP and port 50000 already taken by slapproxy of main instance)
+HOST="{{ proxy_host }}"
+PORT="50001"
+URL="http://$HOST:$PORT"
+$SLAPOS proxy start --cfg $HOME/etc/slapos.cfg --port $PORT >/dev/null 2>&1 &
+SLAPPROXY_PID=$!
+trap "kill $SLAPPROXY_PID" EXIT TERM INT
+sleep 5
+
+echo "Building newest software..."
+$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1 ||
+$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1 ||
+$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1
+# Remove defined scripts to force buildout to recreate them to have updated paths
+rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore || true
+echo "Running slapos node instance..."
+# XXX hardcoded
+$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
+$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
+$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
+
+# Invoke defined scripts for each partition inside of slaprunner
+for partition in $srv_directory/runner/instance/slappart*/
+do
+  script=$partition/srv/runner-import-restore
+  if [ -e "$script" ]; then
+    echo "Running $script script..."
+    $script || RESTORE_EXIT_CODE=$?
+  fi
+done
+
+# Change back slapproxy database to have all instances started
+$SQLITE3 $DATABASE "update partition11 set slap_state='started';"
+
+# Write exit code to an arbitrary file that will be checked by promise/monitor
+RESTORE_EXIT_CODE_FILE="{{ restore_exit_code_file }}"
+echo $RESTORE_EXIT_CODE > $RESTORE_EXIT_CODE_FILE
+exit $RESTORE_EXIT_CODE

software/slaprunner/template/slapos.cfg.in

@@ -5,6 +5,7 @@ master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
 computer_id = slaprunner
 maximal_delay = 0
 root_check = {{ slaprunner['root_check'] }}
+forbid_supervisord_automatic_launch = true
 
 [slapformat]
 partition_amount = {{ slaprunner['partition-amount'] }}

software/test-slave-instance-deployment/software.cfg

@@ -59,7 +59,7 @@ eggs = collective.recipe.template
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
 slapos.recipe.build = 0.13
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7
 
 # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
 [buildout]

software/varnish/software.cfg

@@ -45,6 +45,7 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended

software/wendelin/software-dev.cfg

+# Wendelin with components taken from latest git versions
+[buildout]
+extends = software.cfg
+
+# wendelin.core
+parts -= wendelin.core
+parts += wendelin.core-dev
+
+
+# also tell erp5 to use -dev eggs instead of released ones
+[eggs]
+eggs -= ${wendelin.core:egg}
+eggs += ${wendelin.core-dev:egg}

software/wendelin/software.cfg

+[buildout]
+versions = versions
+extends =
+  ../../software/ipython_notebook/software.cfg
+  ../../component/fluentd/buildout.cfg
+  ../../component/matplotlib/buildout.cfg
+  ../../component/ipython/buildout.cfg
+  ../../component/pandas/buildout.cfg
+  ../../component/wendelin.core/buildout.cfg
+  ../../component/msgpack-python/buildout.cfg
+  ../../component/scipy/buildout.cfg
+  ../../component/scikit-learn/buildout.cfg
+  ../../software/erp5/software.cfg
+parts +=
+  wendelin
+  scipy
+  scikit-learn
+  pandas
+  msgpack-python
+  ipython
+  wendelin.core
+  matplotlib
+  fluentd
+  ipython_notebook
+
+[eggs]
+initialization =
+  import scipy.stats # load our own libstdc++ explicitly at the very beginning
+extra-paths +=
+  ${wendelin:location}
+eggs +=
+  ${scikit-learn:egg}
+  ${scipy:egg}
+  ${pandas:egg}
+  ${msgpack-python:egg}
+  ${wendelin.core:egg}
+  ${ipython:egg}
+  ${matplotlib:egg}
+
+[erp5_repository_list]
+repository_id_list += wendelin
+
+[local-bt5-repository]
+# we need to override it
+list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/
+
+[wendelin]
+<= erp5
+repository = https://lab.nexedi.cn/nexedi/wendelin.git
+branch = master
+
+[versions]
+scikit-learn = 0.16.1
+scipy = 0.15.1
+pandas = 0.16.1
+msgpack-python = 0.4.6
+numpy = 1.9.2
+wendelin.core = 0.3
+ipython = 3.1.0
+matplotlib = 1.4.3
\ No newline at end of file

stack/boinc/buildout.cfg

@@ -88,4 +88,4 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2

stack/cloudooo.cfg

@@ -75,19 +75,18 @@ parts =
 
 
 [versions]
-Paste = 1.7.5.1
-PasteScript = 1.7.5
+Paste = 2.0.2
+PasteScript = 2.0.2
 WSGIUtils = 0.7
-plone.recipe.command = 1.1
 python-magic = 0.4.6
 rdiff-backup = 1.0.5
-slapos.recipe.template = 2.5
+slapos.recipe.template = 2.7
 
 # Required by:
-# PasteScript==1.7.5
-# cloudooo==1.2.5-dev
+# PasteScript==2.0
+# cloudooo==1.2.5.dev0
 PasteDeploy = 1.5.2
 
 # Required by:
-# cloudooo==1.2.5-dev
-erp5.util = 0.4.41
+# cloudooo==1.2.5.dev0
+erp5.util = 0.4.42

stack/erp5/buildout.cfg

@@ -18,6 +18,7 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/haproxy/buildout.cfg
   ../../component/hookbox/buildout.cfg
+  ../../component/findutils/buildout.cfg
   ../../component/librsvg/buildout.cfg
   ../../component/imagemagick/buildout.cfg
   ../../component/inkscape/buildout.cfg
@@ -68,6 +69,7 @@ parts =
   apache
   apache-antiloris
   file
+  findutils
   graphviz
   haproxy
   jsl
@@ -97,7 +99,6 @@ parts =
   wget
 
 # Buildoutish
-  patched-eggs
   eggs
   testrunner
   test_suite_runner
@@ -133,6 +134,7 @@ parts =
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended
@@ -145,22 +147,17 @@ context =
     key slapos_core_version versions:slapos.core
     ${:extra-context}
 
+[mariadb-resiliency-after-import-script]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
+md5sum = a9851920bb22ae58c2eb9dc3e955250c
+mode = 755
+
 [template-mariadb]
-< = template-jinja2-base
-filename = instance-mariadb.cfg
-md5sum = e0c923a8195b5a36c7df766c1fb4bfdb
-extra-context =
-    key coreutils_location coreutils:location
-    key dcron_location dcron:location
-    key gettext_location gettext:location
-    key grep_location grep:location
-    key gzip_location gzip:location
-    key logrotate_location logrotate:location
-    key mariadb_location mariadb:location
-    key perl_location perl:location
-    key perl_siteprefix perl:siteprefix
-    key sed_location sed:location
-    key xtrabackup_location xtrabackup:location
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/instance-mariadb.cfg.in
+md5sum = a5f9888bd882331c35d48f05e468a0a2
+mode = 640
 
 [template-zope]
 recipe = slapos.recipe.build:download
@@ -171,7 +168,7 @@ mode = 640
 [template-kumofs]
 < = template-jinja2-base
 filename = instance-kumofs.cfg
-md5sum = 40817014a41497bceb696e512436e670
+md5sum = 627369560a030b006dbdd8f10ff6694a
 extra-context =
     key dash_location dash:location
     key dcron_location dcron:location
@@ -182,13 +179,13 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 2bc13fc6cd52c7b8e2b4ddb99a69974b
+md5sum = 20ee9db93c57425319cd4b385d327d39
 mode = 640
 
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = c93c881267beadf840dc0b339a3a466a
+md5sum = 79eb68a9c5073535e8a98897385828a8
 mode = 640
 
 [template-zope-conf]
@@ -202,7 +199,7 @@ mode = 640
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 2ed28d874b5494521cb874a2474af7c5
+md5sum = 0d777bfc8377e5b8ddc2e32a10f45b9b
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -217,9 +214,11 @@ extra-context =
     key dmtx_utils_location dmtx-utils:location
     key erp5_location erp5:location
     key file_location file:location
+    key findutils_location findutils:location
     key fontconfig_location fontconfig:location
     key fonts_location fonts:location
     key freetype_location freetype:location
+    key gettext_location gettext:location
     key git_location git:location
     key glib_location glib:location
     key glu_location glu:location
@@ -245,8 +244,11 @@ extra-context =
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
     key mariadb_location mariadb:location
+    key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key mesa_location mesa:location
     key openssl_location openssl:location
+    key perl_location perl:location
+    key perl_siteprefix perl:siteprefix
     key poppler_location poppler:location
     key sed_location sed:location
     key stunnel_location stunnel:location
@@ -254,7 +256,7 @@ extra-context =
     key template_erp5_cluster template-erp5-cluster:target
     key template_erp5_single template-erp5-single:target
     key template_kumofs template-kumofs:rendered
-    key template_mariadb template-mariadb:rendered
+    key template_mariadb template-mariadb:target
     key template_memcached template-memcached:rendered
     key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
@@ -266,12 +268,13 @@ extra-context =
     key wget_location wget:location
     key xdamage_location xdamage:location
     key xfixes_location xfixes:location
+    key xtrabackup_location xtrabackup:location
     key zlib_location zlib:location
 
 [template-memcached]
 < = template-jinja2-base
 filename = instance-memcached.cfg
-md5sum = 346c864c1f119360eddb5e163f16d4f3
+md5sum = de63a79e6812854c44e961f5ac3b465d
 extra-context =
     key dash_location dash:location
     key dcron_location dcron:location
@@ -288,13 +291,13 @@ mode = 640
 [template-erp5-cluster]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-erp5-cluster.cfg.in
-md5sum = 1fafb23019043c069bf6e43d61010a54
+md5sum = 2ff90bb68761b4930f92321f3b6ba590
 mode = 640
 
 [template-varnish]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-varnish.cfg.in
-md5sum = ae0465591c22c0cb316c1706965c6b18
+md5sum = 98158d0b349e3579a20ab520dfa9ebb7
 mode = 640
 
 [bt5-repository]
@@ -384,22 +387,6 @@ initialization =
   repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
   sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
 
-[patched-eggs]
-recipe = minitage.recipe.egg
-eggs =
-  Acquisition
-  Products.DCWorkflow
-  ZODB3
-Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
-Acquisition-patch-options = -p1
-Acquisition-patch-binary = ${patch:location}/bin/patch
-Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
-Products.DCWorkflow-patch-options = -p1
-Products.DCWorkflow-patch-binary = ${patch:location}/bin/patch
-ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
-ZODB3-patch-options = -p1
-ZODB3-patch-binary = ${patch:location}/bin/patch
-
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -415,6 +402,7 @@ eggs =
   Pympler
   SOAPpy
   chardet
+  collective.recipe.template
   coverage
   elementtree
   erp5diff
@@ -422,6 +410,7 @@ eggs =
   interval
   ipdb
   Jinja2
+  jsonschema
   mechanize
   paramiko
   ply
@@ -506,6 +495,15 @@ scripts =
 extra-paths =
   ${erp5:location}
 
+# patches for eggs
+patch-binary = ${patch:location}/bin/patch
+Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
+Acquisition-patch-options = -p1
+Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
+Products.DCWorkflow-patch-options = -p1
+ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
+ZODB3-patch-options = -p1
+
 [zodbanalyze]
 recipe = zc.recipe.egg
 eggs =
@@ -541,16 +539,16 @@ scripts =
 
 [versions]
 # patched eggs
-Acquisition = 2.13.8-ZMinitagePatched-AqDynamic
-Products.DCWorkflow = 2.2.4-ZMinitagePatched-WorkflowMethod
-ZODB3 = 3.10.5-ZMinitagePatched-ZODB33105
+Acquisition = 2.13.8+SlapOSPatched001
+Products.DCWorkflow = 2.2.4+SlapOSPatched001
+ZODB3 = 3.10.5+SlapOSPatched001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
 
 # use newer version than specified in ZTK
 PasteDeploy = 1.5.2
-Pygments = 2.0.1
+Pygments = 2.0.2
 coverage = 3.7.1
 
 # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
@@ -587,59 +585,58 @@ zope.app.testing = 3.8.1
 
 # Pinned versions
 MySQL-python = 1.2.5
-Pillow = 2.6.1
+Pillow = 2.8.2
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.0
 Products.GenericSetup = 1.7.5
 Products.LongRequestLogger = 1.1.0
-Products.MimetypesRegistry = 2.0.6
+Products.MimetypesRegistry = 2.0.7
 Products.PluginRegistry = 1.3
 Products.TIDStorage = 5.4.9
-PyPDF2 = 1.23
+PyPDF2 = 1.24
 PyXML = 0.8.5
-Pympler = 0.3.1
+Pympler = 0.4.1
 StructuredText = 2.11.1
 WSGIUtils = 0.7
-apache-libcloud = 0.16.0
-astroid = 1.2.1
-async = 0.6.1
+apache-libcloud = 0.17.0
+astroid = 1.3.6
 chardet = 2.3.0
+collective.recipe.template = 1.11
 csp-eventlet = 0.7.0
-ecdsa = 0.11
-elementtree = 1.2.7-20070827-preview
-erp5diff = 0.8.1.5
-eventlet = 0.15.2
+ecdsa = 0.13
+elementtree = 1.2.6.post20050316
+erp5diff = 0.8.1.7
+eventlet = 0.17.4
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
-gitdb = 0.6.0
-greenlet = 0.4.5
+gitdb = 0.6.4
+greenlet = 0.4.6
 http-parser = 0.8.3
-httplib2 = 0.9
+httplib2 = 0.9.1
 huBarcode = 1.0.0
 interval = 1.0.0
-ipdb = 0.8
-ipython = 2.3.1
-logilab-common = 0.63.0
-minitage.paste = 1.4.6
-minitage.recipe.egg = 1.107
-numpy = 1.9.1
+ipdb = 0.8.1
+ipython = 3.1.0
+logilab-common = 0.63.2
+numpy = 1.9.2
 plone.recipe.command = 1.1
-ply = 3.4
-polib = 1.0.5
-pprofile = 1.7.2
+ply = 3.6
+polib = 1.0.6
+pprofile = 1.7.3
 pycountry = 1.10
-pyflakes = 0.8.1
-pylint = 1.3.1
-python-ldap = 2.4.17
+pycrypto = 2.6.1
+pyflakes = 0.9.1
+pylint = 1.4.3
+python-ldap = 2.4.19
 python-magic = 0.4.6
-python-memcached = 1.53
+python-memcached = 1.54
 qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
-slapos.recipe.template = 2.5
-slapos.toolbox = 0.43.0
-smmap = 0.8.3
+slapos.recipe.template = 2.7
+slapos.toolbox = 0.48
+smmap = 0.9.0
 socketpool = 0.5.3
 spyne = 2.11.0
 suds = 0.4
@@ -647,57 +644,44 @@ threadframe = 0.2
 timerserver = 2.0.2
 urlnorm = 1.1.2
 uuid = 1.30
-validictory = 1.0.0a2
+validictory = 1.0.0
 xupdate-processor = 0.4
 
 # Required by:
-# slapos.toolbox==0.43.0
-GitPython = 0.3.2.1
+# slapos.toolbox==0.48
+GitPython = 1.0.1
 
 # Required by:
 # Products.CMFCore==2.2.8
 Products.ZSQLMethods = 2.13.4
 
 # Required by:
-# slapos.toolbox==0.43.0
+# slapos.toolbox==0.48
 atomize = 0.2.0
 
 # Required by:
-# cloudooo==1.2.5-dev
-erp5.util = 0.4.41
+# apache-libcloud==0.17.0
+backports.ssl-match-hostname = 3.4.0.2
 
 # Required by:
-# slapos.toolbox==0.43.0
-feedparser = 5.1.3
+# slapos.toolbox==0.48
+feedparser = 5.2.0.post1
 
 # Required by:
-# SOAPpy==0.12.0nxd001
+# SOAPpy===0.12.0nxd001
 fpconst = 0.7.2
 
 # Required by:
-# minitage.recipe.egg==1.107
-iniparse = 0.4
-
-# Required by:
-# minitage.core==2.0.57
-minitage = 2.0.67
-
-# Required by:
-# minitage.recipe.common==1.90
-# minitage.recipe.egg==1.107
-minitage.core = 2.0.57
-
-# Required by:
-# minitage.recipe.egg==1.107
-minitage.recipe.common = 1.90
+# slapos.toolbox==0.48
+lockfile = 0.10.2
 
 # Required by:
-# minitage.recipe.egg==1.107
-ordereddict = 1.1
+# slapos.toolbox==0.48
+paramiko = 1.15.2
 
 # Required by:
-# slapos.toolbox==0.43.0
-paramiko = 1.15.1
+# slapos.toolbox==0.48
+rpdb = 0.1.5
 
 # Required by:
 # zope.app.testing==3.8.1

stack/erp5/instance-cloudoo.cfg.in

@@ -5,6 +5,7 @@
 parts =
   publish-cloudooo-connection-information
   cloudooo-instance
+  resiliency-exclude-file
   promise
   promise-openoffice
 eggs-directory = {{ eggs_directory }}
@@ -49,6 +50,12 @@ url-list = {{ json.get('font_url_list', []) | join(' ') }}
 service-folder = ${directory:service}
 onetimedownload_path = {{ bin_directory }}/onetimedownload
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${directory:srv}/exporter.exclude
+
 [promise]
 recipe = slapos.cookbook:check_port_listening
 path = ${directory:promise}/cloudooo

stack/erp5/instance-erp5-cluster.cfg.in

@@ -29,6 +29,7 @@ partition-id = ${slap-connection:partition-id}
 <=request-common
 name = MariaDB DataBase
 software-type = mariadb
+config-mariadb-json = ${slap-parameter:mariadb-json}
 sla-computer_guid = ${slap-parameter:mariadb-computer-guid}
 
 [request-cloudooo]

stack/erp5/instance-kumofs.cfg.in

@@ -6,6 +6,7 @@ parts =
   logrotate-entry-kumofs
   cron
   cron-entry-logrotate
+  resiliency-exclude-file
   promise-kumofs-server
   promise-kumofs-server-listen
   promise-kumofs-gateway
@@ -115,6 +116,12 @@ name = logrotate
 frequency = 0 0 * * *
 command = ${logrotate:wrapper}
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${rootdirectory:srv}/exporter.exclude
+
 # Deploy zope promises scripts
 [promise-template]
 recipe = slapos.cookbook:check_port_listening

stack/erp5/instance-mariadb.cfg.in

+{% if software_type == slap_software_type -%}
+{% set json = json_module.loads(slapparameter_dict.get('mariadb-json', '{}')) -%}
+{% set backup_periodicity = json.get('backup-periodicity', '0 22 * * *') -%}
+{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
+
 [buildout]
 parts =
   publish-mariadb-url
@@ -7,7 +12,10 @@ parts =
   cron
   cron-entry-logrotate
   cron-entry-mariadb-backup
+  cron-entry-mariadb-backup-expire
   binary-link
+  resiliency-exclude-file
+  resiliency-after-import-script
   promise
 
 eggs-directory = {{ eggs_directory }}
@@ -18,14 +26,44 @@ offline = true
 recipe = slapos.cookbook:publishurl
 url = mysql://${mariadb-instance:user}:${mariadb-instance:password}@${mariadb-instance:ip}:${mariadb-instance:port}/${mariadb-instance:database}
 
+[binary-wrap-base]
+recipe = slapos.cookbook:wrapper
+# Note: --defaults-file must be the first argument, otherwise wrapped binary
+# will reject it.
+command-line = "{{ mariadb_location }}/bin/${:command}" --defaults-file="${mariadb-instance:conf-file}"
+wrapper-path = ${rootdirectory:bin}/${:command}
+parameters-extra = true
+
+[binary-wrap-mysqldump]
+< = binary-wrap-base
+command = mysqldump
+
 [cron-entry-mariadb-backup]
-<= cron
 recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
 name = mariadb-backup
-frequency = 0 0 * * *
-command = ${mariadb-instance:backup-script}
+frequency = {{ backup_periodicity }}
+{# When binlogs are enabled:
+# flush-logs: used so no manipulation on binlogs is needed to restore from
+#   full + binlogs. The first binlog after a dump starts from dump snapshot and
+#   can be fully restored.
+# master-data: use value "2" as we are not in a replication case
+#}
+command = "${binary-wrap-mysqldump:wrapper-path}" -u root --all-databases --single-transaction --flush-logs --master-data=2 --socket=${mariadb-instance:socket} | {{ gzip_location }}/bin/gzip > "${directory:mariadb-backup-full}/$({{ coreutils_location }}/bin/date "+%Y%m%d%H%M%S").sql.gz"
+{# KEEP GLOB PATTERN IN SYNC with generated filenames above
+#           YYYYmmddHHMMSS -#}
+file-glob = ??????????????.sql.gz
+
+[cron-entry-mariadb-backup-expire]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = mariadb-backup-expire
+frequency = {{ backup_periodicity }}
+command = {{ findutils_location }}/bin/find "${directory:mariadb-backup-full}" -maxdepth 1 -name "${cron-entry-mariadb-backup:file-glob}" -daystart -mtime +7 -delete
 
 [mariadb-instance]
+# XXX: remove backup part of this recipe. Backup part is now done in previous sections.
+# Keeping it is harmless because it is supposed to be launched by cron entry, which has been removed.
 recipe = slapos.cookbook:generic.mysql
 
 # Options
@@ -39,6 +77,11 @@ test-database = erp5_test
 mysql-test-database-base = testdb
 mysql-test-user-base = testuser
 
+# Backup
+binlog-path = ${directory:mariadb-backup-incremental}/binlog
+# XXX: binlog rotation happens along with other log's rotation
+binlog-expire-days = 7
+
 # Paths
 wrapper = ${basedirectory:services}/mariadb
 update-wrapper = ${basedirectory:services}/mariadb_update
@@ -153,6 +196,21 @@ link-binary =
   {{ sed_location }}/bin/sed
   {{ mariadb_location }}/bin/mysql
 
+
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: srv/mariadb/**
+output = ${rootdirectory:srv}/exporter.exclude
+
+[resiliency-after-import-script]
+# Generate after import script used by importer instance of webrunner
+recipe = collective.recipe.template
+input = {{ mariadb_resiliency_after_import_script }}
+output = ${rootdirectory:srv}/runner-import-restore
+mode = 755
+dash = {{ dash_location }}/bin/dash
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc
@@ -188,3 +246,4 @@ port = ${mariadb-instance:port}
 
 [slap-parameter]
 test-database-amount = 100
+{%- endif %}

stack/erp5/instance-memcached.cfg.in

@@ -9,6 +9,7 @@ parts =
   logrotate-entry-kumofs
   cron
   cron-entry-logrotate
+  resiliency-exclude-file
   promise-kumofs-server
   promise-kumofs-server-listen
   promise-kumofs-gateway
@@ -120,6 +121,12 @@ name = logrotate
 frequency = 0 0 * * *
 command = ${logrotate:wrapper}
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${rootdirectory:srv}/exporter.exclude
+
 # Deploy zope promises scripts
 [promise-template]
 recipe = slapos.cookbook:check_port_listening

stack/erp5/instance-tidstorage.cfg.in

@@ -8,6 +8,7 @@ Note: all port counters are pre-incremented. No idea why base port is skipped.
 {% set current_apache_port = apache_port_base | int -%}
 {% set current_haproxy_port = haproxy_port_base | int -%}
 {% set json = json_module.loads(slapparameter_dict['json']) -%}
+{% set backup_periodicity = json.get('backup-periodicity', 'daily') -%}
 {% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
 {#
 XXX: This template only supports exactly one IPv4 and one IPv6 per
@@ -105,6 +106,7 @@ instance-tests = ${:instance}/tests
 log = ${:var}/log
 logrotate-backup = ${:backup}/logrotate
 logrotate-entries = ${:etc}/logrotate.d
+promise = ${:etc}/promise
 run = ${:var}/run
 services = ${:etc}/run
 srv = ${buildout:directory}/srv
@@ -113,6 +115,7 @@ tmp = ${buildout:directory}/tmp
 var = ${buildout:directory}/var
 zodb = ${:srv}/zodb
 zodb-backup = ${:backup}/zodb
+zodb-recovery-test = ${:srv}/zodbrecoverytest
 
 #############################
 # Binary symlinking
@@ -145,6 +148,60 @@ link-binary =
   {{ parameter_dict['poppler'] }}/bin/pdfunite
   {{ parameter_dict['dmtx-utils'] }}/bin/dmtxwrite
 
+#############################
+# Resiliency
+#############################
+# Used for ERP5 resiliency or (more probably)
+# webrunner resiliency with erp5 inside.
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: srv/zodb/**
+output = ${directory:srv}/exporter.exclude
+
+[resiliency-after-import-script]
+# Generate after import script used by importer instance of webrunner
+recipe = collective.recipe.template
+input = inline: #!/bin/sh
+  # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+  # OR ZODB DATA WILL BE ERASED.
+
+  # This script will restore the repozo backup to the real
+  # zodb location. It is launched by the clone (importer) instance of webrunner
+  # in the end of the import script.
+
+  # Depending on the output, it will create a file containing
+  # the status of the restoration (success or failure).
+  zodb_directory="${directory:zodb}"
+  zodb_backup_directory="${directory:zodb-backup}"
+  repozo="${tidstorage:repozo-binary}"
+  EXIT_CODE=0
+  {% for zeo_id, zeo_configuration_list in json['zeo'].iteritems() -%}
+  {%   for zeo_slave in zeo_configuration_list -%}
+  storage_name="{{ zeo_slave['storage-name'] }}"
+  zodb_path="$storage_name.fs"
+  pid_file=${zeo-instance-{{ zeo_id }}:pid-path}
+  if [ -e "$pid_file" ]; then
+    pid=$(cat $pid_file) > /dev/null 2>&1
+    if kill -0 "$pid"; then
+      echo "Zeo is already running with pid $pid. Aborting."
+      exit 1
+    fi
+  fi
+  echo "Removing $zodb_path..."
+  echo "Restoring $storage_name into $zodb_path..."
+  $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name"
+  CURRENT_EXIT_CODE=$?
+  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
+    EXIT_CODE="$CURRENT_EXIT_CODE"
+    echo "$storage_name Backup restoration failed."
+  fi
+  {%   endfor -%}
+  {% endfor -%}
+  exit $EXIT_CODE
+output = ${directory:srv}/runner-import-restore
+mode = 755
+
 #############################
 # CA
 #############################
@@ -504,7 +561,7 @@ repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo
 < = cron-base
 recipe = slapos.cookbook:cron.d
 name = tidstorage
-frequency = 0 0 * * *
+frequency = {{ backup_periodicity }}
 command = ${tidstorage:repozo-wrapper}
 
 [logrotate-entry-tidstorage]
@@ -524,6 +581,8 @@ parts =
   cron-entry-logrotate
   certificate-authority
   erp5-certificate-authority
+  resiliency-exclude-file
+  resiliency-after-import-script
   tidstorage
   cron-entry-tidstorage-backup
   logrotate-entry-tidstorage

stack/erp5/instance-varnish.cfg.in

@@ -6,6 +6,7 @@ parts =
   varnish-instance
   cron
   cron-entry-logrotate
+  resiliency-exclude-file
   {# When web_checker related parameter is given, web_checker will be enabled.#}
 {% if web_checker_parameter is defined %}
   web-checker
@@ -111,6 +112,12 @@ sharedscripts = true
 notifempty = true
 create = true
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${rootdirectory:srv}/exporter.exclude
+
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
 services = ${rootdirectory:etc}/run

stack/erp5/instance.cfg.in

@@ -124,6 +124,33 @@ extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type varnish
 
+[dynamic-template-mariadb-parameters]
+buildout-bin-directory = {{ buildout_bin_directory }}
+
+[dynamic-template-mariadb]
+< = jinja2-template-base
+template = {{ template_mariadb }}
+filename = instance-mariadb.cfg
+extra-context =
+    section parameter_dict dynamic-template-mariadb-parameters
+    raw coreutils_location {{ coreutils_location }}
+    raw dash_location {{ dash_location }}
+    raw dcron_location {{ dcron_location }}
+    raw findutils_location {{ findutils_location }}
+    raw gettext_location {{ gettext_location }}
+    raw grep_location {{ grep_location }}
+    raw gzip_location {{ gzip_location }}
+    raw logrotate_location {{ logrotate_location }}
+    raw mariadb_location {{ mariadb_location }}
+    raw mariadb_resiliency_after_import_script {{ mariadb_resiliency_after_import_script }}
+    raw perl_location {{ perl_location }}
+    raw perl_siteprefix {{ perl_siteprefix }}
+    raw sed_location {{ sed_location }}
+    raw xtrabackup_location {{ xtrabackup_location }}
+    import json_module json
+# Must match the key id in [switch-softwaretype] which uses this section.
+    raw software_type mariadb
+
 [dynamic-template-zope-parameters]
 apache = {{ apache_location }}
 aspell = {{ aspell_location }}
@@ -227,7 +254,7 @@ kumofs = {{ template_kumofs }}
 memcached = {{ template_memcached }}
 cloudooo = ${dynamic-template-cloudooo:rendered}
 zope = ${dynamic-template-zope:rendered}
-mariadb = {{ template_mariadb }}
+mariadb = ${dynamic-template-mariadb:rendered}
 tidstorage = ${dynamic-template-tidstorage:rendered}
 varnish = ${dynamic-template-varnish:rendered}
 

stack/erp5/template/instance-mariadb-resiliency-after-import-script.sh.in

+#!${:dash}
+
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# OR MYSQL DATA WILL BE ERASED.
+
+# This script will import the dump of the mysql database to the real
+# database. It is launched by the clone (importer) instance of webrunner
+# in the end of the import script.
+
+# Depending on the output, it will create a file containing
+# the status of the restoration (success or failure)
+
+set -e
+
+mysql_executable="${mariadb-instance:mysql-binary}"
+mysqldump_executable="${binary-wrap-mysqldump:wrapper-path}"
+mariadb_data_directory="${directory:mariadb-data}"
+mariadb_backup_directory="${directory:mariadb-backup-full}"
+instance_directory="${buildout:directory}"
+pid_file="${mariadb-instance:pid-file}"
+binlog_path="${mariadb-instance:binlog-path}"
+
+# Make sure mariadb is not already running
+if [ -e "$pid_file" ]; then
+  pid=$(cat $pid_file) > /dev/null 2>&1
+  if kill -0 "$pid"; then
+    echo "Mariadb is already running with pid $pid. Aborting."
+    exit 1
+  fi
+fi
+
+echo "Deleting existing database..."
+rm -r $mariadb_data_directory/* >/dev/null 2>&1 || true
+
+echo "Adapting binlog database to new paths..."
+new_binlog_directory="$(dirname $binlog_path)"
+binlog_index_file="$new_binlog_directory/binlog.index"
+old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
+sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
+
+echo "Starting mariadb..."
+# XXX hardcoded
+$instance_directory/etc/run/mariadb &
+mysqld_pid=$!
+trap "kill $mysqld_pid" EXIT TERM INT
+sleep 30
+# If mysql has stopped, abort
+if ! [ -d /proc/$mysql_pid ]; then
+  echo "mysqld exited, aborting."
+  exit 1
+fi
+$instance_directory/etc/run/mariadb_update &
+mariadb_update_pid=$!
+sleep 60
+# If mariadb_update is still running, abort
+if [ -d /proc/$mariadb_update_pid ]; then
+  echo "mariadb_update still running after timeout, aborting."
+  kill $mariadb_update_pid
+  exit 1
+fi
+
+echo "Importing data..."
+# Use latest dump XXX can contain funny characters
+dump=$(ls -r $mariadb_backup_directory | head -1)
+zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
+RESTORE_EXIT_CODE=$?
+
+if [ $RESTORE_EXIT_CODE -eq 0 ]; then
+  echo 'Backup restoration successfully completed.'
+else
+  echo 'Backup restoration failed.'
+fi
+
+exit $RESTORE_EXIT_CODE

stack/lamp.cfg

@@ -63,4 +63,4 @@ feedparser = 5.1.1
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2

stack/lamp/buildout.cfg

@@ -206,4 +206,4 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2

stack/lapp/buildout.cfg

@@ -203,7 +203,7 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.40.2
-paramiko = 1.15.1
+paramiko = 1.15.2
 
 # Required by:
 # slapos.recipe.maarch==0.4

stack/monitor/buildout.cfg

@@ -48,7 +48,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = cb2f15850d3dc82459a0044adb4416cf
+md5sum = 5b12e864f1762d7984f7d4863d0b795d
 destination = ${buildout:parts-directory}/monitor-template-monitor-bin
 filename = monitor.py.in
 mode = 0644
@@ -57,7 +57,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 2d48f8b8e01fa0fdde964ed1c1547f05
+md5sum = 93e1dda50cb71bfe29966b2946c02dd1
 filename = cgi-httpd.conf.in
 mode = 0644
 
@@ -119,7 +119,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 5f1b93ccdea7c3031aef396154c64938
+md5sum = 6c84a826778cb059754623f39b33651b
 destination = ${buildout:parts-directory}/monitor-template-rss-bin
 filename = status2rss.py
 mode = 0644

stack/monitor/cgi-httpd.conf.in

 PidFile "{{ httpd_configuration.get('pid-file') }}"
+
+StartServers 1
+ServerLimit 1
+ThreadLimit 4
+ThreadsPerChild 4
+
 ServerName example.com
 ServerAdmin someone@email
 <IfDefine !MonitorPort>

stack/monitor/monitor.py.in

 #!{{ python_executable }}
 
-import datetime
 import json
 import os
 import subprocess
@@ -36,30 +35,32 @@ option_list = [
 ]
 
 class Popen(subprocess.Popen):
-  __timeout = None
-
-  def timeout(self, delay, delay_before_kill=5):
-    if self.__timeout is not None: self.__timeout.cancel()
-    self.__timeout = threading.Timer(delay, self.stop, [delay_before_kill])
-    self.__timeout.start()
-    def waiter():
-        self.wait()
-        self.__timeout.cancel()
-    threading.Thread(target=waiter).start()
-
-  def stop(self, delay_before_kill=5):
-    if self.__timeout is not None: self.__timeout.cancel()
+
+  def set_timeout(self, timeout):
+    self.set_timeout = None # assert we're not called twice
+    event = threading.Event()
+    event.__killed = False # we just need a mutable
+    def t():
+      # do not call wait() or poll() because they're not thread-safe
+      if not event.wait(timeout) and self.returncode is None:
+        # race condition if waitpid completes just before the signal sent ?
         self.terminate()
-    t = threading.Timer(delay_before_kill, self.kill)
+        event.__killed = True
+        if event.wait(5):
+          return
+        if self.returncode is None:
+          self.kill() # same race as for terminate ?
+    t = threading.Thread(target=t)
+    t.daemon = True
     t.start()
-    r = self.wait()
-    t.cancel()
-    return r
-
-def init_db():
-  db = sqlite3.connect(db_path)
-  c = db.cursor()
-  c.executescript("""
+    def killed():
+      event.set()
+      t.join()
+      return event.__killed
+    return killed
+
+def init_db(db):
+  db.executescript("""
 CREATE TABLE IF NOT EXISTS status (
   timestamp INTEGER UNIQUE,
   status VARCHAR(255));
@@ -69,8 +70,6 @@ CREATE TABLE IF NOT EXISTS individual_status (
   element VARCHAR(255),
   output TEXT);
 """)
-  db.commit()
-  db.close()
 
 def getListOfScripts(directory):
   """
@@ -109,59 +108,43 @@ def runServices(directory):
 
 
 def runScripts(directory):
-  scripts = getListOfScripts(directory)
   # XXX script_timeout could be passed as parameters
   script_timeout = 60 # in seconds
   result = {}
-  for script in scripts:
-    command = [os.path.join(promise_dir, script)]
-    script = os.path.basename(command[0])
+  with open(os.devnull, 'r+') as f:
+    for script in getListOfScripts(directory):
+      command = os.path.join(promise_dir, script),
+      script = os.path.basename(script)
       result[script] = ''
 
-    process_handler = Popen(command,
-                            cwd=instance_path,
+      p = Popen(command, cwd=instance_path,
                 env=None if sys.platform == 'cygwin' else {},
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE,
-                            stdin=subprocess.PIPE)
-    process_handler.stdin.flush()
-    process_handler.stdin.close()
-    process_handler.stdin = None
-
-    process_handler.timeout(script_timeout)
-    process_handler.wait()
-
-    if process_handler.poll() is None:
-      process_handler.terminate()
+                stdin=f, stdout=f, stderr=subprocess.PIPE)
+      killed = p.set_timeout(script_timeout)
+      stderr = p.communicate()[1]
+      if killed():
         result[script] = "Time Out"
-    elif process_handler.poll() != 0:
-      stderr = process_handler.communicate()[1]
-      if stderr is not None:
+      elif p.returncode:
         result[script] = stderr.strip()
   return result
 
 
 def writeFiles(monitors):
   timestamp = int(time.time())
-  date = datetime.datetime.now().ctime()
-  init_db()
   db = sqlite3.connect(db_path)
-  fail = False
+  init_db(db)
+  status = SUCCESS
   for key, value in monitors.iteritems():
+    if value:
+      element_status = status = FAILURE
+    else:
       element_status = SUCCESS
-    if value != "" :
-      fail = True
-      element_status = FAILURE
     db.execute("insert into individual_status(timestamp, element, output, status) values (?, ?, ?, ?)", (timestamp, key, value, element_status))
-  db.commit()
-  status = SUCCESS
-  if fail:
-    status = FAILURE
   db.execute("insert into status(timestamp, status) values (?, ?)", (timestamp, status))
   db.commit()
   db.close()
-  monitors['datetime'] = date
-  open(monitoring_file_json, "w+").write(json.dumps(monitors))
+  monitors['datetime'] = time.ctime(timestamp)
+  json.dump(monitors, open(monitoring_file_json, "w+"))
 
 def main():
   parser = OptionParser(option_list=option_list)
@@ -183,10 +166,6 @@ def main():
     print json.dumps(monitors)
   else:
     writeFiles(monitors)
-  if len(monitors) == 0:
-    exit(0)
-  else:
-    exit(1)
 
 
 if __name__ == "__main__":

stack/monitor/status2rss.py

@@ -34,9 +34,12 @@ for row in rows:
 
   event_time = datetime.datetime.fromtimestamp(line_timestamp).strftime('%Y-%m-%d %H:%M:%S')
 
+  individual_rows = db.execute("select status, element, output from individual_status where timestamp=?", (line_timestamp,))
+  description = '\n'.join(['%s: %s %s' % row for row in individual_rows])
+
   rss_item = PyRSS2Gen.RSSItem(
     title = status,
-    description = "%s: %s" % (event_time, status),
+    description = "%s: %s\n%s" % (event_time, status, description),
     link = LINK,
     pubDate = event_time,
     guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, status)))

stack/resilient/buildout.cfg

@@ -38,7 +38,7 @@ eggs = collective.recipe.template
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = e89d8378cc610704b518a89b095d3a19
+md5sum = 7d7cb70cb76a41f853e74cc34a9ef428
 mode = 0644
 
 [pbsready-import]
@@ -47,7 +47,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = a5570ecfeff7a9d1b5f8be08db4feefe
+md5sum = dd13497575d13b92c3abb0a633777e2c
 mode = 0644
 
 [pbsready-export]
@@ -56,20 +56,20 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 879fff114d1dbf1f58774ccbce9bdd22
+md5sum = bfd71e454140cf13179d408e10f95bf8
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 3866b0d4d2872f693b7d9519a668e6bc
+md5sum = 0e6a95e7a9b38d402f94c11b7d10397e
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = c781ae17375c26b08b2a11bd9b800db8
+md5sum = 0641a65142fe0f624bf7c112081c2527
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
@@ -92,7 +92,7 @@ output = ${buildout:directory}/instance-frozen.cfg
 [resilient-web-takeover-cgi-script-download]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
-md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070
+md5sum = 3aa7624af1196062d7d01946d4de9f0e
 mode = 0644
 destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
 
@@ -121,11 +121,6 @@ mode = 0644
 find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
 
 [versions]
-# Pin Jinja2 to 2.6, as 2.7 breaks current code
-Jinja2 = 2.6
-# ... And newer s.r.template requires Jinja2 >= 2.7
-slapos.recipe.template = 2.4.2
 rdiff-backup = 1.3.4nxd2
-slapos.cookbook = 0.92
 
 

stack/resilient/instance-pull-backup.cfg.in

@@ -83,6 +83,7 @@ notifier-callbacks = $${basedirectory:notifier}/callbacks
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
+lockfile = $${basedirectory:run}/equeue.lock
 log = $${basedirectory:log}/equeue.log
 database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue

stack/resilient/pbsready-export.cfg.in

@@ -5,7 +5,6 @@ extends = ${pbsready:output}
 # Explicitely define extended parts from pbsready
 # then add local parts
 parts =
-  resiliency
   logrotate
   logrotate-entry-cron
   logrotate-entry-equeue

stack/resilient/pbsready-import.cfg.in

@@ -5,7 +5,6 @@ extends = ${pbsready:output}
 # Explicitely define extended parts from pbsready
 # then add local parts
 parts =
-  resiliency
   logrotate
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -18,6 +17,7 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+  resiliency-takeover-script
   resilient-web-takeover-cgi-script
   resilient-web-takeover-httpd-wrapper
   resilient-web-takeover-httpd-promise
@@ -48,6 +48,20 @@ recipe = slapos.cookbook:notifier.callback
 on-notification-id = $${slap-parameter:on-notification}
 callback = $${importer:wrapper}
 
+###########
+# Generate the takeover script
+###########
+[resiliency-takeover-script]
+recipe = slapos.cookbook:addresiliency
+wrapper-takeover = $${rootdirectory:bin}/takeover
+takeover-triggered-file-path = $${rootdirectory:srv}/takeover_triggered
+
+# Add path of file created by takeover script when takeover is triggered
+# Takeover script will create this file
+# equeue process will watch for file existence.
+[equeue]
+takeover-triggered-file-path = $${resiliency-takeover-script:takeover-triggered-file-path}
+
 ###########
 # Deploy a webserver allowing to do takeover from a web browser.
 ###########

stack/resilient/pbsready.cfg.in

 [buildout]
 
 parts =
-  resiliency
   logrotate
   logrotate-entry-cron
   logrotate-entry-equeue
@@ -126,18 +125,6 @@ frequency = daily
 rotate-num = 30
 
 
-#----------------
-#--
-#-- Resiliency script for the node takeover
-
-[resiliency]
-recipe = slapos.cookbook:addresiliency
-wrapper-takeover = takeover
-services = $${basedirectory:services}
-bin = $${rootdirectory:bin}
-etc = $${rootdirectory:etc}
-
-
 #----------------
 #--
 #-- Sets up an rdiff-backup server (with a dropbear server for ssh)
@@ -157,6 +144,7 @@ rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
+lockfile = $${basedirectory:run}/equeue.lock
 log = $${basedirectory:log}/equeue.log
 database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue

stack/resilient/resilient-web-takeover-cgi-script.py.in

 #!${buildout:executable}
+
+equeue_database = '${equeue:database}'
+equeue_lockfile = '${equeue:lockfile}'
+takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
+
 import cgi
 import cgitb
+import datetime
+import gdbm
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
 cgitb.enable()
 
+def getLatestBackupDate():
+  """
+  Get the date of the latest successful backup.
+  """
+  # Create a copy of the db (locked by equeue process)
+  temporary_directory = tempfile.mkdtemp()
+  equeue_database_copy = os.path.join(temporary_directory, 'equeue.db')
+  shutil.copyfile(equeue_database, equeue_database_copy)
+  db = gdbm.open(equeue_database_copy)
+  # Usually, there is only one callback (so only one key
+  # in the db), but if there are several:
+  # Take the "oldest" one (oldest value).
+  last_backup = db[db.keys()[0]]
+  for callback in db.keys():
+    timestamp = float(db[callback])
+    if timestamp < last_backup:
+      last_backup = timestamp
+  return datetime.datetime.fromtimestamp(last_backup)
+
+def isBackupInProgress():
+  """
+  Check if backup is in progress (importer script is running)
+  by checking if equeue lockfile exists.
+  """
+  # XXX: check if file is valid
+  return os.path.exists(equeue_lockfile)
+
 print "Content-Type: text/html"
 print
 
@@ -17,12 +53,14 @@ if "password" not in form:
   <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
   <p><b>Warning: submit the form only if you understand what you are doing.</b></p>
   <p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p>
+  <p>Last valid backup: %s</p>
+  <p>Importer script(s) of backup in progress: %s</p>
   <form action="/">
     Password: <input type="text" name="password">
     <input type="submit" value="Take over" style="background: red;">
   </form>
 </body>
-</html>"""
+</html>""" % (getLatestBackupDate().strftime('%Y-%m-%d %H:%M:%S'), isBackupInProgress())
   sys.exit(0)
 
 if form['password'].value != '${:password}':
@@ -31,6 +69,6 @@ if form['password'].value != '${:password}':
   sys.exit(1)
 
 # XXX hardcoded location
-result = subprocess.check_output([os.path.expanduser("~/bin/takeover")], stderr=subprocess.STDOUT)
+result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT)
 print 'Success.'
 print '<pre>%s</pre>' % result

stack/resilient/template-replicated.cfg.in

@@ -70,7 +70,7 @@ name = {{namebase}}{{id}}
 
 software-url = ${slap-connection:software-release-url}
 software-type = {{typeimport}}
-return = ssh-public-key ssh-url notification-url ip
+return = ssh-public-key ssh-url notification-url ip takeover-url takeover-password
 
 pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
 
@@ -99,6 +99,8 @@ sla-{{ key }} = {{ value }}
 
 [publish-connection-informations]
 feed-url-{{namebase}}-{{id}}-push = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
+takeover-{{namebase}}-{{id}}-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-url}
+takeover-{{namebase}}-{{id}}-password = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-password}
 
 {% endfor -%}
 

stack/slapos.cfg

@@ -20,9 +20,11 @@ extensions +=
 
 # Use shacache and lxml
 extends =
-  ../component/python-2.7/buildout.cfg
+  ../component/git/buildout.cfg
   ../component/lxml-python/buildout.cfg
+  ../component/python-2.7/buildout.cfg
   ../component/python-cffi/buildout.cfg
+  ../component/python-cliff/buildout.cfg
   ../component/python-cryptography/buildout.cfg
 
 # Separate from site eggs
@@ -84,10 +86,11 @@ recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   ${python-cffi:egg}
+  ${python-cliff:egg}
   ${python-cryptography:egg}
   pyOpenSSL
   slapos.cookbook
-  cliff
+  slapos.libnetworkcache
   hexagonit.recipe.download
   inotifyx
   netaddr
@@ -100,105 +103,106 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.7.1-dev-SlapOS-003
+zc.buildout = 1.7.1.post9
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
-zc.recipe.egg = 1.3.2nxd001
-# Use own version of h.r.download to be able to open xz-like archives
-hexagonit.recipe.download = 1.7nxd002
+zc.recipe.egg = 1.3.2.post4
+# Use own version of h.r.download to be able to open .xz and .lz archives
+hexagonit.recipe.download = 1.7.post4
 
 Jinja2 = 2.7.3
 MarkupSafe = 0.23
-Werkzeug = 0.9.6
+Werkzeug = 0.10.4
 buildout-versions = 1.7
-cffi = 0.8.6
-cmd2 = 0.6.7
-cryptography = 0.6
+cffi = 1.1.2
+cmd2 = 0.6.8
+collective.recipe.template = 1.11
+cryptography = 0.9.1
+inotifyx = 0.2.2
 itsdangerous = 0.24
-lxml = 3.4.0
-meld3 = 1.0.0
-mr.developer = 1.31
+lxml = 3.4.4
+meld3 = 1.0.2
+mr.developer = 1.33
+netaddr = 0.7.14
+pbr = 1.1.1
 prettytable = 0.7.2
-pyOpenSSL = 0.14
+psutil = 2.2.1
+pyOpenSSL = 0.15.1
 pyparsing = 2.0.3
-setuptools = 7.0
-six = 1.8.0
-simplejson = 3.6.5
-slapos.cookbook = 0.87
-slapos.core = 1.3.4
+pytz = 2015.4
+requests = 2.7.0
+setuptools = 16.0
+simplejson = 3.7.3
+six = 1.9.0
+slapos.cookbook = 0.102
+slapos.core = 1.3.10
+slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.2
-slapos.recipe.build = 0.14
+slapos.recipe.build = 0.21
 slapos.recipe.cmmi = 0.2
-stevedore = 1.1.0
+stevedore = 1.5.0
+xml-marshaller = 0.9.7
+z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==1.3.4
+# slapos.core==1.3.10
 Flask = 0.10.1
 
 # Required by:
-# cliff==1.8.0
-# stevedore==1.1.0
-argparse = 1.2.2
+# cliff==1.13.0
+# stevedore==1.5.0
+argparse = 1.3.0
 
 # Required by:
-# slapos.core==1.3.4
-cliff = 1.8.0
-
-# Required by:
-# slapos.cookbook==0.87
-inotifyx = 0.2.2
+# slapos.core==1.3.10
+cliff = 1.13.0
 
 # Required by:
-# slapos.cookbook==0.87
-lock-file = 2.0
+# cryptography==0.9.1
+enum34 = 1.0.4
 
 # Required by:
-# slapos.cookbook==0.87
-netaddr = 0.7.12
+# jsonschema==2.5.1
+functools32 = 3.2.3.post1
 
 # Required by:
-# slapos.core==1.3.4
-netifaces = 0.10.4
+# cryptography==0.9.1
+idna = 2.0
 
 # Required by:
-# cliff==1.8.0
-# stevedore==1.1.0
-pbr = 0.10.0
+# cryptography==0.9.1
+ipaddress = 1.0.7
 
 # Required by:
-# pbr==0.10.0
-pip = 1.5.6
+# slapos.cookbook==0.102
+jsonschema = 2.5.1
 
 # Required by:
-# slapos.core==1.3.4
-psutil = 2.1.3
+# slapos.cookbook==0.102
+lock-file = 2.0
 
 # Required by:
-# cffi==0.8.6
-pycparser = 2.10
+# slapos.core==1.3.10
+netifaces = 0.10.4
 
 # Required by:
-# slapos.cookbook==0.87
-pytz = 2014.9
+# cryptography==0.9.1
+pyasn1 = 0.1.7
 
 # Required by:
-# slapos.core==1.3.4
-requests = 2.4.3
+# cffi==1.1.2
+pycparser = 2.14
 
 # Required by:
-# slapos.core==1.3.4
+# slapos.core==1.3.10
 supervisor = 3.1.3
 
 # Required by:
-# slapos.core==1.3.4
+# slapos.core==1.3.10
 uritemplate = 0.6
 
 # Required by:
-# slapos.cookbook==0.87
-xml-marshaller = 0.9.7
-
-# Required by:
-# slapos.core==1.3.4
-zope.interface = 4.1.1
+# slapos.core==1.3.10
+zope.interface = 4.1.2
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache