Merge branch 'master' into erp5-cluster

component/dash/buildout.cfg

@@ -6,8 +6,8 @@ parts = dash-output
 
 [dash]
 recipe = slapos.recipe.cmmi
-url = http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.7.tar.gz
-md5sum = f6cedb10ae7258adb5ab17a10ae80d51
+url = http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.8.tar.gz
+md5sum = 5c152209680dab3c319e8923f6c51378
 configure-options =
   --disable-static
   --disable-fnmatch

component/dbus/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [dbus]
 recipe = slapos.recipe.cmmi
-url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.8.tar.gz
-md5sum = b9f4a18ee3faa1e07c04aa1d83239c43
+url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.10.tar.gz
+md5sum = 6be5ef99ae784de9d04589eb067fe038
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 configure-options =
   --disable-static

component/git/buildout.cfg

@@ -8,6 +8,7 @@ extends =
   ../libexpat/buildout.cfg
   ../openssl/buildout.cfg
   ../pcre/buildout.cfg
+  ../perl/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
 parts =
@@ -23,8 +24,8 @@ configure-options =
   --with-zlib=${zlib:location}
   --with-libpcre=${pcre:location}
   --with-expat=${libexpat:location}
+  --with-perl=${perl:location}/bin/perl
 environment =
-  NO_PERL=y
   NO_PYTHON=y
   NO_TCLTK=y
   PATH=${curl:location}/bin:${gettext:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/glib/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [glib]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/glib-2.40.0.tar.xz
-md5sum = 05fb7cb17eacbc718e90366a1eae60d9
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/glib-2.42.1.tar.xz
+md5sum = 89c4119e50e767d3532158605ee9121a
 configure-options =
   --disable-static
   --disable-selinux

component/glibmm/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [glibmm]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/glibmm-2.40.0.tar.xz
-md5sum = f62754f4f5c9030f8ff43c7ed20556c2
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/glibmm-2.42.0.tar.xz
+md5sum = 7c52cc42085d30ac3b73d74c3f2eb22e
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig
 configure-options =
   --disable-documentation

component/grep/buildout.cfg

@@ -8,8 +8,8 @@ parts =
 
 [grep]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/grep/grep-2.20.tar.xz
-md5sum = 2cbea44a4f1548aee20b9ff2d3076908
+url = http://ftp.gnu.org/gnu/grep/grep-2.21.tar.xz
+md5sum = 43c48064d6409862b8a850db83c8038a
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
   CPPFLAGS=-I${pcre:location}/include

component/gtk-2/buildout.cfg

@@ -65,8 +65,8 @@ environment =
 
 [pango]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/pango-1.36.3.tar.xz
-md5sum = d9532826e95bdb374355deebc42441bb
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/pango-1.36.8.tar.xz
+md5sum = 217a9a753006275215fa9fa127760ece
 pkg_config_depends = ${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
 configure-options =
   --disable-static
@@ -79,8 +79,8 @@ environment =
 
 [gdk-pixbuf]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/gdk-pixbuf-2.30.7.tar.xz
-md5sum = 1195d26f14adfe99ff0878d37678a70a
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/gdk-pixbuf-2.31.1.tar.xz
+md5sum = 74cde211f5b7ac1015d1a7c9feee037c
 pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}
 configure-options =
   --disable-static
@@ -96,8 +96,8 @@ environment =
 
 [atk]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/atk-2.12.0.tar.xz
-md5sum = 930238dec55fdbf8eda9975b44f07b76
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/atk-2.14.0.tar.xz
+md5sum = ecb7ca8469a5650581b1227d78051b8b
 environment =
   PATH=${glib:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
@@ -106,8 +106,8 @@ environment =
 
 [gtk-2]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/gtk+-2.24.23.tar.xz
-md5sum = 0be39fbed4ca125645175cd6e22f2548
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/gtk+-2.24.25.tar.xz
+md5sum = 612350704dd3aacb95355a4981930c6f
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/librsvg/buildout.cfg

@@ -23,8 +23,8 @@ environment =
 
 [librsvg]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/librsvg-2.40.2.tar.xz
-md5sum = acdecdb9f08f3bf662a68bf7dafb8b82
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/librsvg-2.40.5.tar.xz
+md5sum = c2b044fccf415902a052d0e978e0ea60
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${zlib:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/libsigc/buildout.cfg

@@ -9,8 +9,8 @@ parts =
 
 [libsigc]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/libsigc++-2.3.1.tar.xz
-md5sum = 06a0569e8ba161d3428e7daa838682f8
+url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/libsigc++-2.4.0.tar.xz
+md5sum = c6cd2259f5ef973e4c8178d0abbdbfa7
 configure-options =
   --disable-documentation
 environment =

component/libtool/buildout.cfg

 [buildout]
+extends =
+  ../xz-utils/buildout.cfg
 parts = libtool
 
 [libtool]
 recipe = slapos.recipe.cmmi
-md5sum = d2f3b7d4627e69e13514a40e72a24d50
-url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.2.tar.gz
+md5sum = b5699a6d58f5594cdb0992c5e1f5e57e
+url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.3.tar.xz
 configure-options =
   --disable-static
+environment =
+  PATH=${xz-utils:location}/bin:%(PATH)s

component/lmsensors/buildout.cfg

+
+[buildout]
+
+extends = 
+  ../bison/buildout.cfg
+  ../rrdtools/buildout.cfg
+  ../flex/buildout.cfg
+
+parts =
+  lmsensors
+
+[lmsensors]
+recipe = slapos.recipe.cmmi
+md5sum = da506dedceb41822e64865f6ba34828a
+url = http://dl.lm-sensors.org/lm-sensors/releases/lm_sensors-3.3.5.tar.bz2
+configure-command = true
+make-options =
+  PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
+  ETCDIR=${buildout:parts-directory}/${:_buildout_section_name_}/etc
+
+environment = 
+  PATH=${bison:location}/bin:${flex:location}/bin:%(PATH)s
+

component/mariadb/buildout.cfg

@@ -21,9 +21,9 @@ parts =
 
 [mariadb]
 recipe = slapos.recipe.cmmi
-version = 10.0.14
+version = 10.0.15
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
-md5sum = 80fea71de54a9cfa7f5508df53d3f06d
+md5sum = 07e2ca1ad684c2dc5cb260717c4db41d
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
@@ -42,6 +42,7 @@ configure-options =
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
+  -DWITHOUT_MROONGA_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
   -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
   -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"

component/mesa/buildout.cfg

@@ -38,6 +38,8 @@ environment =
   ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
   AUTOCONF=${autoconf:location}/bin/autoconf
   AUTOMAKE=${automake:location}/bin/automake
+make-options =
+  LIBTOOL=${libtool:location}/bin/libtool
 
 [glu]
 recipe = slapos.recipe.cmmi

component/nginx/buildout.cfg

@@ -11,8 +11,8 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.7.6.tar.gz
-md5sum = dd444e5333e0d324bec480e2ff67870a
+url = http://nginx.org/download/nginx-1.7.7.tar.gz
+md5sum = 3beaa25fc87ff2a75ab1b46174dc5ebf
 
 [nginx]
 <= nginx-common

component/perl/buildout.cfg

@@ -7,9 +7,9 @@ parts =
 
 [perl]
 recipe = slapos.recipe.cmmi
-version = 5.20.0
+version = 5.20.1
 url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
-md5sum = 20cbecd4e9e880ee7a50a136c8b1484e
+md5sum = ede5166f949d9a07163bc5b086be9759
 siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
 patch-options = -p1
 patches =
@@ -25,4 +25,4 @@ configure-command =
     -Dusethreads
 environment =
   PATH=${patch:location}/bin:%(PATH)s
-post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#7fded8308c1676decf77575c6d6b325f:post_make_hook
+post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#539cb3cd0d1090f7f30a8e5a82b37854:post_make_hook

component/perl/perl-create-libs-symlink.py

@@ -15,7 +15,7 @@ def post_make_hook(options, buildout):
     print "ERROR - no libperl.* found!"
     exit
   elif nr_matches > 1:
-    print "WARNING - several libperl.a found, taking only the first one:", matches.join("\n")
+    print "WARNING - several libperl.a found, taking only the first one:", "\n".join(matches)
   
   # matches[0] is a prefix of "location"
   # For the symlink, we want the relative path.

component/rrdtools/buildout.cfg

+
+[buildout]
+
+parts =
+  rrdtools
+
+[rrdtools]
+recipe = slapos.recipe.cmmi
+md5sum = dbe59386db97fd2f2216729facd74ca8
+url = http://oss.oetiker.ch/rrdtool/pub/rrdtool-1.4.8.tar.gz

component/rsync/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [rsync]
 recipe = slapos.recipe.cmmi
-url = http://rsync.samba.org/ftp/rsync/src/rsync-3.0.9.tar.gz
-md5sum = 5ee72266fe2c1822333c407e1761b92b
+url = http://rsync.samba.org/ftp/rsync/src/rsync-3.1.1.tar.gz
+md5sum = 43bd6676f0b404326eee2d63be3cdcfe
 make-options =
   PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}

component/smartmontools/buildout.cfg

+
+[buildout]
+
+parts =
+  smartmontools
+
+[smartmontools]
+recipe = slapos.recipe.cmmi
+md5sum = 2ea0c62206e110192a97b59291b17f54 
+url = http://freefr.dl.sourceforge.net/project/smartmontools/smartmontools/6.3/smartmontools-6.3.tar.gz 
+
+configure-option = 
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}

component/w3-validator/buildout.cfg

@@ -35,6 +35,6 @@ depends =
   ${perl-XML-LibXML:location}
   ${perl-libwww-perl:location}
 perl-path = ${perl:location}/bin/perl
-# validator-1_2-release in http://dvcs.w3.org/hg/markup-validator/tags
-validator-url = http://dvcs.w3.org/hg/markup-validator/archive/754b7a64cdb4.tar.bz2
+# validator-1_3-release in https://dvcs.w3.org/hg/markup-validator/tags
+validator-url = https://dvcs.w3.org/hg/markup-validator/archive/6c6c7c1f69c5.tar.bz2
 dtd-url =

slapos/recipe/dcron.py

@@ -76,10 +76,12 @@ class Part(GenericBaseRecipe):
 day_of_week_dict = dict((name, dow) for dow, name in enumerate(
     "sunday monday tuesday wednesday thursday friday saturday".split())
   for name in (name, name[:3]))
-symbolic_dict = dict(hourly  = '0 * * * *',
+symbolic_dict = dict(minutely = '* * * * *',
+                     hourly   = '0 * * * *',
                      daily    = '0 0 * * *',
+                     weekly   = '0 0 * * 0',
                      monthly  = '0 0 1 * *',
-                     weekly  = '0 0 * * 0')
+                     yearly   = '0 0 1 1 *')
 
 def systemd_to_cron(spec):
   """Convert from systemd.time(7) calendar spec to crontab spec"""
@@ -118,13 +120,22 @@ def systemd_to_cron(spec):
     raise ValueError
   month, day = day
   hour, minute = time
-  spec = minute, hour, day, month, dow
-  for x, (y, z) in zip(spec, ((0, 60), (0, 24), (1, 31), (1, 12))):
+  spec = [minute, hour, day, month, dow]
+  for i, (y, z) in enumerate(((0, 60), (0, 24), (1, 31), (1, 12))):
+    x = spec[i]
     if x != '*':
       for x in x.split(','):
         x = map(int, x.split('/', 1))
-        x[0] -= y
-        if x[0] < 0 or len(x) > 1 and x[0] >= x[1] or z <= sum(x):
+        a = x[0] - y
+        if 0 <= a < z:
+          if len(x) == 1:
+            continue
+          b = x[1]
+          if b > 0:
+            a = (z - a - 1) // b * b
+            if a:
+              spec[i] = '%s-%s/%s' % (x[0], x[0] + a, b)
+              continue
         raise ValueError
   return ' '.join(spec)
 

slapos/recipe/logrotate.py

@@ -65,6 +65,7 @@ class Part(GenericBaseRecipe):
       'dateext',
       'rotate 3650',
       'compress',
+      'delaycompress',
       'notifempty',
       'sharedscripts',
       'create',

slapos/recipe/pbs.py

@@ -244,6 +244,9 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
                                       promise_dict)
     path_list.append(promise)
 
+    # Create known_hosts file by default.
+    # In some case, we don't want to create it (case where we share IP mong partitions)
+    if not self.isTrueValue(self.options.get('ignore-known-hosts-file')):
       known_hosts_file[parsed_url.hostname] = entry['server-key']
 
     notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)

slapos/recipe/tidstorage/__init__.py

@@ -33,14 +33,15 @@ class Recipe(GenericBaseRecipe):
       self.options['configuration-path'],
       self.substituteTemplate(
         self.getTemplateFilename('tidstorage.py.in'), self.options))
+    r = [configuration_file]
 
-    tidstorage_wrapper = self.createPythonScript(
-      self.options['tidstorage-wrapper'],
+    wrapper = self.options.get('tidstorage-wrapper')
+    wrapper and r.append(self.createPythonScript(wrapper,
       'slapos.recipe.librecipe.execute.execute',
       [self.options['tidstoraged-binary'],  '--nofork', '--config',
-        configuration_file])
+        configuration_file]))
 
-    repozo_wrapper = self.createPythonScript(
+    r.append(self.createPythonScript(
       self.options['repozo-wrapper'],
       'slapos.recipe.librecipe.execute.execute',
       [self.options['tidstorage-repozo-binary'],
@@ -48,6 +49,6 @@ class Recipe(GenericBaseRecipe):
         '--repozo', self.options['repozo-binary'],
         '--gzip',
         '--quick',
-      ])
+      ]))
 
-    return [configuration_file, tidstorage_wrapper, repozo_wrapper]
+    return r

slapos/test/recipe/test_dcron.py

@@ -13,12 +13,13 @@ class TestDcron(unittest.TestCase):
     _("10-15", "0 0 15 10 *")
     _("monday *-12-* 17:00", "00 17 * 12 1")
     _("12,14,13,12:20,10,30", "20,10,30 12,14,13,12 * * *") # TODO: sort
-    _("*-1/2-1,3 *:30", "30 * 1,3 1/2 *")
+    _("*-1/2-1,3 *:30", "30 * 1,3 1-11/2 *")
     _("03-05 08:05", "05 08 05 03 *")
     _("08:05:00", "05 08 * * *")
     _("05:40", "40 05 * * *")
     _("Sat,Sun 12-* 08:05", "05 08 * 12 0,6")
     _("Sat,Sun 08:05", "05 08 * * 0,6")
+    _("*:25/20", "25-45/20 * * * *")
 
     def _(systemd):
       self.assertRaises(Exception, systemd_to_cron, systemd)
@@ -31,7 +32,7 @@ class TestDcron(unittest.TestCase):
     _("08:05:40")
     _("2003-03-05")
 
-    _("0-1"); _("13-1"); _("6/4-1"); _("5/8-1")
-    _("1-0"); _("1-32"); _("1-4/3"); _("1-14/18")
-    _("24:0");_("9/9:0"); _("8/16:0")
-    _("0:60"); _("0:22/22"); _("0:15/45")
+    _("0-1"); _("13-1"); _("8/5-1")
+    _("1-0"); _("1-32"); _("1-14/18")
+    _("24:0"); _("8/16:0")
+    _("0:60"); _("0:15/45")

software/cloudooo/software.cfg

@@ -21,6 +21,7 @@ cert = ${slap-connection:cert-file}
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended

software/erp5testnode/software.cfg

@@ -9,14 +9,8 @@ extends =
   ../../component/pwgen/buildout.cfg
   ../../component/apache/buildout.cfg
 
-# Local development
-develop =
-  ${:parts-directory}/slapos.cookbook-repository
 parts =
-# Local development
   slapos-cookbook
-  slapos.cookbook-repository
-  check-recipe
   template
   lxml-python
   eggs
@@ -24,20 +18,6 @@ parts =
   git
   apache
 
-# Local development
-[slapos.cookbook-repository]
-recipe = plone.recipe.command
-stop-on-error = true
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-command = "${git:location}/bin/git" clone --branch erp5testnode --quiet http://git.erp5.org/repos/slapos.git "${:location}"
-update-command = cd "${:location}" && "${git:location}/bin/git" fetch --quiet && "${git:location}/bin/git" reset --hard @{upstream}
-
-[check-recipe]
-recipe = plone.recipe.command
-stop-on-error = true
-update-command = ${:command}
-command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
-
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -79,7 +59,10 @@ md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
 
 [versions]
 PyXML = 0.8.5
-erp5.util = 0.4.41
-plone.recipe.command = 1.1
+erp5.util = 0.4.42
+slapos.cookbook = 0.92
 slapos.recipe.template = 2.5
+
+# Required by:
+# slapos.cookbook==0.92
 jsonschema = 2.4.0

software/kvm/common.cfg

@@ -48,6 +48,7 @@ eggs =
   slapos.toolbox
   erp5.util
   cns.recipe.symlink
+  collective.recipe.template
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -100,14 +101,14 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 412d8540e054aa436bee723d85213a2b
+md5sum = 7071b61a862ee681c2e4323e6435e98d
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = 080b5790b95a979e44c4a8e26dc02835
+md5sum = e3d275621420f0b8c081228caeb571f9
 mode = 0644
 download-only = true
 on-update = true

software/kvm/development.cfg

@@ -6,30 +6,38 @@ extends =
 parts +=
   slapos.cookbook-repository
   slapos.toolbox-repository
+  slapos.core-repository
   erp5.util-repository
   check-recipe
 
 develop =
   ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.core-repository
   ${:parts-directory}/slapos.toolbox-repository
   ${:parts-directory}/erp5.util-repository
 
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = kvm
+branch = master
+git-executable = ${git:location}/bin/git
+
+[slapos.core-repository]
+recipe = slapos.recipe.build:gitclone
+repository = http://git.erp5.org/repos/slapos.core.git
+branch = master
 git-executable = ${git:location}/bin/git
 
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = kvmresiliency
+branch = master
 git-executable = ${git:location}/bin/git
 
 [erp5.util-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
-branch = scalability-master2
+branch = master
 git-executable = ${git:location}/bin/git
 
 [check-recipe]
@@ -38,5 +46,12 @@ stop-on-error = true
 update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
+
+[versions]
+slapos.cookbook =
+slapos.core =
+slapos.toolbox =
+erp5.util =
\ No newline at end of file

software/kvm/instance-for-erp5testnode.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:test}
+kvm = $${dynamic-template-kvm:rendered}
+nbd = ${template-nbd:output}
+frontend = ${template-frontend:output}
+
+kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
+kvm-import = ${template-kvm-import:output}
+kvm-export = $${dynamic-template-kvm-export:rendered}
+
+# Used for the test of resiliency. The system wants a "test" software_type.
+test = $${dynamic-template-kvm-resilient-test:rendered}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[dynamic-template-kvm]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm:location}/instance-kvm.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw curl_executable_location ${curl:location}/bin/curl
+    raw dash_executable_location ${dash:location}/bin/dash
+    raw dcron_executable_location ${dcron:location}/sbin/crond
+    raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
+    raw novnc_location ${noVNC:location}
+    raw openssl_executable_location ${openssl:location}/bin/openssl
+    raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
+    raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
+    raw sixtunnel_executable_location ${6tunnel:location}/bin/6tunnel
+    raw websockify_executable_location ${buildout:directory}/bin/websockify
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+
+[dynamic-template-kvm-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm-resilient.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw curl_executable_location ${curl:location}/bin/curl
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+
+[dynamic-template-kvm-export]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm-export.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    raw kvm_template $${dynamic-template-kvm:rendered}
+    raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
+    raw pbsready_export_template ${pbsready-export:output}
+    raw gzip_binary ${gzip:location}/bin/gzip
+mode = 0644
+
+[dynamic-template-kvm-resilient-test]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+    raw bin_directory ${buildout:bin-directory}
+mode = 0644

software/kvm/instance-kvm-resilient-input-schema.json

@@ -40,6 +40,12 @@
                 "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
                 "type": "integer",
                 "default": 2
+            },
+            "ignore-known-hosts-file": {
+                "title": "Ignore known_hosts file",
+                "description": "Set either to fill known_hosts file for ssh or not. Useful if main instance and PBS are using the same IP (slapos proxy, webrunner).",
+                "type": "boolean",
+                "default": false
             }
         }
     }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -5,7 +5,8 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 parts =
-  deploy-resiliency-test
+  deploy-unit-test
+  deploy-scalability-test
   request-resilient-kvm
   deploy-standalone-resiliency-test
 
@@ -21,20 +22,30 @@ log = ${:var}/log/
 services = ${:etc}/service/
 scripts = ${:etc}/run/
 
-[deploy-resiliency-test]
+[deploy-scalability-test]
 recipe = slapos.cookbook:wrapper
-wrapper-path = ${directory:scripts}/runKVMResiliencyTestSuite
+# XXX: put it in etc/run in case of scalability test so that it runs automatically.
+wrapper-path = ${directory:bin}/runKVMResiliencyTestSuite
 testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
-kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm kvm_rootinstance_name='${request-resilient-kvm:name}'
-
-command-line = {{ bin_directory }}/runResiliencyTest ${:testnode-parameters} ${:kvm-test-parameters}
+kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm root_instance_name='${request-resilient-kvm:name}'
+command-line = {{ bin_directory }}/runResiliencyScalabilityTestNode ${:testnode-parameters} ${:kvm-test-parameters}
 
+[deploy-unit-test]
+recipe = collective.recipe.template
+#testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
+kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm root_instance_name='${request-resilient-kvm:name}'
+input = inline:
+  #!/bin/sh
+  exec {{ bin_directory }}/runResiliencyUnitTestTestNode $@ ${:kvm-test-parameters}
+output = ${directory:bin}/runTestSuite
+mode = 755
 
 [deploy-standalone-resiliency-test]
 # Used to manually run the KVM test if we don't have a running testnode.
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${directory:bin}/runStandaloneResiliencyTestSuite
-command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=kvm ${deploy-resiliency-test:kvm-test-parameters}
+command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=kvm ${:kvm-test-parameters}
+kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm root_instance_name='${request-resilient-kvm:name}'
 
 [request-resilient-kvm]
 <= slap-connection
@@ -48,9 +59,12 @@ config-{{ key }} = {{ dumps(value) }}
 config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
 config-resiliency-backup-periodicity = */5 * * * *
+config-resilient-clone-number = 1
+config-ignore-known-hosts-file = true
+return = ipv6
 # XXX What to do?
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]
-virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/fbd4ad.php?dl=true
+virtual-hard-drive-url = http://www.nexedi.org/static/slapos/kvm_resiliency_test/virtual.qcow2
 virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -32,12 +32,13 @@ return =
   url ssh-public-key ssh-url notification-id ip
 # KVM related parameters
 # XXX: return ALL parameters (like nat rules), through jinja
-  backend-url url
+  backend-url url ip
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
 backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
+ipv6 = ${request-kvm:connection-ip}
 
 [kvm-frontend-url-promise]
 # Check that url parameter is complete

software/kvm/software-for-erp5testnode.cfg

+[buildout]
+extends = development.cfg
+
+# Change default software-type to be "test", so that it can be run using erp5testnode.
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in
+md5sum = e8afd5aa5b41df79238e9a84984a6aa5
+output = ${buildout:directory}/template.cfg
+mode = 0644
\ No newline at end of file

software/monitor/instance.cfg → software/monitor/instance.cfg.in

@@ -19,6 +19,8 @@ parts =
   monitor-promise
   monitor-instance-log-access
   cron-rsync-logs
+  cron-smart-monitor-logs
+  cron-network-bench
   
 extends = ${monitor-template:output}
 
@@ -34,6 +36,20 @@ name = rsync-logs
 frequency = * * * * *
 command = $${rsync-logs:rendered}
 
+[cron-smart-monitor-logs]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = smartctl-mon
+frequency = 11 * * * *
+command = $${smartmon-logs:rendered}
+
+[cron-network-bench]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = network-bench-test
+frequency = */3 * * * *
+command = ${buildout:bin-directory}/networkbench $${network-bench-configuration:rendered} $${monitor-directory:monitor-log}
+
 [rsync-logs]
 recipe = slapos.recipe.template:jinja2
 template = ${rsync-logs-script:output}
@@ -43,6 +59,27 @@ context =
   key buildout_directory buildout:directory
   raw rsync_binary ${rsync:location}/bin/rsync
   key server_log_directory monitor-directory:server-log 
+  raw re6st_log_directory /var/log/re6stnet
+  key system_log_directory monitor-directory:system-log 
+
+[smartmon-logs]
+recipe = slapos.recipe.template:jinja2
+template = ${smartmon-logs-script:output}
+rendered = $${monitor-directory:bin}/smartmon-logs.py
+mode = 0744
+context =
+  key buildout_directory buildout:directory
+  raw smartctl_binary ${smartmontools:location}/sbin/smartctl
+  key monitor_log_directory monitor-directory:monitor-log 
+  raw python_executable ${buildout:executable}
+
+[network-bench-configuration]
+recipe = slapos.recipe.template:jinja2
+template = ${network-bench-cfg:output}
+rendered = $${monitor-directory:etc}/network_bench.cfg
+mode = 0744
+context =
+  
 
 [pwgen]
 recipe = slapos.cookbook:generate.password
@@ -54,11 +91,12 @@ recipe = slapos.cookbook:generate.password
 user = admin
 bytes = 16
 
-
 [monitor-directory]
 server-log = $${:private-directory}/server-log
+monitor-log = $${:private-directory}/monitor-log
 cache = $${:var}/cache
 mod-ssl = $${:cache}/httpd_mod_ssl
+system-log = $${:private-directory}/system-log 
 
 [slap-parameter]
 private-hash = $${pwgen:passwd}$${pwgen32:passwd}
@@ -71,6 +109,7 @@ name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
+config = url domain
 config-url = $${monitor-parameters:url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain

software/monitor/network_bench.cfg

+[network_bench]
+
+url =
+    http://www.tiolive.com/robots.txt
+    http://www.erp5.com/robots.txt
+    http://www.erp5.cn/robots.txt
+    http://www.nexedi.cn/robots.txt
+    http://www.osoe-project.cn/robots.txt
+    http://www.nexedi.cn/robots.txt
+    http://www.osoe-project.org/robots.txt
+    http://www.osoe-project.cn/robots.txt
+    http://www.google.com/robots.txt
+    http://ipv6.google.com/robots.txt
+
+    https://[2001:67c:1254:e:a7::2e20]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:69::115c]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:8a::2621]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:e:c7::89ae]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:e:93::2e31]:9685/monitor-public/rssfeed.html
+    https://[2001:67c:1254:1f::2723]:9685/monitor-public/rssfeed.html
+    https://[2001:41d0:1:a78b::3c2a]:9685/monitor-public/rssfeed.html
+    http://[2001:67c:1254:4::1]/index.html
+
+dns =
+    erp5.com
+    www.erp5.com
+    erp5.cn
+    www.erp5.cn
+    nexedi.cn
+    www.nexedi.com
+    www.nexedi.cn
+    www.osoe-project.org
+    www.osoe-project.cn
+    free.fr
+    google.com
+
+ping =
+    erp5.com
+    erp5.cn
+    google.com
+    free.fr
+    frontend1.nexedi.com
+    frontend2.nexedi.com
+    frontend3.nexedi.cn
+    42.192.4.15
+    195.208.185.22

software/monitor/rsync-logs.sh.in

@@ -2,6 +2,13 @@
 
 RSYNC_BIN={{ rsync_binary }}
 SLAPGRID_LOGS={{ buildout_directory }}/../var/data-log/
+RE6STLOGS={{ re6st_log_directory }}
+
+SYSTEMLOGS={{ system_log_directory }}
 SERVERLOGS={{ server_log_directory }}
 
 $RSYNC_BIN -a --delete $SLAPGRID_LOGS $SERVERLOGS
+
+if [ -d $RE6STLOGS ]; then
+  $RSYNC_BIN -a --delete $RE6STLOGS $SYSTEMLOGS
+fi

software/monitor/smartmon.py.in

+#!{{ python_executable }}
+
+SMARCTBIN={{ smartctl_binary }}
+LOGFOLDER={{ monitor_log_directory }}
+
+import subprocess
+import logging
+import os
+
+logger = logging.getLogger("SMARTMON")
+logger.setLevel(logging.DEBUG)
+
+ch = logging.FileHandler(os.path.join(LOGFOLDER, "smartmontools.log"))
+ch.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+def _call(cmd):
+  p = subprocess.Popen(
+     cmd,
+     stdout=subprocess.PIPE,
+     stdin=subprocess.PIPE,
+     shell=True)
+
+  return p.communicate()
+
+def get_disk_list():
+  output, err = _call("%s --scan" % SMARCTBIN)
+  disk_list = []
+  for line in output.split("\n"):
+    if line != '':
+      disk_path = line.split(" ")[0]
+      if disk_path.startswith("/dev/"):
+        disk_list.append(disk_path)
+
+  return disk_list
+
+logger.debug("Starting new Data collection.")
+
+disk_list = get_disk_list()
+
+logger.debug("Identified disks: %s" % disk_list)
+
+for disk in disk_list:
+  logger.info("Starting test for %s" % disk)
+  output, err = _call("%s --all %s" % (SMARCTBIN, disk))
+  for l in output.split("\n"):
+    logger.info("%s - %s" % (disk, l))
+  if err is not None:
+    logger.warning(err)
+
+logger.debug("Stopped to Collect Data.")

software/monitor/software.cfg

@@ -4,18 +4,23 @@ extends =
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
   ../../component/rsync/buildout.cfg
+  ../../component/smartmontools/buildout.cfg
+  ../../component/lmsensors/buildout.cfg
 
 parts = 
   slapos-cookbook
+  slapos-toolbox
   template
   eggs
   rsync
+  smartmontools
+  lmsensors
   
 [template]
 recipe = slapos.recipe.template 
-url = ${:_profile_base_location_}/instance.cfg
+url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-md5sum = b86685e4007296f4808c2f1f1b74a290
+md5sum = 184a27424b48514cfe9374e881e1cc0c 
 mode = 0644
 
 # stupify index for now
@@ -30,10 +35,34 @@ md5sum = 1f30b17f5ea11fd033984d17c38bcfd3
 [rsync-logs-script]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/rsync-logs.sh.in
-md5sum = 33900b1f30f9d1de19814d6d3bacb03b
+md5sum = 8828868268a6f9761eccbcec1660cc9a 
 output = ${buildout:directory}/template-rsync-logs.sh.in
 mode = 0644
 
+[smartmon-logs-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/smartmon.py.in
+md5sum = a3ea2604abf53067af1b952b278fba94 
+output = ${buildout:directory}/template-smartmon-logs.py.in
+mode = 0644
+
+[network-bench-cfg]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/network_bench.cfg
+md5sum  = 44f8db5fca5e16ae87f570623a3ad0b4 
+output = ${buildout:directory}/template-network-bench-cfg.in
+mode = 0644
+
+[slapos-toolbox]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  slapos.toolbox
+
+scripts =
+  networkbench
+  onetimedownload
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs = 
@@ -49,3 +78,26 @@ cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 slapos.recipe.template = 2.5
 z3c.recipe.scripts = 1.0.1
+
+apache-libcloud = 0.16.0
+ecdsa = 0.11
+gitdb = 0.6.0
+pycrypto = 2.6.1
+slapos.toolbox = 0.45.1
+smmap = 0.8.3
+
+# Required by:
+# slapos.toolbox==0.45.1
+GitPython = 0.3.2.1
+
+# Required by:
+# slapos.toolbox==0.45.1
+atomize = 0.2.0
+
+# Required by:
+# slapos.toolbox==0.45.1
+feedparser = 5.1.3
+
+# Required by:
+# slapos.toolbox==0.45.1
+paramiko = 1.15.1

software/slaprunner/common.cfg

@@ -46,7 +46,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = 8ebc0e59db33f0105968a813a3130236
+md5sum = 39458e6a1d8cddd0000d6d6a0517f017
 mode = 0644
 
 [template-runner-import-script]
@@ -82,7 +82,7 @@ mode = 0644
 [template-resilient]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = aec7fb7fe57b48c5c26c7adcdff36d26
+md5sum = 489cfdbe128c21ff5847f92d804fe460
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 
@@ -90,7 +90,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = 105c7c427def7b2643b7480b2bada3a5
+md5sum = b95b230bcdd7d4eaf9452f73156b27e8
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 

software/slaprunner/development.cfg

@@ -11,8 +11,8 @@ extends = common.cfg
 parts =
 # Development eggs
   slapos.toolbox-repository
-#  slapos.cookbook-repository
-#  slapos.core-repository
+  slapos.cookbook-repository
+  slapos.core-repository
   check-recipe
 #  erp5.util-repository
 # Good elements
@@ -27,11 +27,10 @@ parts =
   collective.recipe.template-egg
 
 
-
 develop =
   ${:parts-directory}/slapos.toolbox-repository
-#  ${:parts-directory}/slapos.cookbook-repository
-#  ${:parts-directory}/slapos.core-repository
+  ${:parts-directory}/slapos.cookbook-repository
+  ${:parts-directory}/slapos.core-repository
 #  ${:parts-directory}/erp5.util-repository
 
 
@@ -39,7 +38,6 @@ develop =
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
 branch = slaprunner-paas
-#revision = 2de7ba00a75e73cae2d6b71f7266f6261416b057
 git-executable = ${git:location}/bin/git
 
 [slapos.cookbook-repository]
@@ -52,8 +50,7 @@ git-executable = ${git:location}/bin/git
 [erp5.util-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/erp5.git
-#branch = scalability-master2
-revision = f9a812f4e97ffeed359bc5611366a5c8675ec749
+branch = master
 git-executable = ${git:location}/bin/git
 
 [slapos.core-repository]
@@ -67,8 +64,12 @@ recipe = plone.recipe.command
 stop-on-error = true
 update-command = ${:command}
 command =
-  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
-#  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
-#  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
+  grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link
 #  grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
 
+[versions]
+slapos.cookbook =
+slapos.core =
+slapos.toolbox =

software/slaprunner/instance-for-erp5testnode.cfg.in

+[buildout]
+extends = ${template:output}
+
+[switch_softwaretype]
+default = $${:test}

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -5,8 +5,9 @@ develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 parts =
-  deploy-resiliency-test
   request-resilient-instance
+  deploy-unit-test
+  deploy-scalability-test
   deploy-standalone-resiliency-test
 
 [directory]
@@ -21,18 +22,27 @@ log = ${:var}/log/
 services = ${:etc}/service/
 scripts = ${:etc}/run/
 
-[deploy-resiliency-test]
+[deploy-scalability-test]
 recipe = slapos.cookbook:wrapper
 testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=runner slaprunner_rootinstance_name='${request-resilient-instance:name}'
-command-line = {{ bin_directory }}/runResiliencyTest ${:testnode-parameters} ${:test-parameters}
+test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=runner root_instance_name='${request-resilient-instance:name}'
+command-line = {{ bin_directory }}/runResiliencyScalabilityTestNode ${:testnode-parameters} ${:test-parameters}
 wrapper-path = ${directory:scripts}/runResiliencyTestSuite
 
+[deploy-unit-test]
+recipe = collective.recipe.template
+test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=runner root_instance_name='${request-resilient-instance:name}'
+input = inline:
+  #!/bin/sh
+  exec {{ bin_directory }}/runResiliencyUnitTestTestNode $@ ${:test-parameters}
+output = ${directory:bin}/runTestSuite
+mode = 755
+
 [deploy-standalone-resiliency-test]
 # Used to manually run the resilient test if we don't have a running testnode.
 recipe = slapos.cookbook:wrapper
 test-suite-title = slaprunner
-command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=${:test-suite-title} ${deploy-resiliency-test:test-parameters}
+command-line = {{ bin_directory }}/runStandaloneResiliencyTestSuite --test-suite-title=${:test-suite-title} ${deploy-scalability-test:test-parameters}
 wrapper-path = ${directory:bin}/runStandaloneResiliencyTestSuite
 
 [request-resilient-instance]
@@ -45,7 +55,8 @@ name = Resilient Instance (Root Instance)
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-resiliency-backup-periodicity = */6 * * * *
-config-resilient-clone-number = 2
+config-resilient-clone-number = 1
+config-ignore-known-hosts-file = true
 # XXX hardcoded
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance

software/slaprunner/instance-resilient.cfg.jinja2

@@ -28,7 +28,7 @@ parts +=
 
 # Bubble up the parameters
 [request-runner]
-return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url webdav_url public_url git_public_url git_private_url
+return = url ssh-public-key ssh-url notification-id ip backend_url url ssh_command access_url 1_info 2_info monitor_url monitor_backend_url webdav_url public_url git_public_url git_private_url
 
 [publish-connection-informations]
 recipe = slapos.cookbook:publish
@@ -39,6 +39,7 @@ access_url = ${request-runner:connection-access_url}
 url = ${request-runner:connection-url}
 ssh_command = ${request-runner:connection-ssh_command}
 monitor_url = ${request-runner:connection-monitor_url}
+monitor_backend_url = ${request-runner:connection-monitor_backend_url}
 webdav_url = ${request-runner:connection-webdav_url}
 public_url = ${request-runner:connection-public_url}
 git_public_url = ${request-runner:connection-git_public_url}

software/slaprunner/instance-runner.cfg

@@ -355,7 +355,7 @@ path_pid = $${directory:run}/gunicorn.pid
 
 [gunicorn-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = $${gunicorn:bin_gunicorn} slapos.runner:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload 
+command-line = $${gunicorn:bin_gunicorn} slapos.runner.run:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --error-logfile $${directory:log}/$${:error-log-file} --log-level error --preload
 error-log-file = gunicorn-error.log
 wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${environ:PATH}:${git:location}/bin/
@@ -446,6 +446,7 @@ access_url = $${:url}/login
 url =  https://$${request-frontend:connection-domain}
 ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
 monitor_url = https://$${monitor-frontend:connection-domain}
+monitor_backend_url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port}
 webdav_url = $${:monitor_url}/share/
 public_url =  $${:monitor_url}/public/
 git_public_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:monitor_port}/git-public/

software/slaprunner/software-for-erp5testnode.cfg

+[buildout]
+extends = development.cfg
+
+parts += template-erp5testnode
+
+# Change default software-type to be "test", so that it can be run using erp5testnode.
+[template-erp5testnode]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-for-erp5testnode.cfg.in
+md5sum = 59a13145d3df30f38442ce172330dfb3
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[template]
+output = ${buildout:directory}/template-original.cfg

software/slaprunner/software.cfg

@@ -9,39 +9,37 @@ extends = common.cfg
 [versions]
 Flask-Auth = 0.85
 PyRSS2Gen = 1.1
-apache-libcloud = 0.15.1
-async = 0.6.1
+apache-libcloud = 0.16.0
 cns.recipe.symlink = 0.2.3
 collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.11
 ecdsa = 0.11
-erp5.util = 0.4.41
-gitdb = 0.5.4
+erp5.util = 0.4.42
+gitdb = 0.6.0
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-slapos.recipe.build = 0.12
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.4
-smmap = 0.8.2
+slapos.toolbox = 0.45.2
+smmap = 0.8.3
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.4
-GitPython = 0.3.2.RC1
+# slapos.toolbox==0.45.2
+GitPython = 0.3.2.1
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.45.2
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.45.2
 feedparser = 5.1.3
 
 # Required by:
-# slapos.cookbook==0.87
+# slapos.cookbook==0.92
 jsonschema = 2.4.0
 
 # Required by:
-# slapos.toolbox==0.40.4
+# slapos.toolbox==0.45.2
 paramiko = 1.15.1

software/varnish/software.cfg

@@ -45,6 +45,7 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended

stack/erp5/buildout.cfg

@@ -144,6 +144,7 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 
 [template-jinja2-base]
 recipe = slapos.recipe.template:jinja2
+mode = 640
 template = ${:_profile_base_location_}/${:filename}.in
 rendered = ${buildout:directory}/${:filename}
 # XXX: extra-context is needed because we cannot append to a key of an extended
@@ -651,7 +652,7 @@ Pympler = 0.3.1
 StructuredText = 2.11.1
 WSGIUtils = 0.7
 apache-libcloud = 0.16.0
-astroid = 1.2.1
+astroid = 1.3.2
 async = 0.6.1
 chardet = 2.3.0
 csp-eventlet = 0.7.0
@@ -679,7 +680,7 @@ polib = 1.0.5
 pprofile = 1.7.2
 pycountry = 1.10
 pyflakes = 0.8.1
-pylint = 1.3.1
+pylint = 1.4.0
 python-ldap = 2.4.17
 python-magic = 0.4.6
 python-memcached = 1.53
@@ -687,7 +688,7 @@ qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 slapos.recipe.template = 2.5
-slapos.toolbox = 0.43.0
+slapos.toolbox = 0.45.1
 smmap = 0.8.3
 socketpool = 0.5.3
 spyne = 2.11.0
@@ -701,7 +702,7 @@ xupdate-processor = 0.4
 xfw = 0.10
 
 # Required by:
-# slapos.toolbox==0.43.0
+# slapos.toolbox==0.45.1
 GitPython = 0.3.2.1
 
 # Required by:
@@ -709,7 +710,7 @@ GitPython = 0.3.2.1
 Products.ZSQLMethods = 2.13.4
 
 # Required by:
-# slapos.toolbox==0.43.0
+# slapos.toolbox==0.45.1
 atomize = 0.2.0
 
 # Required by:
@@ -717,7 +718,7 @@ atomize = 0.2.0
 erp5.util = 0.4.41
 
 # Required by:
-# slapos.toolbox==0.43.0
+# slapos.toolbox==0.45.1
 feedparser = 5.1.3
 
 # Required by:
@@ -746,7 +747,7 @@ minitage.recipe.common = 1.90
 ordereddict = 1.1
 
 # Required by:
-# slapos.toolbox==0.43.0
+# slapos.toolbox==0.45.1
 paramiko = 1.15.1
 
 # Required by:

stack/resilient/buildout.cfg

@@ -63,13 +63,13 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 08b80d1b7beb1610077151904526a2ec
+md5sum = 3866b0d4d2872f693b7d9519a668e6bc
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = a21d751078dbf1fc80ecb93a6f499287
+md5sum = c781ae17375c26b08b2a11bd9b800db8
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 

stack/resilient/instance-pull-backup.cfg.in

@@ -136,6 +136,7 @@ run-directory = $${basedirectory:run}
 # XXX: this should be named "notifier-host"
 notifier-url = http://[$${notifier:host}]:$${notifier:port}
 slave-instance-list = $${slap-parameter:slave_instance_list}
+ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
 
 
 #----------------
@@ -235,6 +236,7 @@ identity-file = $${basedirectory:ssh-home}/id_rsa
 
 [slap-parameter]
 slave_instance_list = []
+ignore-known-hosts-file = false
 
 
 

stack/resilient/template-replicated.cfg.in

@@ -154,6 +154,7 @@ software-type = pull-backup
 [request-pbs-{{namebase}}-{{id}}]
 <= request-pbs-common
 name = PBS ({{namebase}} / {{id}})
+config-ignore-known-hosts-file = ${slap-parameter:ignore-known-hosts-file}
 return = ssh-key notification-url feeds-url
 slave = false
 {% if sla_parameter_dict == {} -%}
@@ -231,6 +232,7 @@ sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
 {{namebase}}{{id}}-computer-guid =
 pbs-{{namebase}}{{id}}-computer-guid =
 {% endfor %}
+ignore-known-hosts-file = false
 
 {% endmacro %}
 

stack/slapos.cfg

@@ -114,7 +114,7 @@ cffi = 0.8.6
 cmd2 = 0.6.7
 cryptography = 0.6
 itsdangerous = 0.24
-lxml = 3.4.0
+lxml = 3.4.1
 meld3 = 1.0.0
 mr.developer = 1.31
 prettytable = 0.7.2
@@ -124,14 +124,14 @@ setuptools = 7.0
 six = 1.8.0
 simplejson = 3.6.5
 slapos.cookbook = 0.87
-slapos.core = 1.3.3
+slapos.core = 1.3.5
 slapos.libnetworkcache = 0.14.2
-slapos.recipe.build = 0.14
+slapos.recipe.build = 0.15
 slapos.recipe.cmmi = 0.2
 stevedore = 1.1.0
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 Flask = 0.10.1
 
 # Required by:
@@ -140,7 +140,7 @@ Flask = 0.10.1
 argparse = 1.2.2
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 cliff = 1.8.0
 
 # Required by:
@@ -156,7 +156,7 @@ lock-file = 2.0
 netaddr = 0.7.12
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 netifaces = 0.10.4
 
 # Required by:
@@ -169,7 +169,7 @@ pbr = 0.10.0
 pip = 1.5.6
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 psutil = 2.1.3
 
 # Required by:
@@ -178,18 +178,18 @@ pycparser = 2.10
 
 # Required by:
 # slapos.cookbook==0.87
-pytz = 2014.9
+pytz = 2014.10
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 requests = 2.4.3
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 supervisor = 3.1.3
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 uritemplate = 0.6
 
 # Required by:
@@ -197,7 +197,7 @@ uritemplate = 0.6
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.core==1.3.3
+# slapos.core==1.3.5
 zope.interface = 4.1.1
 
 [networkcache]