Merge branch 'master' into erp5-cluster

Conflicts:
	stack/erp5/buildout.cfg
	stack/erp5/instance-kumofs.cfg.in
	stack/erp5/instance-mariadb.cfg.in
	stack/erp5/instance-memcached.cfg.in
	stack/erp5/instance-tidstorage.cfg.in
	stack/erp5/instance-varnish.cfg.in

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2
-md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd
+url = http://curl.haxx.se/download/curl-7.40.0.tar.bz2
+md5sum = 8d30594212e65657a5c32030f0998fa9
 configure-options =
   --disable-static
   --disable-ldap

component/file/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.21.tar.gz
-md5sum = 549fe96e09041eabece9de2bb28ef923
+url = ftp://ftp.astron.com/pub/file/file-5.22.tar.gz
+md5sum = 8fb13e5259fe447e02c4a37bc7225add
 configure-options =
   --disable-static
 environment =

component/groonga/buildout.cfg

@@ -12,8 +12,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/groonga/groonga-4.0.9.tar.gz
-md5sum = ab6c15de7764ba5028b5037cc22368e8
+url = http://packages.groonga.org/source/groonga/groonga-4.1.0.tar.gz
+md5sum = 93c763da6c298595da4b6964d6b80769
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/lunzip/buildout.cfg

@@ -4,7 +4,7 @@ parts =
 
 [lunzip]
 recipe = slapos.recipe.cmmi
-url = http://download.savannah.gnu.org/releases/lunzip/lunzip-1.6.tar.gz
+url = http://download.savannah.gnu.org/releases/lzip/lunzip/lunzip-1.6.tar.gz
 md5sum = 5e6ad4fe91f235be64227bc9930986fe
 configure-options =
   --disable-static

component/mesa/buildout.cfg

@@ -39,6 +39,7 @@ environment =
   AUTOCONF=${autoconf:location}/bin/autoconf
   AUTOMAKE=${automake:location}/bin/automake
 make-options =
+  -j1
   LIBTOOL=${libtool:location}/bin/libtool
 
 [glu]

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.92'
+version = '0.93'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/equeue.py

@@ -32,7 +32,8 @@ class Recipe(GenericBaseRecipe):
 
     parameters = [
       '--database', self.options['database'],
-      '-l', self.options['log'],
+      '--logfile', self.options['log'],
+      '--lockfile', self.options['lockfile']
     ]
 
     if 'loglevel' in self.options:

slapos/recipe/generic_mysql/__init__.py

@@ -53,6 +53,15 @@ class Recipe(GenericBaseRecipe):
     else:
       networking = 'skip-networking'
 
+    log_bin = self.options.get('binlog-path', '')
+    if log_bin:
+      log_bin = 'log_bin = %s' % log_bin
+    expire_logs_days = self.options.get('binlog-expire-days')
+    if expire_logs_days > 0:
+      expire_logs_days = 'expire_logs_days = %s' % expire_logs_days
+    else:
+      expire_logs_days = ''
+
     mysql_conf_file = self.createFile(
       self.options['conf-file'],
       self.substituteTemplate(template_filename, {
@@ -62,6 +71,8 @@ class Recipe(GenericBaseRecipe):
         'socket': self.options['socket'],
         'error_log': self.options['error-log'],
         'slow_query_log': self.options['slow-query-log'],
+        'log_bin': log_bin,
+        'expire_logs_days': expire_logs_days,
       })
     )
     path_list.append(mysql_conf_file)

slapos/recipe/generic_mysql/template/my.cnf.in

@@ -48,9 +48,8 @@ innodb_locks_unsafe_for_binlog = 1
 #innodb_doublewrite = 0
 #sync_frm = 0
 
-# Uncomment the following if you need binary logging, which is recommended
-# on production instances (either for replication or incremental backups).
-#log-bin=mysql-bin
+%(log_bin)s
+%(expire_logs_days)s
 
 # Force utf8 usage
 collation_server = utf8_unicode_ci

software/erp5testnode/software.cfg

@@ -60,9 +60,4 @@ md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.42
-slapos.cookbook = 0.92
 slapos.recipe.template = 2.6
-
-# Required by:
-# slapos.cookbook==0.92
-jsonschema = 2.4.0

software/helloworld/instance.cfg.in

@@ -65,7 +65,7 @@ log = $${:var}/log
 # This recipe will try to "exec" the command-line after separating parameters.
 recipe = slapos.cookbook:wrapper
 # Notice that there is only one $ at ${dash:location}, it is because it comes from the Software Release buildout profile.
-command-line = ${dash:location}/bin/dash -c 'echo "Hello $${instance-parameter:configuration.name}, it is $(date). If I were a TCP server, I would be listening on [$${instance-parameter:ipv6-random}] or $${instance-parameter:ipv4-random}" > $${directory:log}/log.log; sleep 1000000;'
+command-line = ${dash:location}/bin/dash -c 'echo "Hello $${instance-parameter:configuration.name}, it is $(date). If I were a server application, you would reach me at $${instance-parameter:global-ipv6}" > $${directory:log}/log.log; sleep 1000000;'
 # Put this shell script in the "etc/service" directory. Each executable of this
 # repository will be started and monitored by supervisord. If a service
 # exits/crashes, it will trigger a "bang" and cause a re-run of the instance.

software/helloworld/software.cfg

 [buildout]
 
-extends = 
+extends =
 # "slapos" stack describes basic things needed for 99.9% of SlapOS Software
 # Releases
   ../../stack/slapos.cfg
@@ -26,5 +26,5 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
 # MD5 checksum can be skipped for development (easier to develop), but must be filled for production
-md5sum = 532d13c3aee4f0252181da39765317f0
+md5sum = 1fea8a474f3b2eb7847685659441a3f9
 mode = 0644

software/kvm/software.cfg

@@ -34,10 +34,6 @@ atomize = 0.2.0
 # slapos.toolbox==0.40.4
 feedparser = 5.1.3
 
-# Required by:
-# slapos.cookbook==0.87
-jsonschema = 2.4.0
-
 # Required by:
 # websockify==0.6.0
 numpy = 1.9.0

software/slaprunner/common.cfg

@@ -45,14 +45,14 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = 39458e6a1d8cddd0000d6d6a0517f017
+md5sum = 3cac749e81d8b94dbbf1f1833bc031ce
 mode = 0644
 
 [template-runner-import-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
 download-only = true
-md5sum = c0d05a26b06ce172efaad03c52ef92ca
+md5sum = 4134ea7e191d0c3b552d2efbae6b5894
 filename = runner-import.sh.jinja2
 mode = 0644
 
@@ -60,14 +60,14 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-import.cfg.in
 output = ${buildout:directory}/instance-runner-import.cfg
-md5sum = a85d054b3e2ae9243d8d188c897dc121
+md5sum = e9e3692ce4af4d603666c969ec1964d2
 mode = 0644
 
 [template-runner-export-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = 0f290b46c0b89ff84aee5c10477e07ca
+md5sum = 41c0213f5cc07ecbe5c2852ef1844ac9
 filename = runner-export.sh.jinja2
 mode = 0644
 
@@ -75,7 +75,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner-export.cfg.in
 output = ${buildout:directory}/instance-runner-export.cfg
-md5sum = 521bad4c571b5b2dc3eee6090802de95
+md5sum = aaf86d03409018b5d3dce2dea27287cf
 mode = 0644
 
 [template-resilient]
@@ -171,6 +171,15 @@ destination = ${buildout:parts-directory}/monitor-template-cors-domain-cgi
 filename = cors-domain.jinja
 mode = 0644
 
+[monitor-check-webrunner-internal-instance]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/${:filename}
+download-only = true
+#md5sum = 4c44d617d5bfd1db8695200e896480c0
+destination = ${buildout:parts-directory}/${:filename}
+filename = monitor-check-webrunner-internal-instances.py
+mode = 0644
+
 [eggs]
 recipe = z3c.recipe.scripts
 eggs =

software/slaprunner/instance-runner-export.cfg.in

@@ -16,13 +16,14 @@ parts +=
   publish-connection-informations
   slaprunner-promise
   slaprunner-frontend-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-backup
   cron-entry-prepare-software
   deploy-instance-parameters
@@ -50,6 +51,7 @@ parts +=
 ## Monitor for runner
   monitor-current-log-access
   monitor-check-resilient-feed-file
+  monitor-check-webrunner-internal-instance
 
 
 [exporter]

software/slaprunner/instance-runner-import.cfg.in

@@ -14,18 +14,20 @@ parts +=
   dropbear-server-add-authorized-key
   sshkeys-authority
   slaprunner-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   shellinabox
   symlinks
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software-type
   bash-profile
   supervisord-wrapper
+  importer-consistency-promise
 # have to repeat the next one, as it's not inherited from pbsready-import
   import-on-notification
 ## Monitoring part
@@ -52,6 +54,14 @@ parts +=
 ## Monitor for import runner
   monitor-latest-restored-backup
 
+# For the needs of importer, we run the full slaprunner
+# In case both exporter and importer (aka main instance and clone instance)
+# run with the same IP (usually for testing purposes),
+# run slaprunner using different ports.
+[slaprunner]
+proxy_port = 50000
+runner_port = 50005
+
 [importer]
 recipe = slapos.recipe.template:jinja2
 template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
@@ -59,12 +69,30 @@ rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
 # backward compatibility for resilient stack
 wrapper = $${:rendered}
 mode = 700
+restore-exit-code-file=$${directory:srv}/importer-exit-code-file
 context =
   key backend_url slaprunner:access-url
   section directory directory
   raw  shell_binary ${dash:location}/bin/dash
   raw  rsync_binary ${rsync:location}/bin/rsync
   raw  curl_binary ${curl:location}/bin/curl
+  raw  restore_exit_code_file $${:restore-exit-code-file}
+
+[importer-consistency-promise]
+# Test that the importer script and "after-import" subscripts:
+# 1/ Have been run in the last 24 hours
+# 2/ Have succeeded
+recipe = collective.recipe.template
+input = inline: #!/bin/sh
+  EXIT_CODE_FILE=$(find "$${importer:restore-exit-code-file}" -mtime -1)
+  if [ -z "$EXIT_CODE_FILE" ]; then
+     echo "Consistency check is too old."
+     exit 1
+   fi
+  EXIT_CODE=$(cat $EXIT_CODE_FILE)
+  exit $EXIT_CODE
+output = $${directory:promises}/importer-consistency-promise
+mode = 755
 
 [slap-parameter]
 auto-deploy-instance = false

software/slaprunner/instance-runner.cfg

@@ -13,15 +13,17 @@ parts =
   publish-connection-informations
   slaprunner-promise
   slaprunner-frontend-promise
+  slaprunner-supervisord-wrapper
   dropbear-promise
   runtestsuite
   shellinabox-promise
   symlinks
   shellinabox
   slapos-cfg
-  slapos-repo-config
+  slapos-repo
   cron-entry-prepare-software
   deploy-instance-parameters
+  instance-software
   instance-software-type
   minishell-cwd
   bash-profile
@@ -51,6 +53,7 @@ parts =
 ## Monitor for runner
   monitor-current-log-access
   monitor-deploy-cors-domain-cgi
+  monitor-check-webrunner-internal-instance
 
 extends = ${monitor-template:output}
 
@@ -105,11 +108,12 @@ custom-frontend-url = https://$${request-custom-frontend:connection-domain}
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
+home = $${buildout:directory}
+etc = $${:home}/etc/
+var = $${:home}/var/
+srv = $${:home}/srv/
+bin = $${:home}/bin/
+tmp = $${:home}/tmp/
 
 sshkeys = $${:srv}/sshkeys
 services = $${:etc}/service/
@@ -120,7 +124,7 @@ run = $${:var}/run/
 backup = $${:srv}/backup/
 promises = $${:etc}/promise/
 test = $${:etc}/test/
-nginx-data = $${directory:srv}/nginx
+nginx-data = $${:srv}/nginx
 ca-dir = $${:srv}/ssl
 project = $${:srv}/runner/project
 
@@ -189,6 +193,17 @@ software_info_json = $${runnerdirectory:home}/software_info.json
 instance_info_json = $${runnerdirectory:home}/instance_info.json
 path = $${shell:path}
 
+
+#---------------------------
+#--
+#-- supervisord managing slaprunner instance processes
+[slaprunner-supervisord-wrapper]
+recipe = slapos.cookbook:wrapper
+# XXX hardcoded locations
+command-line = $${buildout:directory}/bin/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
+wrapper-path = $${directory:services}/slaprunner-supervisord
+
+
 [test-runner]
 <= slaprunner
 slapos.cfg = $${directory:etc}/slapos-test.cfg
@@ -262,7 +277,6 @@ scgi_temp_path = $${directory:tmp}/scgi_temp_path
 nb_workers = 2
 # Network
 local-ip = $${slap-network-information:local-ipv4}
-port = 30001
 global-ip = $${slap-network-information:global-ipv6}
 global-port = $${slaprunner:runner_port}
 # Backend
@@ -558,12 +572,7 @@ repository = $${slap-parameter:slapos-repository}
 git-executable = ${git:location}/bin/git
 develop = true
 location = $${directory:project}/slapos
-
-[slapos-repo-config]
-recipe = plone.recipe.command
-stop-on-error = true
-command = cd $${slapos-repo:location} && ${git:location}/bin/git checkout $${slap-parameter:slapos-reference} && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
-update-command = true
+branch = $${slap-parameter:slapos-reference}
 
 [prepare-software]
 recipe = slapos.cookbook:wrapper
@@ -599,6 +608,11 @@ stop-on-error = true
 software-type-path = $${directory:etc}/.software_type.xml
 command = if [ ! -f $${:software-type-path} -a "$${slap-parameter:slapos-software-type}" != "" ]; then echo "$${slap-parameter:slapos-software-type}" > $${:software-type-path}; fi
 
+[instance-software]
+recipe = plone.recipe.command
+stop-on-error = true
+command = SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
@@ -628,6 +642,11 @@ context =
     raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin
     key workdir runnerdirectory:home
 
+
+#---------------------------
+#--
+#-- supervisord managing slaprunner automation features
+
 [supervisord]
 autorestart = false
 autostart = false
@@ -696,6 +715,7 @@ path = $${directory:promises}/supervisord
 hostname = $${slaprunner:ipv4}
 port = $${supervisord:port}
 
+
 [monitor-current-log-access]
 < = monitor-directory-access
 source = $${directory:log}
@@ -715,6 +735,13 @@ context =
   key this_file :filename
   key httpd_graceful cgi-httpd-graceful-wrapper:rendered
 
+[monitor-check-webrunner-internal-instance]
+recipe = slapos.recipe.template:jinja2
+template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
+rendered = $${monitor-directory:monitor-custom-scripts}/$${:filename}
+filename = monitor-check-webrunner-internal-instance.py
+mode = 0744
+
 [monitor-httpd-cors]
 recipe = plone.recipe.command
 command = if [ ! -f $${:location} ]; then touch $${:location}; fi

software/slaprunner/software.cfg

@@ -15,31 +15,32 @@ collective.recipe.environment = 0.2.0
 collective.recipe.template = 1.11
 ecdsa = 0.11
 erp5.util = 0.4.42
-gitdb = 0.6.0
+gitdb = 0.6.4
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.45.3
-smmap = 0.8.3
+slapos.recipe.template = 2.6
+slapos.toolbox = 0.46.0
+smmap = 0.9.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.45.2
-GitPython = 0.3.2.1
+# slapos.toolbox==0.46.0
+GitPython = 0.3.5
 
 # Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.46.0
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.46.0
 feedparser = 5.1.3
 
 # Required by:
-# slapos.cookbook==0.92
-jsonschema = 2.4.0
+# slapos.toolbox==0.46.0
+lockfile = 0.10.2
 
 # Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.45.3
 paramiko = 1.15.2

software/slaprunner/template/monitor-check-webrunner-internal-instances.py

+#!/usr/bin/python
+
+import os
+import subprocess
+import sys
+
+def runPromise(promise_path):
+  promise_relative_path = promise_path.replace(os.path.expanduser('~'), '~')
+  print 'Running promise %s...' % promise_relative_path
+  promise_process = subprocess.Popen(promise_path, stderr=subprocess.PIPE)
+  stdout, stderr = promise_process.communicate()
+  return_code = promise_process.returncode
+  if return_code == 0:
+    print 'Success.'
+    return True
+  else:
+    sys.stderr.write('Failure while running promise %s. %s\n' % (promise_relative_path, stderr))
+
+def getPromisePathListFromPartitionPath(partition_path):
+  promise_directory_path = os.path.join(partition_path, 'etc/promise')
+  try:
+    promise_name_list = os.listdir(promise_directory_path)
+    return [os.path.join(promise_directory_path, promise_name) for promise_name in promise_name_list]
+  except OSError:
+    return []
+
+def main():
+  # XXX hardcoded
+  partition_root_path = os.path.expanduser('~/srv/runner/instance')
+  success = True
+  for partition_name in os.listdir(partition_root_path):
+    partition_path = os.path.join(partition_root_path, partition_name)
+    for promise_path in getPromisePathListFromPartitionPath(partition_path):
+      result = runPromise(promise_path)
+      if not result:
+        success = False
+  if not success:
+    sys.exit(1)
+
+if __name__ == '__main__':
+  main()
+

software/slaprunner/template/runner-export.sh.jinja2

@@ -2,20 +2,44 @@
 LC_ALL=C
 export LC_ALL
 umask 077
+
+srv_directory={{ directory['srv'] }}
+
 sync_element () {
   path=$1
   backup_path=$2
   shift 2
   element_list=$*
+
+  # Concatenate the exclude file of each partition of webrunner
+  # to create a global exclude file.
+  for partition in $srv_directory/runner/instance/slappart*
+  do
+    exclude_file="$partition/srv/exporter.exclude"
+    if [ -e "$exclude_file" ]; then
+      partition_exclude_content_relative=$(cat "$exclude_file")
+      # For every line of the local exclude file, add the absolute path
+      for line in "$partition_exclude_content_relative"
+      do
+        if [ ! -z "$line" ]; then
+          exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
+       fi
+      done
+    fi
+  done
+  echo "$exclude_content" > $srv_directory/exporter.exclude
+
   for element in $element_list
   do
+    echo "Changing current directory to $path."
     cd $path;
     if [ -f $element ] || [ -d $element ]; then
-       {{ rsync_binary }} -rlptgov --safe-links --delete $element  $backup_path;
+       echo "Running {{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path"
+       {{ rsync_binary }} -rlptgov --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element  $backup_path;
     fi
   done
 }
-sync_element {{ directory['srv'] }}/runner  {{ directory['backup'] }}/runner/ instance project  proxy.db
+sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
 # We sync .* appart
 date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
 cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/

software/slaprunner/template/runner-import.sh.jinja2

 #!{{ shell_binary }}
+set -e
 LC_ALL=C
 export LC_ALL
 umask 077
+srv_directory={{ directory['srv'] }}
 restore_element () {
   backup_path=$1
   restore_path=$2
@@ -18,14 +20,66 @@ restore_element () {
 
 write_backup_proof () {
   cd {{ directory['backup'] }}
-  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
-  diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
+  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > $srv_directory/proof.signature
+  diff -ruw {{ directory['backup'] }} $srv_directory/proof.signature > $srv_directory/backup.diff || true # diff exits with code 1 when files are different
 }
 
-# For now we just make the diff before 
+# For now we just make the diff before
 write_backup_proof
 
-restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner  instance project  proxy.db
+restore_element {{ directory['backup'] }}/runner/ $srv_directory/runner  instance project  proxy.db
 restore_element  {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
 cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
-{{ curl_binary }} --insecure -vg6L --max-time 5  {{ backend_url }}/isSRReady;
+{{ curl_binary }} --insecure -g6L --max-time 5  {{ backend_url }}/isSRReady;
+
+
+# Invoke arbitrary script to perform specific restoration
+# procedure.
+RESTORE_EXIT_CODE=0
+runner_import_restore=$srv_directory/runner-import-restore
+if [ ! -e "$runner_import_restore" ]; then
+  touch $runner_import_restore
+  chmod +x $runner_import_restore
+fi
+echo "Running $runner_import_restore script..."
+$srv_directory/runner-import-restore || RESTORE_EXIT_CODE=$?
+
+echo "Updating slapproxy database, software release and instances..."
+HOME="{{ directory['home'] }}"
+SLAPOS="{{ directory['bin'] }}/slapos"
+# XXX hardcoded
+SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
+DATABASE="$HOME/srv/runner/proxy.db"
+# Change slapproxy database to point instances to new software release
+# XXX hardcoded
+PARTITION=$(basename $HOME)
+OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
+SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/slappart[0-9][0-9]/'"$PARTITION"'/')
+$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
+# Change slapproxy database to have all instances stopped
+$SQLITE3 $DATABASE "update partition11 set slap_state='stopped';"
+# Build newest software
+$SLAPOS node software --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-software.pid --all >/dev/null 2>&1 || true
+# Remove defined scripts to force buildout to recreate them to have updated paths
+rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore
+# Run slapos node instance
+# XXX hardcoded
+$SLAPOS node instance --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
+
+# Invoke defined scripts for each partition inside of slaprunner
+for partition in $srv_directory/runner/instance/slappart*/
+do
+  script=$partition/srv/runner-import-restore
+  if [ -e "$script" ]; then
+    echo "Running $script script..."
+    $script || RESTORE_EXIT_CODE=$?
+  fi
+done
+
+# Change back slapproxy database to have all instances started
+$SQLITE3 $DATABASE "update partition11 set slap_state='started';"
+
+# Write exit code to an arbitrary file that will be checked by promise/monitor
+RESTORE_EXIT_CODE_FILE="{{ restore_exit_code_file }}"
+echo $RESTORE_EXIT_CODE > $RESTORE_EXIT_CODE_FILE
+exit $RESTORE_EXIT_CODE

stack/erp5/buildout.cfg

@@ -16,6 +16,7 @@ extends =
   ../../component/gzip/buildout.cfg
   ../../component/haproxy/buildout.cfg
   ../../component/hookbox/buildout.cfg
+  ../../component/findutils/buildout.cfg
   ../../component/librsvg/buildout.cfg
   ../../component/imagemagick/buildout.cfg
   ../../component/inkscape/buildout.cfg
@@ -97,7 +98,6 @@ parts =
   userhosts
 
 # Buildoutish
-  patched-eggs
   eggs
   testrunner
   test_suite_runner
@@ -157,10 +157,16 @@ context =
     key slapos_core_version versions:slapos.core
     ${:extra-context}
 
+[mariadb-resiliency-after-import-script]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
+md5sum = 8db483ef7c3da79a1cb5ea07ba79a0ed
+mode = 755
+
 [template-mariadb]
 < = download-base
 filename = instance-mariadb.cfg.in
-md5sum = 8a59e1730391fb3a859cd973cd7c3a5b
+md5sum = fc4a7585fe1bfea0c198b1826f904595
 link-binary =
   ${coreutils:location}/bin/basename
   ${coreutils:location}/bin/cat
@@ -176,12 +182,12 @@ link-binary =
 [template-kumofs]
 < = download-base
 filename = instance-kumofs.cfg.in
-md5sum = 5ebf310655d5de27da039d71a63d209b
+md5sum = 00e29ccfdd2b482f7d0db35a85af2877
 
 [template-cloudooo]
 < = download-base
 filename = instance-cloudoo.cfg.in
-md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64
+md5sum = 050fa6ff4eb397c5d4cb41a9f75afb3f
 
 [template-zope-conf]
 < = download-base
@@ -260,6 +266,7 @@ extra-context =
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
     key mariadb_location mariadb:location
+    key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key mesa_location mesa:location
     key openssl_location openssl:location
     key sixtunnel_location 6tunnel:location
@@ -428,22 +435,6 @@ initialization =
   repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
   sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
 
-[patched-eggs]
-recipe = minitage.recipe.egg
-eggs =
-  Acquisition
-  Products.DCWorkflow
-  ZODB3
-Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
-Acquisition-patch-options = -p1
-Acquisition-patch-binary = ${patch:location}/bin/patch
-Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
-Products.DCWorkflow-patch-options = -p1
-Products.DCWorkflow-patch-binary = ${patch:location}/bin/patch
-ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
-ZODB3-patch-options = -p1
-ZODB3-patch-binary = ${patch:location}/bin/patch
-
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -459,6 +450,7 @@ eggs =
   Pympler
   SOAPpy
   chardet
+  collective.recipe.template
   coverage
   elementtree
   erp5diff
@@ -466,6 +458,7 @@ eggs =
   interval
   ipdb
   Jinja2
+  jsonschema
   mechanize
   paramiko
   ply
@@ -556,6 +549,15 @@ scripts =
 extra-paths =
   ${erp5:location}
 
+# patches for eggs
+patch-binary = ${patch:location}/bin/patch
+Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch
+Acquisition-patch-options = -p1
+Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch
+Products.DCWorkflow-patch-options = -p1
+ZODB3-patches = ${:_profile_base_location_}/../../component/egg-patch/ZODB3-3.10.5.patch
+ZODB3-patch-options = -p1
+
 [zodbanalyze]
 recipe = zc.recipe.egg
 eggs =
@@ -592,9 +594,9 @@ scripts =
 
 [versions]
 # patched eggs
-Acquisition = 2.13.8-ZMinitagePatched-AqDynamic
-Products.DCWorkflow = 2.2.4-ZMinitagePatched-WorkflowMethod
-ZODB3 = 3.10.5-ZMinitagePatched-ZODB33105
+Acquisition = 2.13.8+SlapOSPatched001
+Products.DCWorkflow = 2.2.4+SlapOSPatched001
+ZODB3 = 3.10.5+SlapOSPatched001
 
 # specify dev version to be sure that an old released version is not used
 cloudooo = 1.2.5-dev
@@ -654,16 +656,16 @@ StructuredText = 2.11.1
 WSGIUtils = 0.7
 apache-libcloud = 0.16.0
 astroid = 1.3.2
-async = 0.6.1
 chardet = 2.3.0
+collective.recipe.template = 1.11
 csp-eventlet = 0.7.0
 ecdsa = 0.11
-elementtree = 1.2.7-20070827-preview
+elementtree = 1.2.6.post20050316
 erp5diff = 0.8.1.5
-eventlet = 0.16.0
+eventlet = 0.16.1
 five.formlib = 1.0.4
 five.localsitemanager = 2.0.5
-gitdb = 0.6.1
+gitdb = 0.6.4
 greenlet = 0.4.5
 http-parser = 0.8.3
 httplib2 = 0.9
@@ -673,25 +675,23 @@ ipdb = 0.8
 ipython = 2.3.1
 jsonschema = 2.4.0
 logilab-common = 0.63.2
-minitage.paste = 1.4.6
-minitage.recipe.egg = 1.107
 numpy = 1.9.1
 plone.recipe.command = 1.1
 ply = 3.4
-polib = 1.0.5
+polib = 1.0.6
 pprofile = 1.7.2
 pycountry = 1.10
 pyflakes = 0.8.1
 pylint = 1.4.0
-python-ldap = 2.4.18
+python-ldap = 2.4.19
 python-magic = 0.4.6
 python-memcached = 1.53
 qrcode = 5.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 slapos.recipe.template = 2.6
-slapos.toolbox = 0.45.2
-smmap = 0.8.3
+slapos.toolbox = 0.45.3
+smmap = 0.9.0
 socketpool = 0.5.3
 spyne = 2.11.0
 suds = 0.4
@@ -704,52 +704,27 @@ xupdate-processor = 0.4
 xfw = 0.10
 
 # Required by:
-# slapos.toolbox==0.45.2
-GitPython = 0.3.3
+# slapos.toolbox==0.45.3
+GitPython = 0.3.5
 
 # Required by:
 # Products.CMFCore==2.2.8
 Products.ZSQLMethods = 2.13.4
 
 # Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.45.3
 atomize = 0.2.0
 
 # Required by:
-# cloudooo==1.2.5-dev
-erp5.util = 0.4.41
-
-# Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.45.3
 feedparser = 5.1.3
 
 # Required by:
-# SOAPpy==0.12.0nxd001
+# SOAPpy===0.12.0nxd001
 fpconst = 0.7.2
 
 # Required by:
-# minitage.recipe.egg==1.107
-iniparse = 0.4
-
-# Required by:
-# minitage.core==2.0.57
-minitage = 2.0.67
-
-# Required by:
-# minitage.recipe.common==1.90
-# minitage.recipe.egg==1.107
-minitage.core = 2.0.57
-
-# Required by:
-# minitage.recipe.egg==1.107
-minitage.recipe.common = 1.90
-
-# Required by:
-# minitage.recipe.egg==1.107
-ordereddict = 1.1
-
-# Required by:
-# slapos.toolbox==0.45.2
+# slapos.toolbox==0.45.3
 paramiko = 1.15.2
 
 # Required by:

stack/erp5/instance-cloudoo.cfg.in

@@ -4,6 +4,7 @@
 parts =
   publish-cloudooo-connection-information
   cloudooo-instance
+  resiliency-exclude-file
   promise
   promise-openoffice
   {% if use_ipv6 %}promise-tunnel{% endif %}
@@ -52,6 +53,12 @@ url-list = {{ dumps(slapparameter_dict.get('font-url-list', []) | join(' ')) }}
 service-folder = ${directory:service}
 onetimedownload_path = {{ bin_directory }}/onetimedownload
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${directory:srv}/exporter.exclude
+
 [promise]
 recipe = slapos.cookbook:check_port_listening
 path = ${directory:promise}/cloudooo

stack/erp5/instance-kumofs.cfg.in

@@ -5,6 +5,7 @@ parts +=
   publish-kumofs-connection-information
   kumofs-instance
   logrotate-entry-kumofs
+  resiliency-exclude-file
   promise-kumofs-server
   promise-kumofs-server-listen
   promise-kumofs-gateway
@@ -73,6 +74,12 @@ services = ${buildout:directory}/etc/run
 promise = ${buildout:directory}/etc/promise
 kumofs-data = ${buildout:directory}/srv/kumofs
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: **
+output = ${directory:srv}/exporter.exclude
+
 # Deploy zope promises scripts
 [promise-template]
 recipe = slapos.cookbook:check_port_listening

stack/erp5/instance-mariadb.cfg.in

@@ -211,6 +211,12 @@ var = ${buildout:directory}/var
 log = ${:var}/log
 run = ${:var}/run
 
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: srv/mariadb/**
+output = ${directory:srv}/exporter.exclude
+
 [promise]
 recipe = slapos.cookbook:wrapper
 command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
@@ -225,5 +231,6 @@ parts +=
   binary-link
   update-mysql
   mysqld
+  resiliency-exclude-file
   promise
   {{ part_list | join('\n  ') }}

stack/erp5/template/instance-mariadb-resiliency-after-import-script.sh.in

+#!/bin/sh
+
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# OR MYSQL DATA WILL BE ERASED.
+
+# This script will import the dump of the mysql database to the real
+# database. It is launched by the clone (importer) instance of webrunner
+# in the end of the import script.
+
+# Depending on the output, it will create a file containing
+# the status of the restoration (success or failure)
+
+mysql_executable="${mariadb-instance:mysql-binary}"
+mysqldump_executable="${binary-wrap-mysqldump:wrapper-path}"
+mariadb_data_directory="${directory:mariadb-data}"
+mariadb_backup_directory="${directory:mariadb-backup-full}"
+instance_directory="${buildout:directory}"
+pid_file="${mariadb-instance:pid-file}"
+binlog_path="${mariadb-instance:binlog-path}"
+
+# Make sure mariadb is not already running
+if [ -e "$pid_file" ]; then
+  pid=$(cat $pid_file) > /dev/null 2>&1
+  if kill -0 "$pid"; then
+    echo "Mariadb is already running with pid $pid. Aborting."
+    exit 1
+  fi
+fi
+
+echo "Deleting existing database..."
+rm -r $mariadb_data_directory/*
+
+echo "Adapting binlog database to new paths..."
+new_binlog_directory="$(dirname $binlog_path)"
+binlog_index_file="$new_binlog_directory/binlog.index"
+old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
+sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
+
+echo "Starting mariadb..."
+# XXX hardcoded
+$instance_directory/etc/run/mariadb &
+mysqld_pid=$!
+$instance_directory/etc/run/mariadb_update > /dev/null 2>&1
+
+echo "Importing data..."
+# Use latest dump XXX can contain funny characters
+dump=$(ls -r $mariadb_backup_directory | head -1)
+zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
+RESTORE_EXIT_CODE=$?
+
+kill "$mysqld_pid"
+
+if [ $RESTORE_EXIT_CODE -eq 0 ]; then
+  echo 'Backup restoration successfully completed.'
+else
+  echo 'Backup restoration failed.'
+fi
+
+exit $RESTORE_EXIT_CODE

stack/monitor/buildout.cfg

@@ -119,7 +119,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/${:filename}
 download-only = true
-md5sum = 5f1b93ccdea7c3031aef396154c64938
+md5sum = 6c84a826778cb059754623f39b33651b
 destination = ${buildout:parts-directory}/monitor-template-rss-bin
 filename = status2rss.py
 mode = 0644

stack/monitor/status2rss.py

@@ -34,9 +34,12 @@ for row in rows:
 
   event_time = datetime.datetime.fromtimestamp(line_timestamp).strftime('%Y-%m-%d %H:%M:%S')
 
+  individual_rows = db.execute("select status, element, output from individual_status where timestamp=?", (line_timestamp,))
+  description = '\n'.join(['%s: %s %s' % row for row in individual_rows])
+
   rss_item = PyRSS2Gen.RSSItem(
     title = status,
-    description = "%s: %s" % (event_time, status),
+    description = "%s: %s\n%s" % (event_time, status, description),
     link = LINK,
     pubDate = event_time,
     guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, status)))

stack/resilient/buildout.cfg

@@ -38,7 +38,7 @@ eggs = collective.recipe.template
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-md5sum = e89d8378cc610704b518a89b095d3a19
+md5sum = cbc5bdb360fb5c72418dba03135df526
 mode = 0644
 
 [pbsready-import]
@@ -63,7 +63,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 3866b0d4d2872f693b7d9519a668e6bc
+md5sum = 0e6a95e7a9b38d402f94c11b7d10397e
 mode = 0644
 
 [template-replicated]
@@ -92,7 +92,7 @@ output = ${buildout:directory}/instance-frozen.cfg
 [resilient-web-takeover-cgi-script-download]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
-md5sum = e6262c5cf9b1c4d1ea4d959fdcbe3070
+md5sum = 5c90dae56885eeb490cc5d7f82d2dc5b
 mode = 0644
 destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
 
@@ -121,11 +121,6 @@ mode = 0644
 find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
 
 [versions]
-# Pin Jinja2 to 2.6, as 2.7 breaks current code
-Jinja2 = 2.6
-# ... And newer s.r.template requires Jinja2 >= 2.7
-slapos.recipe.template = 2.4.2
 rdiff-backup = 1.3.4nxd2
-slapos.cookbook = 0.92
 
 

stack/resilient/instance-pull-backup.cfg.in

@@ -83,6 +83,7 @@ notifier-callbacks = $${basedirectory:notifier}/callbacks
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
+lockfile = $${basedirectory:run}/equeue.lock
 log = $${basedirectory:log}/equeue.log
 database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue

stack/resilient/pbsready.cfg.in

@@ -157,6 +157,7 @@ rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
 [equeue]
 recipe = slapos.cookbook:equeue
 socket = $${basedirectory:run}/equeue.sock
+lockfile = $${basedirectory:run}/equeue.lock
 log = $${basedirectory:log}/equeue.log
 database = $${rootdirectory:srv}/equeue.db
 wrapper = $${basedirectory:services}/equeue

stack/resilient/resilient-web-takeover-cgi-script.py.in

 #!${buildout:executable}
+
+equeue_database = '${equeue:database}'
+equeue_lockfile = '${equeue:lockfile}'
+
 import cgi
 import cgitb
+import datetime
+import gdbm
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
 cgitb.enable()
 
+def getLatestBackupDate():
+  """
+  Get the date of the latest successful backup.
+  """
+  # Create a copy of the db (locked by equeue process)
+  temporary_directory = tempfile.mkdtemp()
+  equeue_database_copy = os.path.join(temporary_directory, 'equeue.db')
+  shutil.copyfile(equeue_database, equeue_database_copy)
+  db = gdbm.open(equeue_database_copy)
+  # Usually, there is only one callback (so only one key
+  # in the db), but if there are several:
+  # Take the "oldest" one (oldest value).
+  last_backup = db[db.keys()[0]]
+  for callback in db.keys():
+    timestamp = float(db[callback])
+    if timestamp < last_backup:
+      last_backup = timestamp
+  return datetime.datetime.fromtimestamp(last_backup)
+
+def isBackupInProgress():
+  """
+  Check if backup is in progress (importer script is running)
+  by checking if equeue lockfile exists.
+  """
+  # XXX: check if file is valid
+  return os.path.exists(equeue_lockfile)
+
 print "Content-Type: text/html"
-print    
+print
 
 form = cgi.FieldStorage()
 if "password" not in form:
@@ -17,12 +52,14 @@ if "password" not in form:
   <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
   <p><b>Warning: submit the form only if you understand what you are doing.</b></p>
   <p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p>
+  <p>Last valid backup: %s</p>
+  <p>Importer script(s) of backup in progress: %s</p>
   <form action="/">
     Password: <input type="text" name="password">
     <input type="submit" value="Take over" style="background: red;">
   </form>
 </body>
-</html>"""
+</html>""" % (getLatestBackupDate().strftime('%Y-%m-%d %H:%M:%S'), isBackupInProgress())
   sys.exit(0)
 
 if form['password'].value != '${:password}':

stack/slapos.cfg

@@ -20,8 +20,9 @@ extensions +=
 
 # Use shacache and lxml
 extends =
-  ../component/python-2.7/buildout.cfg
+  ../component/git/buildout.cfg
   ../component/lxml-python/buildout.cfg
+  ../component/python-2.7/buildout.cfg
   ../component/python-cffi/buildout.cfg
   ../component/python-cliff/buildout.cfg
   ../component/python-cryptography/buildout.cfg
@@ -102,9 +103,9 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.7.1-dev-SlapOS-005
+zc.buildout = 1.7.1.post6
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
-zc.recipe.egg = 1.3.2nxd001
+zc.recipe.egg = 1.3.2.post2
 # Use own version of h.r.download to be able to open .xz and .lz archives
 hexagonit.recipe.download = 1.7nxd003
 
@@ -115,6 +116,7 @@ buildout-versions = 1.7
 cffi = 0.8.6
 cmd2 = 0.6.8
 cryptography = 0.7.1
+inotifyx = 0.2.2
 itsdangerous = 0.24
 lxml = 3.4.1
 meld3 = 1.0.0
@@ -122,19 +124,20 @@ mr.developer = 1.31
 netaddr = 0.7.13
 pbr = 0.10.7
 prettytable = 0.7.2
-psutil = 2.1.3
+psutil = 2.2.0
 pyOpenSSL = 0.14
 pyparsing = 2.0.3
+pytz = 2014.10
 requests = 2.5.1
-setuptools = 7.0
+setuptools = 11.3.1
 simplejson = 3.6.5
 six = 1.9.0
-slapos.cookbook = 0.87
-slapos.core = 1.3.5
+slapos.cookbook = 0.93
+slapos.core = 1.3.6
 slapos.libnetworkcache = 0.14.2
 slapos.recipe.build = 0.16
 slapos.recipe.cmmi = 0.2
-stevedore = 1.1.0
+stevedore = 1.2.0
 xml-marshaller = 0.9.7
 
 # Required by:
@@ -143,7 +146,7 @@ Flask = 0.10.1
 
 # Required by:
 # cliff==1.9.0
-# stevedore==1.1.0
+# stevedore==1.2.0
 argparse = 1.3.0
 
 # Required by:
@@ -151,15 +154,15 @@ argparse = 1.3.0
 cliff = 1.9.0
 
 # Required by:
-# cryptography==0.7
+# cryptography==0.7.1
 enum34 = 1.0.4
 
 # Required by:
-# slapos.cookbook==0.87
-inotifyx = 0.2.2
+# slapos.cookbook==0.93
+jsonschema = 2.4.0
 
 # Required by:
-# slapos.cookbook==0.87
+# slapos.cookbook==0.93
 lock-file = 2.0
 
 # Required by:
@@ -171,17 +174,13 @@ netifaces = 0.10.4
 pip = 6.0.6
 
 # Required by:
-# cryptography==0.7
+# cryptography==0.7.1
 pyasn1 = 0.1.7
 
 # Required by:
 # cffi==0.8.6
 pycparser = 2.10
 
-# Required by:
-# slapos.cookbook==0.87
-pytz = 2014.10
-
 # Required by:
 # slapos.core==1.3.5
 supervisor = 3.1.3