Merge branch 'master' into slaprunner

component/gnutls/buildout.cfg

 [buildout]
 extends =
-  ../readline/buildout.cfg
+  ../gmp/buildout.cfg
+  ../nettle/buildout.cfg
   ../ncurses/buildout.cfg
+  ../readline/buildout.cfg
   ../zlib/buildout.cfg
 
 parts = gnutls
@@ -22,14 +24,13 @@ environment =
   LDFLAGS=-lgpg-error -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
 
 [gnutls]
-# XXX-Cedric : update to latest gnutls
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v2.8/gnutls-2.8.6.tar.bz2
-md5sum = eb0a6d7d3cb9ac684d971c14f9f6d3ba
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.0.tar.xz
+md5sum = e0cba4ddd923420026ff9739b3bc069a
 configure-options =
   --with-libgcrypt-prefix=${gcrypt:location}
   --disable-static
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gcrypt:location}/include -I${gpg-error:location}/include
-  LDFLAGS=-lgcrypt -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
-  PKG_CONFIG=${zlib:location}/lib/pkgconfig
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${gmp:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${nettle:location}/include
+  LDFLAGS=-lgcrypt -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${nettle:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
+

component/nettle/buildout.cfg

+[buildout]
+extends =
+  ../gmp/buildout.cfg
+  ../m4/buildout.cfg
+
+[nettle-lib-location.patch]
+recipe = hexagonit.recipe.download
+download-only = true
+filename = ${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+md5sum = 41dd0ce2a73487929bdc637b75dd62c9
+
+[nettle]
+recipe = slapos.recipe.cmmi
+url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
+md5sum = 003d5147911317931dd453520eb234a5
+patches =
+  ${nettle-lib-location.patch:location}/${nettle-lib-location.patch:filename}
+configure-option =
+  --disable-static
+  --disable-assembler
+  --disable-openssl
+environment =
+  PATH=${m4:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gmp:location}/include
+  LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib

component/nettle/nettle-lib-location.patch

+--- configure.orig	2013-07-05 15:37:28.000000000 +0200
++++ configure	2013-07-05 15:47:48.000000000 +0200
+@@ -4680,52 +4680,6 @@
+ if test "x$ABI" != xstandard ; then
+   { $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
+ $as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
+-  if test "$libdir" = '${exec_prefix}/lib' ; then
+-    # Try setting a better default
+-    case "$host_cpu:$host_os:$ABI" in
+-      *:solaris*:32|*:sunos*:32)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *:solaris*:64|*:sunos*:64)
+-	libdir='${exec_prefix}/lib/64'
+-	;;
+-      # Linux conventions are a mess... According to the Linux File
+-      # Hierarchy Standard, all architectures except IA64 puts 32-bit
+-      # libraries in lib, and 64-bit in lib64. Some distributions,
+-      # e.g., Fedora and Gentoo, adhere to this standard, while at
+-      # least Debian has decided to put 64-bit libraries in lib and
+-      # 32-bit libraries in lib32.
+-
+-      # We try to figure out the convention, except if we're cross
+-      # compiling. We use lib${ABI} if /usr/lib${ABI} exists and
+-      # appears to not be a symlink to a different name.
+-      *:linux*:32|*:linux*:64)
+-	if test "$cross_compiling" = yes ; then
+-          { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&5
+-$as_echo "$as_me: WARNING: Cross compiling for linux. Can't guess if libraries go in lib${ABI} or lib." >&2;}; 	else
+-	  # The dash builtin pwd tries to be "helpful" and remember
+-	  # symlink names. Use -P option, and hope it's portable enough.
+-	  test -d /usr/lib${ABI} \
+-	    && (cd /usr/lib${ABI} && pwd -P | grep >/dev/null "/lib${ABI}"'$') \
+-	    && libdir='${exec_prefix}/'"lib${ABI}"
+-	fi
+-	;;
+-      # On freebsd, it seems 32-bit libraries are in lib32,
+-      # and 64-bit in lib. Don't know about "kfreebsd", does
+-      # it follow the Linux fhs conventions?
+-      *:freebsd*:32)
+-	libdir='${exec_prefix}/lib32'
+-	;;
+-      *:freebsd*:64)
+-	libdir='${exec_prefix}/lib'
+-	;;
+-      *)
+-        { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
+-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
+-    esac
+-    { $as_echo "$as_me:${as_lineno-$LINENO}: Libraries to be installed in $libdir." >&5
+-$as_echo "$as_me: Libraries to be installed in $libdir." >&6;}
+-  fi
+ fi
+ 
+ # Select assembler code

component/qemu-kvm/buildout.cfg

 [buildout]
-extends = 
+extends =
   ../../component/gnutls/buildout.cfg
   ../../component/libpng/buildout.cfg
   ../../component/libuuid/buildout.cfg
+  ../../component/pkgconfig/buildout.cfg
   ../../component/xorg/buildout.cfg
   ../../component/zlib/buildout.cfg
 
-[kvm]
-# Backward compatibility
-<= qemu-kvm
+# XXX Change all reference to kvm section to qemu section, then
+# use qemu as main name section.
+[qemu]
+<= kvm
 
-[qemu-kvm]
+[kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.4.1.tar.bz2
-md5sum = eb2d696956324722b5ecfa46e41f9a75
+url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
+md5sum = b56e73bdcfdb214d5c68e13111aca96f
 depends =
   ${libpng:so_version}
 configure-options =
-  --target-list=""
+  --target-list=x86_64-softmmu
+  --enable-system
+  --with-system-pixman
   --disable-sdl
   --disable-xen
   --enable-vnc-tls
@@ -36,4 +40,27 @@ configure-options =
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
+  LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
+
+# The following is only available in buildout2, which we don't use yet.
+[kvm-bits64]
+configure-options =
+  --target-list=x86_64-softmmu
+  ${kvm:configure-options}
+
+[kvm-bits32]
+configure-options =
+  --target-list=i386-softmmu
+  ${kvm:configure-options}
+
+
+[debian-amd64-netinst.iso]
+# Download the installer of Debian 7 (Wheezy)
+recipe = slapos.recipe.download
+url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
+filename = ${:_buildout_section_name_}
+md5sum = 80f498a1f9daa76bc911ae13692e4495
+download-only = true
+mode = 0644
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
 

slapos/recipe/addresiliency/renamer.py

@@ -59,8 +59,6 @@ class Renamer(object):
 
         cp_broken.rename(new_name=broken_new_ref)
 
-        cp_broken.stopped()
-
         log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
 
         # update name (and later, software type) for the partition that will take over

slapos/recipe/generic_varnish/__init__.py

@@ -37,9 +37,11 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     ip = self.options['ip']
-    backend_url = self.parameter_dict.get(
-      'backend-url',
-      self.parameter_dict.get('tidstorage-url') # BBB
+    backend_url = self.options.get('backend-url',
+      # BBB: Peeking in partition parameters directly. Eew.
+      self.parameter_dict.get('backend-url',
+        self.parameter_dict.get('tidstorage-url') # BBB
+      )
     )
     backend_server, backend_port = self._getBackendServer(backend_url)
     path_list = []

slapos/recipe/kvm/__init__.py

@@ -61,7 +61,8 @@ class Recipe(GenericBaseRecipe):
       shell_path=self.options['shell-path'],
       qemu_path=self.options['qemu-path'],
       qemu_img_path=self.options['qemu-img-path'],
-      vnc_passwd=self.options['passwd']
+      vnc_passwd=self.options['passwd'],
+      default_disk_image=self.options['default-disk-image'],
     )
 
     # Runners

slapos/recipe/kvm/template/kvm_run.in

@@ -7,6 +7,9 @@ import os
 import socket
 import subprocess
 
+# XXX: give all of this through parameter, don't use this as template
+default_disk_image = '%(default_disk_image)s'
+
 def getSocketStatus(host, port):
   s = None
   for res in socket.getaddrinfo(host, port,
@@ -26,7 +29,8 @@ def getSocketStatus(host, port):
     break
   return s
 
-# create disk if doesn't exist
+# Create disk if doesn't exist
+# XXX: move to Buildout profile
 disk_path = '%(disk_path)s'
 if not os.path.exists(disk_path):
   subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
@@ -58,4 +62,10 @@ for nbd_ip, nbd_port in (
           '-drive',
           'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
 
+# If no NBD is specified/available: use internal disk image
+else:
+  kvm_argument_list.extend([
+      '-drive', 'file=%%s,media=cdrom' %% default_disk_image
+  ])
+
 os.execv('%(qemu_path)s', kvm_argument_list)

slapos/recipe/softwaretype.py

@@ -87,8 +87,6 @@ class Recipe:
       computer_partition_id)
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
     software_type = self.parameter_dict['slap_software_type']
-    self.logger.info('Deploying instance with software type %s' % \
-        software_type)
 
     # Raise if request software_type does not exist ...
     if software_type not in self.options:

slapos/recipe/web_checker/__init__.py

@@ -34,11 +34,15 @@ class Recipe(GenericSlapRecipe):
   """
   def _install(self):
     path_list = []
-    web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
-    web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
-    web_checker_frontend_url = self.parameter_dict.get(
-                                 'web-checker-frontend-url',
-                                 self.options['frontend-url'])
+    try:
+      web_checker_mail_address = self.options['mail-address']
+      web_checker_smtp_host = self.options['smtp-host']
+      web_checker_frontend_url = self.options['frontend-url']
+    except KeyError:
+      # BBB
+      web_checker_mail_address = self.parameter_dict['web-checker-mail-address']
+      web_checker_smtp_host = self.parameter_dict['web-checker-smtp-host']
+      web_checker_frontend_url = self.parameter_dict['web-checker-frontend-url']
     web_checker_working_directory = \
       self.options['web-checker-working-directory']
     config = dict(

software/cloudooo/instance-cloudoo-input-schema.json

@@ -17,7 +17,7 @@
         "type": "string"
       },
       "type": "array"
-    }
+    },
     "backend-count": {
       "description": "Number of backend cloudooo instances",
       "default": 1,

software/kvm/common.cfg

@@ -9,13 +9,18 @@ extends =
   ../../component/logrotate/buildout.cfg
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
+  ../../component/dcron/buildout.cfg
   ../../stack/nodejs.cfg
+  ../../stack/resilient/buildout.cfg
   ../../stack/slapos.cfg
 
 parts =
   template
   eggs
 
+# XXX: we have to manually add this for resilience
+  rdiff-backup
+
 #XXX-Cedric : Currently, one can only access to KVM using noVNC.
 #             Ideally one should be able to access KVM by using either NoVNC or VNC.
 #             Problem is : no native crypto support in web browsers. So we have to disable ssl
@@ -67,13 +72,58 @@ command =
   ${nodejs:location}/bin/node ${nodejs:location}/bin/npm install pkginfo@0.2.3
 
 
+# Create all templates that will be used to deploy instances
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg.in
+md5sum = 0e84223169661462f439c164d62c2a6a
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg.in
-md5sum = 87197471aa93863c310204e8865b5ac1
+md5sum = f7c0e2172dac4ee70daae50f38d610ef
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
+[template-kvm-resilient]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja
+mode = 644
+md5sum = 3ee64c654aae503b93b39e9ccd6d3643
+
+[template-kvm-import]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
+md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
+output = ${buildout:directory}/template-kvm-import.cfg
+mode = 0644
+
+[template-kvm-import-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-import.sh.in
+filename = kvm-import.sh.in
+md5sum = e03ed049cddd8d157228b09e1ebc071a
+download-only = true
+mode = 0755
+
+[template-kvm-export]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
+md5sum = 64a1a505aff9fde52afac46240811047
+output = ${buildout:directory}/template-kvm-export.cfg
+mode = 0644
+
+[template-kvm-export-script]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/template/kvm-export.sh.in
+filename = kvm-export.sh.in
+md5sum = 08cd8da2221f09095b14e35e6acd2a56
+download-only = true
+mode = 0755
+
 [template-nbd]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-nbd.cfg.in
@@ -87,10 +137,3 @@ url = ${:_profile_base_location_}/instance-frontend.cfg.in
 md5sum = cdb690495e9eb007d2b7d2f8e12f5c59
 output = ${buildout:directory}/template-frontend.cfg
 mode = 0644
-
-[template]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 0a98e34aaec7097a84066c0665e3a49a
-output = ${buildout:directory}/template.cfg
-mode = 0644

software/kvm/development.cfg

@@ -31,3 +31,6 @@ update-command = ${:command}
 command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
+
+[versions]
+websockify = 0.3.0
\ No newline at end of file

software/kvm/instance-kvm-export.cfg.in

+[buildout]
+extends = ${template-kvm:output}
+          ${pbsready-export:output}
+
+parts +=
+  cron-entry-backup
+
+  certificate-authority
+  publish-connection-information
+  kvm-promise
+  websockify-sighandler
+  novnc-promise
+  cron
+  frontend-promise
+
+# Create the exporter executable, which is a simple shell script
+[exporter]
+recipe = slapos.recipe.template
+url = ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-exporter
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+# Extends publish section with resilient parameters
+[publish-connection-information]
+<= resilient-publish-connection-parameter
+

software/kvm/instance-kvm-import.cfg.in

+[buildout]
+# Here, we don't need KVM to run to import data, so we don't
+# even extend the kvm instance profile.
+extends = ${pbsready-import:output}
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promises = $${:etc}/promise
+novnc-conf = $${:etc}/novnc
+run = $${:var}/run
+ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
+
+[importer]
+recipe = slapos.recipe.template
+url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+output = $${directory:bin}/$${slap-parameter:namebase}-importer
+mode = 0755
+backup-disk-path = $${directory:backup}/virtual.qcow2
+disk-path = $${directory:srv}/virtual.qcow2
+# Resilient stack wants a "wrapper" parameter
+wrapper = $${:output}
+
+backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient.cfg.jinja2

+# vim: set ft=cfg:
+
+{% import 'parts' as parts %}
+{% import 'replicated' as replicated with context %}
+
+[buildout]
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
+parts +=
+  {{ parts.replicate("kvm", "2") }}
+  publish-connection-informations
+
+{{ replicated.replicate("kvm", "2", "kvm-export", "kvm-import") }}
+
+# Bubble down the parameters of the requested instance to the user
+[request-kvm]
+# Note: += doesn't work.
+return =
+# Resilient related parameters
+  url ssh-public-key ssh-url notification-id ip
+# KVM related parameters
+  backend-url url
+
+[publish-connection-informations]
+recipe = slapos.cookbook:publish
+backend-url = ${request-kvm:connection-backend-url}
+url = ${request-kvm:connection-url}
+

software/kvm/instance-kvm.cfg.in

@@ -6,10 +6,13 @@
 [buildout]
 parts =
   certificate-authority
-  publish-kvm-connection-information
+  publish-connection-information
   kvm-promise
   websockify-sighandler
   novnc-promise
+#  kvm-monitor
+  cron
+#  cron-entry-monitor
   frontend-promise
 
 eggs-directory = ${buildout:eggs-directory}
@@ -22,12 +25,16 @@ etc = $${buildout:directory}/etc
 bin = $${buildout:directory}/bin
 srv = $${buildout:directory}/srv
 var = $${buildout:directory}/var
+log = $${:var}/log
 scripts = $${:etc}/run
 services = $${:etc}/service
 promises = $${:etc}/promise
 novnc-conf = $${:etc}/novnc
 run = $${:var}/run
 ca-dir = $${:srv}/ssl
+cron-entries = $${:etc}/cron.d
+crontabs = $${:etc}/crontabs
+cronstamps = $${:etc}/cronstamps
 
 [create-mac]
 recipe = slapos.cookbook:generate.mac
@@ -44,6 +51,7 @@ bytes = 8
 recipe = slapos.cookbook:kvm
 vnc-ip = $${slap-network-information:local-ipv4}
 vnc-port = 5901
+default-disk-image = ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
 nbd-host = $${slap-parameter:nbd-host}
 nbd-port = $${slap-parameter:nbd-port}
 nbd2-host = $${slap-parameter:nbd2-host}
@@ -122,11 +130,43 @@ hostname = $${novnc-instance:ip}
 port = $${novnc-instance:port}
 
 
-[kvm-monitor]
-recipe = slapos.cookbook:generic.slapmonitor
-db-path = $${directory:srv}/slapmonitor_database
+#[kvm-monitor]
+#recipe = slapos.cookbook:wrapper
+#wrapper-path = $${directory:services}/kvm_monitor
+#command-line = ${buildout:bin-directory}/kvm.monitor.test
+#             $${buildout:directory}/buildout-switch-softwaretype.cfg
+#             $${buildout:directory}/report.xml
+#             -s slap-parameter
+#             -opts disk-size ram-size cpu-count
 
 
+
+#----------------
+#--
+#-- Deploy cron.
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron:location}/sbin/crond
+cron-entries = $${directory:cron-entries}
+crontabs = $${directory:crontabs}
+cronstamps = $${directory:cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:services}/crond
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/crond.log
+
+
+#[cron-entry-monitor]
+#<= cron
+#recipe = slapos.cookbook:cron.d
+#name = kvm_monitor
+#frequency = 0 0 * * *
+#command = $${kvm-monitor:wrapper-path}
+
 [request-slave-frontend]
 recipe = slapos.cookbook:requestoptional
 software-url = $${slap-parameter:frontend-software-url}
@@ -146,17 +186,16 @@ sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 
 
-[publish-kvm-connection-information]
+[publish-connection-information]
 recipe = slapos.cookbook:publish
-backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1
-password = $${kvm-instance:passwd}
-url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}
+backend-url = https://[$${novnc-instance:ip}]:$${novnc-instance:port}/vnc_auto.html?host=[$${novnc-instance:ip}]&port=$${novnc-instance:port}&encrypt=1&password=$${kvm-instance:passwd}
+url = $${request-slave-frontend:connection-url}/vnc_auto.html?host=$${request-slave-frontend:connection-domainname}&port=$${request-slave-frontend:connection-port}&encrypt=1&path=$${request-slave-frontend:connection-resource}&password=$${kvm-instance:passwd}
 
 
 [frontend-promise]
 recipe = slapos.cookbook:check_url_available
 path = $${directory:promises}/frontend_promise
-url = $${publish-kvm-connection-information:url}
+url = $${publish-connection-information:url}
 dash_path = ${dash:location}/bin/dash
 curl_path = ${curl:location}/bin/curl
 
@@ -164,9 +203,9 @@ curl_path = ${curl:location}/bin/curl
 # Default values if not specified
 frontend-software-type = frontend
 frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
-
+frontend-instance-guid =
 nbd-port = 1024
-nbd-host = debian.nbd.vifib.net
+nbd-host =
 nbd2-port = 1024
 nbd2-host =
 

software/kvm/instance.cfg.in

@@ -4,7 +4,6 @@ parts =
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
 
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
@@ -13,11 +12,33 @@ kvm = ${template-kvm:output}
 nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
 
-[slap-connection]
-# part to migrate to new - separated words
-computer-id = $${slap_connection:computer_id}
-partition-id = $${slap_connection:partition_id}
-server-url = $${slap_connection:server_url}
-software-release-url = $${slap_connection:software_release_url}
-key-file = $${slap_connection:key_file}
-cert-file = $${slap_connection:cert_file}
+kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
+kvm-import = ${template-kvm-import:output}
+kvm-export = ${template-kvm-export:output}
+
+frozen = ${instance-frozen:output}
+pull-backup = ${template-pull-backup:output}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[dynamic-template-kvm-resilient]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
+md5sum = 1b7a2fcc884649b4d08f238e828899c1
+rendered = $${buildout:directory}/template-kvm-resilient.cfg
+context = key buildout buildout:bin-directory
+          key develop_eggs_directory buildout:develop-eggs-directory
+          key eggs_directory buildout:eggs-directory
+          key slapparameter_dict slap-configuration:configuration
+template-parts-destination = ${template-parts:destination}
+template-replicated-destination = ${template-replicated:destination}
+import-list = file parts :template-parts-destination
+              file replicated :template-replicated-destination
+mode = 0644
+

software/kvm/software.cfg

@@ -47,98 +47,96 @@ signature-certificate-list =
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
 
-
 [versions]
-numpy = 1.6.2
-Jinja2 = 2.6
-Werkzeug = 0.8.3
-apache-libcloud = 0.12.1
+Werkzeug = 0.9.3
+apache-libcloud = 0.13.0
 async = 0.6.1
 buildout-versions = 1.7
 gitdb = 0.5.4
-hexagonit.recipe.cmmi = 1.6
-lxml = 3.1.0
+itsdangerous = 0.22
+lxml = 3.2.3
 meld3 = 0.6.10
 plone.recipe.command = 1.1
 pycrypto = 2.6
-slapos.cookbook = 0.73.1
+rdiff-backup = 1.0.5
+slapos.cookbook = 0.78.3
+slapos.recipe.cmmi = 0.2
+slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.33.1
+slapos.toolbox = 0.35.0
 smmap = 0.8.2
-websockify = 0.3.0
+websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-Flask = 0.9
+# slapos.toolbox==0.35.0
+Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 atomize = 0.1.1
 
 # Required by:
-# slapos.toolbox==0.33.1
+# slapos.toolbox==0.35.0
 feedparser = 5.1.3
 
 # Required by:
-# hexagonit.recipe.cmmi==1.6
-hexagonit.recipe.download = 1.6nxd002
+# slapos.cookbook==0.78.3
+inotifyx = 0.2.0-1
 
 # Required by:
-# slapos.cookbook==0.73.1
-inotifyx = 0.2.0
+# slapos.cookbook==0.78.3
+lock-file = 2.0
 
 # Required by:
-# slapos.cookbook==0.73.1
+# slapos.cookbook==0.78.3
 netaddr = 0.7.10
 
 # Required by:
 # slapos.core==0.35.1
-netifaces = 0.8
+netifaces = 0.8-1
 
 # Required by:
-# slapos.toolbox==0.33.1
-paramiko = 1.10.0
+# websockify==0.5.1
+numpy = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.33.1
-psutil = 0.6.1
+# slapos.toolbox==0.35.0
+paramiko = 1.11.0
 
 # Required by:
-# slapos.core==0.35.1
-pyflakes = 0.6.1
+# slapos.toolbox==0.35.0
+psutil = 1.0.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-pytz = 2012j
+# slapos.core==0.35.1
+pyflakes = 0.7.3
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.core==0.35.1
-# slapos.toolbox==0.33.1
-setuptools = 0.6c12dev-r88846
+# slapos.cookbook==0.78.3
+pytz = 2013b
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 slapos.core = 0.35.1
 
 # Required by:
 # slapos.core==0.35.1
-supervisor = 3.0b1
+supervisor = 3.0b2
 
 # Required by:
 # slapos.core==0.35.1
 unittest2 = 0.5.1
 
 # Required by:
-# slapos.cookbook==0.73.1
-# slapos.toolbox==0.33.1
+# slapos.cookbook==0.78.3
+# slapos.toolbox==0.35.0
 xml-marshaller = 0.9.7
 
 # Required by:

software/kvm/template/kvm-export.sh.in

+#!/bin/bash
+# Create a backup of the disk image of the virtual machine
+QEMU_IMG=${kvm-instance:qemu-img-path}
+SNAPSHOT_NAME=$(date +%s)
+DISK_PATH=${kvm-instance:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+if [ ! -f $DISK_PATH ]; then
+  echo "Nothing to backup, disk image doesn't exist yet."
+  exit 0;
+fi
+
+$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
+if [ -f $BACKUP_PATH ]; then
+  rm $BACKUP_PATH
+fi
+$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
+$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH

software/kvm/template/kvm-import.sh.in

+#!/bin/bash
+DISK_PATH=${:disk-path}
+BACKUP_PATH=${:backup-disk-path}
+
+# TODO: Use rdiff
+rm $DISK_PATH
+cp $BACKUP_PATH $DISK_PATH

software/varnish/instance-varnish-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "extends": "./schema-definitions.json#",
+  "properties": {
+    "tcpv4-port": {
+      "allOf": [{
+        "$ref": "#/definitions/tcpv4port"
+      }, {
+        "description": "Start allocating ports at this value, going upward",
+        "default": 6001
+      }]
+    },
+    "backend-url": {
+      "description": "The backend url that varnish will cache",
+      "type": "string"
+    },
+    "web-checker": {
+      "description": "Controls automated cache checker, disabled if null or empty"
+      "properties": {
+        "frontend-url": {
+          "description": "Override entry-point-url web checker will check the HTTP headers of all links in the web site, '%(ip)s' and '%(port)s' being substituted with varnish's listening ip and port, respectively",
+          "default": "http://%(ip)s:%(port)s/",
+          "type": "string"
+        },
+        "mail-address": {
+          "description": "Email address to which web checker result is sent",
+          "type": "string"
+        },
+        "smtp-host": {
+          "description": "The smtp server to be used to send the web checker result",
+          "type": "string"
+        }
+      },
+      "type": "object"
+    }
+  }
+}

software/varnish/instance-varnish-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Varnish instanciation",
+  "properties": {
+    "url": {
+      "description": "Varnish HTTP service access information",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/varnish/instance-varnish.cfg.in

-{% set web_checker_parameter = slapparameter_dict['web-checker-mail-address'] -%}
+{% set tcpv4_port = slapparameter_dict.get('tcpv4_port', 6001) -%}
+{% set ip = (ipv4_set | list)[0] %}
 [buildout]
 parts =
   publish-varnish-connection-information
   varnish-instance
   cron
   cron-entry-logrotate
-  {# When web_checker related parameter is given, web_checker will be enabled.#}
-{% if web_checker_parameter is defined %}
+  {# When web_checker related parameter is given, web_checker will be enabled.-#}
+{% set web_checker_dict = slapparameter_dict.get('web-checker', {}) -%}
+{% if web_checker_dict -%}
   web-checker
   cron-entry-web-checker
   logrotate-entry-web-checker
-{% endif %}
+
+[cron-entry-web-checker]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = web-checker
+frequency = 0 0 * * *
+command = ${varnish-instance:web-checker} ${web-checker:web-checker-config}
+
+[web-checker]
+recipe = slapos.cookbook:webchecker
+web-checker-config = ${rootdirectory:etc}/web_checker.cfg
+web-checker-working-directory = ${directory:web-checker}
+frontend-url = {{ web_checker_dict.get('frontend-url', 'http://%(ip)s:%(port)s/') % {
+  'ip': ip,
+  'port': tcpv4_port,
+} }}
+mail-address = {{ web_checker_dict['mail-address'] }}
+smtp-host = {{ web_checker_dict['smtp-host'] }}
+wget-binary-path = {{ parameter_dict['wget'] }}/bin/wget
+varnishlog-binary-path = ${varnish-instance:varnishlog-wrapper}
+web-checker-log = ${basedirectory:log}/web-checker.log
+
+[logrotate-entry-web-checker]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = web-checker
+log = ${web-checker:web-checker-log}
+frequency = daily
+rotate-num = 30
+sharedscripts = true
+notifempty = true
+create = true
+{%- endif %}
 
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 [publish-varnish-connection-information]
-recipe = slapos.cookbook:publishurl
+recipe = slapos.cookbook:publish.serialised
 url = http://${varnish-instance:ip}:${varnish-instance:server-port}/
 
 [varnish-instance]
 recipe = slapos.cookbook:generic.varnish
+backend-url = {{ slapparameter_dict['backend-url'] }}
 
 # Network options
-ip = ${slap-network-information:local-ipv4}
-server-port = 6001
-manager-port = 6002
-stunnel-port = 6003
+ip = {{ ip }}
+server-port =  {{ tcpv4_port }}
+manager-port = {{ tcpv4_port + 1 }}
+stunnel-port = {{ tcpv4_port + 2}}
 
 # Paths: Running wrappers
 varnishd-wrapper = ${basedirectory:services}/varnishd
@@ -50,22 +85,6 @@ varnish-data = ${directory:varnish-data}
 varnish-instance-name = ${directory:varnish-instance}
 web-checker = {{ parameter_dict['buildout-bin-directory'] }}/web_checker_utility
 
-[cron-entry-web-checker]
-<= cron
-recipe = slapos.cookbook:cron.d
-name = web-checker
-frequency = 0 0 * * *
-command = ${varnish-instance:web-checker} ${web-checker:web-checker-config}
-
-[web-checker]
-recipe = slapos.cookbook:webchecker
-web-checker-config = ${rootdirectory:etc}/web_checker.cfg
-web-checker-working-directory = ${directory:web-checker}
-frontend-url = ${varnish-instance:ip}:${varnish-instance:server-port}
-wget-binary-path = {{ parameter_dict['wget'] }}/bin/wget
-varnishlog-binary-path = ${varnish-instance:varnishlog-wrapper}
-web-checker-log = ${basedirectory:log}/web-checker.log
-
 [cron]
 recipe = slapos.cookbook:cron
 dcrond-binary = {{ parameter_dict['dcron'] }}/sbin/crond
@@ -100,17 +119,6 @@ logrotate-entries = ${directory:logrotate-entries}
 backup = ${directory:logrotate-backup}
 state-file = ${rootdirectory:srv}/logrotate.status
 
-[logrotate-entry-web-checker]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = web-checker
-log = ${web-checker:web-checker-log}
-frequency = daily
-rotate-num = 30
-sharedscripts = true
-notifempty = true
-create = true
-
 [basedirectory]
 recipe = slapos.cookbook:mkdirectory
 services = ${rootdirectory:etc}/run

software/varnish/instance.cfg.in

@@ -6,8 +6,8 @@ eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
-[slap-parameters]
-recipe = slapos.cookbook:slapconfiguration
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
 partition = ${slap-connection:partition-id}
 url = ${slap-connection:server-url}
@@ -21,7 +21,8 @@ extra-context =
 context =
     key eggs_directory buildout:eggs-directory
     key develop_eggs_directory buildout:develop-eggs-directory
-    key slapparameter_dict slap-parameters:configuration
+    key slapparameter_dict slap-configuration:configuration
+    key ipv4_set slap-configuration:ipv4
     ${:extra-context}
 
 [dynamic-template-varnish-parameters]

software/varnish/software.cfg

@@ -44,7 +44,7 @@ parts =
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.git
-branch = erp5
+branch = master
 git-executable = ${git:location}/bin/git
 
 [check-recipe]
@@ -71,7 +71,7 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = c380d2101bfb26b90f84a624809254a0
+md5sum = 8e906d749e19ee13fe5b7f4d9bfcf896
 extra-context =
     key buildout_bin_directory buildout:bin-directory
     key dash_location dash:location
@@ -87,7 +87,7 @@ extra-context =
 [template-varnish]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-varnish.cfg.in
-md5sum = 6b6d748bf089038234f864417a6182cc
+md5sum = 4334d900f212d170fd0ca35865879bdf
 mode = 640
 
 [eggs]

stack/resilient/buildout.cfg

 [buildout]
 extends =
+  ../../component/dropbear/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
+  ../../component/rsync/buildout.cfg
 
 parts =
   rdiff-backup
@@ -11,7 +13,6 @@ parts =
   template-replicated
   template-parts
   instance-frozen
-  template-resilient
 
   # needed tools for resiliency
   gzip
@@ -38,7 +39,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = 1b1308fd39476d48b5ca13db48ea6dc9
+md5sum = 3c2e73f49abdc52282fc045e6d91f3e9
 mode = 0644
 
 [pbsready-export]
@@ -47,20 +48,20 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-export.cfg.in
 output = ${buildout:directory}/pbsready-export.cfg
-md5sum = 5d9e20c436fd307e8e4ab224a9a65792
+md5sum = a0e22a5de727544c5767d6bee059a77a
 mode = 0644
 
 [template-pull-backup]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
 output = ${buildout:directory}/instance-pull-backup.cfg
-md5sum = 453d96f5a6c1230c01c878cc7640bae6
+md5sum = c67a9dad66490ae264f9e7003521bf59
 mode = 0644
 
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = 9e20f283bf709c63c9c6692d5e1f8972
+#md5sum = 9e20f283bf709c63c9c6692d5e1f8972
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
@@ -77,12 +78,10 @@ destination = ${buildout:directory}/template-parts.cfg.in
 # which will run without removing any content because it raises an error.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-frozen.cfg.in
+md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 output = ${buildout:directory}/instance-frozen.cfg
 
-[template-resilient]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/resilient.cfg.in
-output = ${buildout:directory}/resilient.cfg
-md5sum = 59e74d290d623de2c1e147e48f284fba
-mode = 0644
+[versions]
+# Pin Jinja2 to 2.6, as 2.7 breaks current code
+Jinja2 = 2.6
 

stack/resilient/instance-frozen.cfg.in

 [buildout]
-  parts = 
+eggs-directory = ${buildout:eggs-directory}
\ No newline at end of file
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+parts = 
+

stack/resilient/instance-pull-backup.cfg.in

@@ -105,7 +105,8 @@ promises-directory = $${basedirectory:promises}
 directory = $${directory:pbs-backup}
 cron-entries = $${cron:cron-entries}
 wrappers-directory = $${directory:pbs-wrappers}
-notifier-url = http://[$${notifier:host}]:$${notifier:port}/
+# XXX: this should be named "notifier-host"
+notifier-url = http://[$${notifier:host}]:$${notifier:port}
 slave-instance-list = $${slap-parameter:slave_instance_list}
 
 

stack/resilient/pbsready-export.cfg.in

@@ -2,7 +2,21 @@
 
 extends = ${pbsready:output}
 
-parts +=
+# Explicitely define extended parts from pbsready
+# then add local parts
+parts =
+  resiliency
+  logrotate
+  logrotate-entry-cron
+  logrotate-entry-equeue
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  dropbear-server
+  sshkeys-dropbear
+  dropbear-server-pbs-authorized-key
+  notifier
+
   cron-entry-backup
 
 [resilient-publish-connection-parameter]

stack/resilient/pbsready-import.cfg.in

@@ -2,7 +2,21 @@
 
 extends = ${pbsready:output}
 
-parts +=
+# Explicitely define extended parts from pbsready
+# then add local parts
+parts =
+  resiliency
+  logrotate
+  logrotate-entry-cron
+  logrotate-entry-equeue
+  cron
+  cron-entry-logrotate
+  sshkeys-authority
+  dropbear-server
+  sshkeys-dropbear
+  dropbear-server-pbs-authorized-key
+  notifier
+
   import-on-notification
   resilient-publish-connection-parameter
 

stack/resilient/resilient.cfg.in

-## not used at the moment
-
-[buildout]
-
-parts =
-  request-pull-backup-server
-
-[request-pull-backup-server]
-<= slap-connection
-recipe = slapos.cookbook:request
-name = PBS (Pull Backup Server)
-software-url = $${slap-connection:software-release-url}
-software-type = pull-backup
-return = ssh-key notification-url feeds-url
-slave = false
\ No newline at end of file

stack/resilient/template-replicated.cfg.in

@@ -18,12 +18,22 @@ software-type = {{typeexport}}
 name = {{namebase}}0
 return = ssh-public-key ssh-url notification-id ip
 
-config = number authorized-key notify ip-list namebase
+config =
+# Resilient related parameters
+  number authorized-key notify ip-list namebase
+{% if slapparameter_dict is defined %}
+# Software Instance related parameters
+  {% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
+{% endif %}
 config-number = 0
 config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
 config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
 config-ip-list =
-
+# Bubble up all the instance parameters to the requested export instance.
+{% if slapparameter_dict is defined %}
+{% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
+{% endfor %}
+{% endif %}
 
 {% for id in range(1,nbbackup|int) %}
 
@@ -135,3 +145,4 @@ pbs-{{namebase}}{{id}}-computer-guid =
 {% endfor %}
 
 {% endmacro %}
+

stack/slapos.cfg

@@ -78,6 +78,9 @@ zc.buildout = 1.6.0-dev-SlapOS-010
 zc.recipe.egg = 1.3.2
 # Use own version of h.r.download to be able to open xz-like archives
 hexagonit.recipe.download = 1.6nxd002
+# Use pinned version of setuptools. Other versions work, but changing
+# version makes buildout recompile everything. Developers' nightmare.
+setuptools = 0.9.8
 
 [networkcache]
 download-cache-url = http://www.shacache.org/shacache