Merge remote-tracking branch 'origin/master' into erp5-component

component/qemu-kvm/buildout.cfg

@@ -15,8 +15,8 @@ extends =
 [kvm]
 recipe = slapos.recipe.cmmi
 # qemu-kvm and qemu are now the same since 1.3.
-url = http://wiki.qemu-project.org/download/qemu-1.5.1.tar.bz2
-md5sum = b56e73bdcfdb214d5c68e13111aca96f
+url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
+md5sum = 3a897d722457c5a895cd6ac79a28fda0
 depends =
   ${libpng:so_version}
 configure-options =
@@ -57,9 +57,9 @@ configure-options =
 [debian-amd64-netinst.iso]
 # Download the installer of Debian 7 (Wheezy)
 recipe = hexagonit.recipe.download
-url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
+url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso 
 filename = ${:_buildout_section_name_}
-md5sum = 80f498a1f9daa76bc911ae13692e4495
+md5sum = b86774fe4de88be6378ba3d71b8029bd 
 download-only = true
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}

software/kvm/common.cfg

@@ -80,14 +80,14 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 8617a8cc345a55688c5449528daef4d1
+md5sum = 24090ade9336a12a8fd30c5225a16267
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-kvm]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm.cfg.in
-#md5sum = c3c888c78bbff334135be9e8ad5885a9
+md5sum = c06bb498593aabc9c76eb7dc892da15a 
 output = ${buildout:directory}/template-kvm.cfg
 mode = 0644
 
@@ -95,14 +95,14 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 45a846378215eded6c001d0dd729a1ec
+md5sum = 038c338e3ce545a73393ceee38a9ac7d
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = b58427f93d5fcca94bdc90661fe6080b
+md5sum = 4057e7662ac36a4f591c17fc48e1603e
 mode = 0644
 download-only = true
 on-update = true
@@ -125,7 +125,7 @@ mode = 0755
 [template-kvm-export]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.in
-md5sum = 64a1a505aff9fde52afac46240811047
+md5sum = 2f5fdf1e88e6e3454f877b80074bed05 
 output = ${buildout:directory}/template-kvm-export.cfg
 mode = 0644
 
@@ -133,7 +133,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-export.sh.in
 filename = kvm-export.sh.in
-md5sum = bf03a90f6960b37cba812ee936a13342
+md5sum = 95fde96f35cbf90d677c44d18b60fafb 
 download-only = true
 mode = 0755
 

software/kvm/instance-kvm-export.cfg.in

@@ -7,7 +7,8 @@ parts +=
 
   certificate-authority
   publish-connection-information
-  kvm-promise
+  kvm-vnc-promise
+  kvm-disk-image-corruption-promise
   websockify-sighandler
   novnc-promise
   cron

software/kvm/instance-kvm-resilient-input-schema.json

@@ -28,10 +28,10 @@
       "title": "Periodicity of backup",
       "description": "Periodicity of backup, in cron format.",
       "type": "string"
-    }
+    },
     "remove-backup-older-than": {
       "title": "Remove backups older than...",
-      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible."
+      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
       "type": "string",
       "default": "3B"
     }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -55,5 +55,5 @@ sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]
-virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
+virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/fbd4ad.php?dl=true
 virtual-hard-drive-md5sum = 465e1024447997e7b86ee2e5151e031b

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -17,6 +17,11 @@ parts +=
 
 {{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+promises = ${:etc}/promise
+
 # Bubble down the parameters of the requested instance to the user
 [request-kvm]
 # Note: += doesn't work.
@@ -47,6 +52,6 @@ mode = 700
 # Check that backend url is reachable
 recipe = slapos.cookbook:check_url_available
 path = ${directory:promises}/frontend_promise
-url = ${publish-connection-information:url}
+url = ${publish-connection-informations:url}
 dash_path = /bin/sh
 curl_path = {{ curl_executable_location }}

software/kvm/instance-kvm.cfg.in

@@ -7,7 +7,8 @@
 parts =
   certificate-authority
   publish-connection-information
-  kvm-promise
+  kvm-vnc-promise
+  kvm-disk-image-corruption-promise
   websockify-sighandler
   novnc-promise
 #  kvm-monitor
@@ -96,12 +97,20 @@ qemu-img-path = ${kvm:location}/bin/qemu-img
 6tunnel-path = ${6tunnel:location}/bin/6tunnel
 
 
-[kvm-promise]
+[kvm-vnc-promise]
 recipe = slapos.cookbook:check_port_listening
 path = $${directory:promises}/vnc_promise
 hostname = $${kvm-instance:vnc-ip}
 port = $${kvm-instance:vnc-port}
 
+[kvm-disk-image-corruption-promise]
+# Check that disk image is not corrupted
+recipe = collective.recipe.template
+input = inline:#!/bin/sh
+  $${kvm-instance:qemu-img-path} check $${kvm-instance:disk-path}
+output = $${directory:promises}/kvm-disk-image-corruption
+mode = 700
+
 
 [novnc-instance]
 recipe = slapos.cookbook:novnc

software/kvm/instance.cfg.in

@@ -39,6 +39,7 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
+    raw curl_executable_location ${curl:location}/bin/curl
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}
 import-list = file parts :template-parts-destination
@@ -54,5 +55,4 @@ context =
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
     raw bin_directory ${buildout:bin-directory}
-    raw curl-executable-location ${curl:location}/bin/curl
 mode = 0644

software/kvm/software.cfg

@@ -135,22 +135,22 @@ rdiff-backup = 1.0.5
 slapos.cookbook = 0.84.2
 slapos.recipe.cmmi = 0.2
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.37.2
+slapos.toolbox = 0.37.3
 smmap = 0.8.2
 websockify = 0.5.1
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
 # slapos.core==0.35.1
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 Flask = 0.10.1
 
 # Required by:
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 atomize = 0.1.1
 
 # Required by:
@@ -158,7 +158,7 @@ atomize = 0.1.1
 ecdsa = 0.9
 
 # Required by:
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 feedparser = 5.1.3
 
 # Required by:
@@ -182,7 +182,7 @@ netifaces = 0.8-1
 numpy = 1.7.1
 
 # Required by:
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 paramiko = 1.12.0
 
 # Required by:
@@ -195,7 +195,7 @@ pytz = 2013.7
 
 # Required by:
 # slapos.cookbook==0.84.2
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 slapos.core = 0.35.1
 
 # Required by:
@@ -208,7 +208,7 @@ unittest2 = 0.5.1
 
 # Required by:
 # slapos.cookbook==0.84.2
-# slapos.toolbox==0.37.2
+# slapos.toolbox==0.37.3
 xml-marshaller = 0.9.7
 
 # Required by:

software/kvm/template/kvm-export.sh.in

 #!/bin/bash
 # Create a backup of the disk image of the virtual machine
-QEMU_IMG=${kvm-instance:qemu-img-path}
-SNAPSHOT_NAME=$(date +%s)
-DISK_PATH=${kvm-instance:disk-path}
 BACKUP_PATH=${:backup-disk-path}
 QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
 
@@ -11,12 +8,5 @@ if [ ! -f $DISK_PATH ]; then
   exit 0;
 fi
 
-$QMP_CLIENT ${kvm-instance:socket-path} suspend && \
-$QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
-$QMP_CLIENT ${kvm-instance:socket-path} resume
+$QMP_CLIENT --socket ${kvm-instance:socket-path} --drive-backup $BACKUP_PATH
 
-if [ -f $BACKUP_PATH ]; then
-  rm $BACKUP_PATH
-fi
-$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH && \
-$QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH

stack/resilient/buildout.cfg

 [buildout]
 extends =
-  ../../component/dash/buildout.cfg
+  ../../component/apache/buildout.cfg
+  ../../component/bash/buildout.cfg
   ../../component/dropbear/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
@@ -36,7 +37,7 @@ eggs = collective.recipe.template
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready.cfg.in
 output = ${buildout:directory}/pbsready.cfg
-#md5sum = 46f9d33e642467a72c599c8dc767e6c3
+#md5sum = fcb6d12fc34e7b34bb97786ef4f85f01
 mode = 0644
 
 [pbsready-import]
@@ -45,7 +46,7 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/pbsready-import.cfg.in
 output = ${buildout:directory}/pbsready-import.cfg
-md5sum = cb562bd954b9e809c8748d0f96de4116
+#md5sum = cb562bd954b9e809c8748d0f96de4116
 mode = 0644
 
 [pbsready-export]
@@ -67,7 +68,7 @@ mode = 0644
 [template-replicated]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/template-replicated.cfg.in
-md5sum = 9e236726678d89a5359e1571a91e59e8
+md5sum = b70902e9f247ab710a26cedb2eae7559
 mode = 0644
 destination = ${buildout:directory}/template-replicated.cfg.in
 
@@ -87,6 +88,13 @@ url = ${:_profile_base_location_}/instance-frozen.cfg.in
 md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 output = ${buildout:directory}/instance-frozen.cfg
 
+[resilient-web-takeover-cgi-script-download]
+recipe = slapos.recipe.download
+url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
+#md5sum = 
+mode = 0644
+destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
+
 [versions]
 # Pin Jinja2 to 2.6, as 2.7 breaks current code
 Jinja2 = 2.6

stack/resilient/parameter-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "title": "Resiliency Parameters",
+  "description": "List of possible parameters used in the resilient stack",
+  "type": "object",
+
+  "properties": {
+    "-sla-0-computer_guid": {
+      "title": "Target computer for main instance",
+      "description": "Target computer GUID for main instance.",
+      "type": "string"
+    },
+    "-sla-1-computer_guid": {
+      "title": "Target computer for first clone",
+      "description": "Target computer for first clone and PBS.",
+      "type": "string"
+    },
+    "-sla-2-computer_guid": {
+      "title": "Target computer for second clone",
+      "description": "Target computer for second clone and PBS.",
+      "type": "string"
+    },
+    "resiliency-backup-periodicity": {
+      "title": "Periodicity of backup",
+      "description": "Periodicity of backup, in cron format. Default is every hour.",
+      "type": "string"
+    },
+    "remove-backup-older-than": {
+      "title": "Remove backups older than...",
+      "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
+      "type": "string",
+      "default": "3B"
+    }
+  }
+}
\ No newline at end of file

stack/resilient/pbsready-import.cfg.in

@@ -18,11 +18,17 @@ parts =
   dropbear-server-pbs-authorized-key
   notifier
 
+  resilient-web-takeover-cgi-script
+  resilient-web-takeover-httpd-wrapper
+  resilient-web-takeover-httpd-promise
+
   import-on-notification
   resilient-publish-connection-parameter
 
 [resilient-publish-connection-parameter]
 notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
+takeover-url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
+takeover-password = $${resilient-web-takeover-password:passwd}
 
 # Define port of ssh server. It has to be different from import so that it
 # supports export/import using same IP (slaprunner, slapos-in-partition,
@@ -37,3 +43,67 @@ port = 22220
 recipe = slapos.cookbook:notifier.callback
 on-notification-id = $${slap-parameter:on-notification}
 callback = $${importer:wrapper}
+
+###########
+# Deploy a webserver allowing to do takeover from a web browser.
+###########
+[resilient-web-takeover-password]
+recipe = slapos.cookbook:generate.password
+storage-path = $${directory:srv}/passwd
+bytes = 8
+
+[resilient-web-takeover-cgi-script]
+recipe = collective.recipe.template
+input = ${resilient-web-takeover-cgi-script-download:destination}
+output = $${directory:cgi-bin}/web-takeover.cgi
+password = $${resilient-web-takeover-password:passwd}
+mode = 700
+
+# XXX could it be something lighter?
+# XXX Add SSL
+[resilient-web-takeover-httpd-configuration-file]
+recipe = collective.recipe.template
+input = inline:
+  PidFile "$${:pid-file}"
+  Listen [$${:listening-ip}]:$${:listening-port}
+  ServerAdmin someone@email
+  DocumentRoot "$${:document-root}"
+  ErrorLog "$${:error-log}"
+  LoadModule unixd_module modules/mod_unixd.so
+  LoadModule access_compat_module modules/mod_access_compat.so
+  LoadModule authz_core_module modules/mod_authz_core.so
+  LoadModule authz_host_module modules/mod_authz_host.so
+  LoadModule mime_module modules/mod_mime.so
+  LoadModule cgid_module modules/mod_cgid.so
+  LoadModule dir_module modules/mod_dir.so
+  ScriptSock $${:cgid-pid-file}
+  <Directory $${:document-root}>
+    # XXX: security????
+    Options +ExecCGI
+    AddHandler cgi-script .cgi
+    DirectoryIndex web-takeover.cgi
+  </Directory>
+output = $${directory:etc}/resilient-web-takeover-httpd.conf
+# md5sum =
+listening-ip = $${slap-network-information:global-ipv6}
+# XXX: randomize-me
+listening-port = 9263
+htdocs = $${directory:cgi-bin}
+pid-file = $${directory:run}/resilient-web-takeover-httpd.pid
+cgid-pid-file = $${directory:run}/resilient-web-takeover-httpd-cgid.pid
+document-root = $${directory:cgi-bin}
+error-log = $${directory:log}/resilient-web-takeover-httpd-error-log
+
+[resilient-web-takeover-httpd-wrapper]
+recipe = slapos.cookbook:wrapper
+apache-executable = ${apache:location}/bin/httpd
+command-line = $${:apache-executable} -f $${resilient-web-takeover-httpd-configuration-file:output} -DFOREGROUND
+wrapper-path = $${basedirectory:services}/resilient-web-takeover-httpd
+
+[resilient-web-takeover-httpd-promise]
+recipe = slapos.cookbook:check_url_available
+path = $${basedirectory:promises}/resilient-web-takeover-httpd
+url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
+dash_path = ${dash:location}/bin/dash
+curl_path = ${curl:location}/bin/curl
+

stack/resilient/pbsready.cfg.in

@@ -50,6 +50,7 @@ crontabs = $${rootdirectory:etc}/crontabs
 cronstamps = $${rootdirectory:etc}/cronstamps
 logrotate-entries = $${rootdirectory:etc}/logrotate.d
 logrotate-backup = $${basedirectory:backup}/logrotate
+cgi-bin = $${rootdirectory:srv}/cgi-bin
 
 #----------------
 #--
@@ -230,9 +231,9 @@ wrapper = $${basedirectory:services}/sshd
 [resilient-sshkeys-dropbear-promise]
 # Check that public key file exists and is not empty
 recipe = collective.recipe.template
-input = inline:#!${dash:location}/bin/dash
+input = inline:#!${bash:location}/bin/bash
   PUBLIC_KEY_CONTENT="$${sshkeys-dropbear:public-key-value}"
-  if [ ! -n "$PUBLIC_KEY_CONTENT" ]; then
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
 output = $${basedirectory:promises}/public-key-existence

stack/resilient/resilient-web-takeover-cgi-script.py.in

+#!${buildout:executable}
+import cgi
+import cgitb
+import os
+import subprocess
+import sys
+cgitb.enable()
+
+print "Content-Type: text/html"
+print    
+
+form = cgi.FieldStorage()
+if "password" not in form:
+  print """<html>
+<body>
+  <h1>This is takeover web interface.</h1>
+  <p>Calling takeover will stop and freeze the current main instance, and make this clone instance the new main instance, replacing the old one.</p>
+  <p><b>Warning: submit the form only if you understand what you are doing.</b></p>
+  <p>Note: the password asked here can be found within the parameters of your SlapOS instance page.</p>
+  <form action="/">
+    Password: <input type="text" name="password">
+    <input type="submit" value="Take over" style="background: red;">
+  </form>
+</body>
+</html>"""
+  sys.exit(0)
+
+if form['password'].value != '${:password}':
+  print "<H1>Error</H1>"
+  print "Password is invalid."
+  sys.exit(1)
+
+# XXX hardcoded location
+result = subprocess.check_output([os.path.expanduser("~/bin/takeover")], stderr=subprocess.STDOUT)
+print 'Success.'
+print '<pre>%s</pre>' % result

stack/resilient/template-replicated.cfg.in

@@ -131,7 +131,7 @@ recipe = collective.recipe.template
 # XXX: don't use system executable
 input = inline:#!/bin/sh
   PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})"
-  if [[ ! -n "$PUBLIC_KEY_CONTENT" -o "$PUBLIC_KEY_CONTENT" == None ]]; then
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
 output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-key
@@ -149,7 +149,7 @@ recipe = collective.recipe.template
 # XXX: don't use system executable
 input = inline:#!/bin/sh
   PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})"
-  if [ ! -n "$PUBLIC_KEY_CONTENT" -a  "$PUBLIC_KEY_CONTENT" == None ]; then
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
 output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key
@@ -207,7 +207,7 @@ recipe = collective.recipe.template
 # XXX: don't use system executable
 input = inline:#!/bin/sh
   PUBLIC_KEY_CONTENT="${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}:connection-ssh-key})"
-  if [ ! -n "$PUBLIC_KEY_CONTENT" -a  "$PUBLIC_KEY_CONTENT" == None ]; then
+  if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
     exit 1
   fi
 output = ${resilient-directory:promise}/resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key