Merge branch 'kvm'

component/qemu-kvm/buildout.cfg

@@ -56,7 +56,7 @@ configure-options =
 
 [debian-amd64-netinst.iso]
 # Download the installer of Debian 7 (Wheezy)
-recipe = slapos.recipe.download
+recipe = hexagonit.recipe.download
 url = http://cdimage.debian.org/debian-cd/7.1.0/amd64/iso-cd/debian-7.1.0-amd64-netinst.iso
 filename = ${:_buildout_section_name_}
 md5sum = 80f498a1f9daa76bc911ae13692e4495

slapos/recipe/addresiliency/takeover.py

@@ -3,6 +3,7 @@ import logging
 import time
 
 import slapos
+from slapos.slap.slap import NotFoundError
 
 log = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
@@ -58,8 +59,15 @@ def takeover(server_url, key_file, cert_file, computer_guid,
   log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
 
   # update name (and later, software type) for the partition that will take over
+  while True:
+    time.sleep(10)
+    try:
+      cp_winner.rename(new_name=cp_exporter_ref)
+      break
+    except NotFoundError:
+      log.warning('Impossible to rename. Retrying in a few seconds...')
+  log.debug('Renamed.')
 
-  cp_winner.rename(new_name=cp_exporter_ref)
   cp_winner.bang(message='partitions have been renamed!')
   # Note: Root instance will reconfigure itself the winning instance (software_type
   # and parameters.)

software/kvm/common.cfg

@@ -79,7 +79,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-#md5sum = 461187060d12a6bea4d5b9699687c0e9
+#md5sum = bdd0495ef729e7272ec9c97aca919c09
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,11 +98,11 @@ md5sum = 6753004b582c0470bd028253ce1964ad
 download-only = true
 
 [template-kvm-resilient-test]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.in
-#md5sum = 9dead7572072307b6c38cdf47052225b
-output = ${buildout:directory}/template-kvm-resilient-test.cfg
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
+md5sum = 027d68d9decbc6aec59365fa723975d7
 mode = 0644
+download-only = true
 
 [template-kvm-import]
 recipe = slapos.recipe.template
@@ -115,7 +115,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-import.sh.in
 filename = kvm-import.sh.in
-md5sum = e03ed049cddd8d157228b09e1ebc071a
+md5sum = a731372420dc59c0b5ba7bc5f39a14ad
 download-only = true
 mode = 0755
 
@@ -130,7 +130,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-export.sh.in
 filename = kvm-export.sh.in
-md5sum = 08cd8da2221f09095b14e35e6acd2a56
+md5sum = 3e878b3343c76f0d6950986fffcb6a8c
 download-only = true
 mode = 0755
 

software/kvm/development.cfg

@@ -23,7 +23,7 @@ git-executable = ${git:location}/bin/git
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = http://git.erp5.org/repos/slapos.toolbox.git
-branch = master
+branch = kvmresiliency
 git-executable = ${git:location}/bin/git
 
 [erp5.util-repository]

software/kvm/instance-kvm-resilient-test.cfg.in → software/kvm/instance-kvm-resilient-test.cfg.jinja2

 [buildout]
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
 offline = true
 
 parts =
@@ -10,45 +10,47 @@ parts =
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
-srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
-tmp = $${buildout:directory}/tmp/
-
-services = $${:etc}/service/
-scripts = $${:etc}/run/
+home = ${buildout:directory}
+etc = ${:home}/etc/
+var = ${:home}/var/
+srv = ${:home}/srv/
+bin = ${:home}/bin/
+tmp = ${:home}/tmp/
+log = ${:var}/log/
+services = ${:etc}/service/
+scripts = ${:etc}/run/
 
 [deploy-resiliency-test]
 recipe = slapos.cookbook:wrapper
-#log-path = {directory:log}
-wrapper-path = $${directory:bin}/runKVMResiliencyTestSuite
-#output = $${binary-wrap-launcher:binary-path}
-#site-id = "{site_id}"
-# Suppose that there is a user zope family in the configuration
-#erp5-url = $${erp5-cluster:connection-family-user}{:site-id}
-#parameters-extra = $*
-command-line = ${buildout:bin-directory}/runKVMResiliencyTestSuite --server-url=$${slap-connection:server-url} --key-file=$${slap-connection:key-file} --cert-file=$${slap-connection:cert-file} --computer-id=$${slap-connection:computer-id} --partition-id=$${slap-connection:partition-id} --software=$${slap-connection:software-release-url} --namebase=kvm --kvm-rootinstance-name='$${request-resilient-kvm:name}'
-#return = url
+wrapper-path = ${directory:scripts}/runKVMResiliencyTestSuite
+testnode-parameters = --test-result-path={{ slapparameter_dict.get('test-result-path') }} --revision={{ slapparameter_dict.get('test-suite-revision') }} --node-title={{ slapparameter_dict.get('scalability-launcher-title') }} --test-suite={{ slapparameter_dict.get('test-suite') }} --test-suite-master-url={{ slapparameter_dict.get('test-suite-master-url') }} --log-path=${directory:log}
+kvm-test-parameters = server_url=${slap-connection:server-url} key_file=${slap-connection:key-file} cert_file=${slap-connection:cert-file} computer_id=${slap-connection:computer-id} partition_id=${slap-connection:partition-id} software=${slap-connection:software-release-url} namebase=kvm kvm_rootinstance_name='${request-resilient-kvm:name}'
+
+command-line = {{ bin_directory }}/runResiliencyTest ${:testnode-parameters} ${:kvm-test-parameters}
+
+
+[deploy-standalone-resiliency-test]
+# Used to manually run the KVM test if we don't have a running testnode.
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/runStandaloneKVMResiliencyTestSuite
+command-line = {{ bin_directory }}/runStandaloneResiliencyTest --test-suite-title=kvm ${deploy-resiliency-test:kvm-test-parameters}
 
 [request-resilient-kvm]
 <= slap-connection
 recipe = slapos.cookbook:request
-software-url = $${slap-connection:software-release-url}
+software-url = ${slap-connection:software-release-url}
 software-type = kvm-resilient
 name = Resilient KVM (Root Instance)
 config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity
-config-virtual-hard-drive-url = $${slap-parameter:virtual-hard-drive-url}
-config-virtual-hard-drive-md5sum = $${slap-parameter:virtual-hard-drive-md5sum}
+config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
+config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
 config-resiliency-backup-periodicity = */5
 # We don't use url parameter, but we want it to be there to make sure root instance is ready.
 return = url
 # XXX What to do?
 #sla = instance_guid
-#sla-instance_guid = $${slap-parameter:frontend-instance-guid}
+#sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 
 [slap-parameter]
 virtual-hard-drive-url = https://softinst43236.host.vifib.net/data/public/8e2138.php?dl=true
 virtual-hard-drive-md5sum = de0f10c7c6538e9928879332afd9be7a
-# XXX: what to do about pbs-kvm1-computer-guid and kvm1-computer-guid ?
-

software/kvm/instance.cfg.in

@@ -17,7 +17,7 @@ kvm-import = ${template-kvm-import:output}
 kvm-export = ${template-kvm-export:output}
 
 # Used for the test of resiliency. The system wants a "test" software_type.
-test = ${template-kvm-resilient-test:output}
+test = $${dynamic-template-kvm-resilient-test:rendered}
 
 frozen = ${instance-frozen:output}
 pull-backup = ${template-pull-backup:output}
@@ -34,13 +34,24 @@ cert = $${slap-connection:cert-file}
 recipe = slapos.recipe.template:jinja2
 template = ${template-kvm-resilient:location}/instance-kvm-resilient.cfg.jinja2
 rendered = $${buildout:directory}/template-kvm-resilient.cfg
-context = key buildout buildout:bin-directory
-          key develop_eggs_directory buildout:develop-eggs-directory
-          key eggs_directory buildout:eggs-directory
-          key slapparameter_dict slap-configuration:configuration
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}
 import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 mode = 0644
 
+[dynamic-template-kvm-resilient-test]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-resilient-test:location}/instance-kvm-resilient-test.cfg.jinja2
+rendered = $${buildout:directory}/template-kvm-resilient-test.cfg
+bin-directory = ${buildout:bin-directory}
+context =
+    key bin_directory dynamic-template-kvm-resilient-test:bin-directory
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    key slapparameter_dict slap-configuration:configuration
+mode = 0644

software/kvm/template/kvm-export.sh.in

@@ -14,5 +14,5 @@ $QEMU_IMG snapshot -c $SNAPSHOT_NAME $DISK_PATH
 if [ -f $BACKUP_PATH ]; then
   rm $BACKUP_PATH
 fi
-$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH
+$QEMU_IMG convert -f qcow2 -O qcow2 -s $SNAPSHOT_NAME $DISK_PATH $BACKUP_PATH && \
 $QEMU_IMG snapshot -d $SNAPSHOT_NAME $DISK_PATH

software/kvm/template/kvm-import.sh.in

@@ -3,5 +3,5 @@ DISK_PATH=${:disk-path}
 BACKUP_PATH=${:backup-disk-path}
 
 # TODO: Use rdiff
-rm $DISK_PATH
+rm $DISK_PATH && \
 cp $BACKUP_PATH $DISK_PATH

stack/resilient/buildout.cfg

@@ -84,4 +84,6 @@ output = ${buildout:directory}/instance-frozen.cfg
 [versions]
 # Pin Jinja2 to 2.6, as 2.7 breaks current code
 Jinja2 = 2.6
+# ... And newer s.r.template requires Jinja2 >= 2.7
+slapos.recipe.template = 2.4.2
 

stack/slapos.cfg

@@ -73,11 +73,11 @@ eggs =
 
 [versions]
 # Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-010
+zc.buildout = 1.6.0-dev-SlapOS-012
 # zc.recipe.egg 2.x is for Buildout 2
 zc.recipe.egg = 1.3.2
 # Use own version of h.r.download to be able to open xz-like archives
-hexagonit.recipe.download = 1.6nxd002
+hexagonit.recipe.download = 1.7nxd002
 # Use pinned version of setuptools. Other versions work, but changing
 # version makes buildout recompile everything. Developers' nightmare.
 setuptools = 0.9.8