Update Release Candidate

software/beremiz-runtime/software.cfg

@@ -40,7 +40,7 @@ post-install =
 [beremiz-source]
 recipe = slapos.recipe.build:gitclone
 repository = https://github.com/beremiz/beremiz.git
-revision = 2d061a6
+revision = 9eb53af
 git-executable = ${git:location}/bin/git
 
 [nevow-py3-source]

software/grafana/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = 32c772c593d2c3c38c26186b91b78cf8
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = b4330fbe0c9c3631f4f477c06d3460b3
+md5sum = 86e379ac6ba789d65369ca1e0d4dfa3f
 
 [instance-agent]
 filename = instance-agent.cfg.in
-md5sum = 9fc368a3ac16bee297a4f2ad87b3bbb4
+md5sum = 89fd1b010351cea9968360054e0c4aa9
 
 
 [influxdb-config-file]

software/grafana/instance-agent.cfg.in

@@ -214,13 +214,13 @@ init =
                 },
                 {
                   "query": """
-                    select 'message' as cmf_activity_queue, count(*) as failed_message_count
-                      from message where processing_node between -2 and -10
-                    union all select 'message_queue' as cmf_activity_queue, count(*) as failed_message_count
-                      from message_queue where processing_node between -2 and -10
+                    select 'message' as cmf_activity_queue, method_id, count(*) as failed_message_count
+                      from message where processing_node between -10 and -2 group by method_id
+                    union all select 'message_queue' as cmf_activity_queue, method_id, count(*) as failed_message_count
+                      from message_queue where processing_node between -10 and -2 group by method_id
                   """,
                   "field_columns_include": ["failed_message_count"],
-                  "tag_columns_include": ["cmf_activity_queue"],
+                  "tag_columns_include": ["cmf_activity_queue", "method_id"],
                 },
                 # TODO: these queries are slow and maybe not correct
                 # {
@@ -263,7 +263,8 @@ init =
     urls = application.get("urls", [])
     if urls:
       inputs["http_response"].append({
-        "interval": "5m",
+        "interval": "2m",
+        "response_timeout": "1m",
         "urls": urls,
         "tags": {"app": application["name"]},
       })

software/grafana/instance-default.cfg.in

@@ -637,7 +637,7 @@ init =
       "type": "system",
       "partitions": [
         {
-          "name": "grafana",
+          "name": "influxdb",
           "static-tags": {
             "partition_reference": slap_connection['partition-id'],
           },
@@ -646,7 +646,7 @@ init =
           ]
         },
         {
-          "name": "influxdb",
+          "name": "grafana",
           "static-tags": {
             "partition_reference": slap_connection['partition-id'],
           },

software/kvm/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 9ae66fb63a3cdd8072582622aa1bb36c
+md5sum = bf0a70140e4775efaacdd5fe03ac3b62
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = bd3a7229e4fdfa9372ee61b6054acf78
+md5sum = 20554e40f9fe666839ef86ff8216cebf
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -31,19 +31,19 @@ md5sum = 839fc16c112d3b87e2dbd2e382e326de
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
-md5sum = a463a5e3cd2287d275d6943c2a11b7e4
+md5sum = 0829d08337a09b8ef017e71fbb9384fb
 
 [template-kvm-import-script]
 filename = template/kvm-import.sh.jinja2
-md5sum = 013725987114c82ca3fd11097d0a7f9f
+md5sum = a6e6abb3b17fae37b84233d63014b885
 
 [template-kvm-export]
 filename = instance-kvm-export.cfg.jinja2
-md5sum = 34d1b7cc8ca62bfdfce759a1dfbbaccd
+md5sum = a02f0694dcb944c18d99f7f79afa2384
 
 [template-kvm-export-script]
 filename = template/kvm-export.sh.jinja2
-md5sum = 64aa1ce8785f6b94aabd787fa3443082
+md5sum = a1da7809d547b4c61e7c6337bf9f8b8a
 
 [template-nginx]
 filename = template/nginx_conf.in
@@ -96,3 +96,7 @@ md5sum = e9d40162ba77472775256637a2617d14
 [boot-image-select-source-config]
 filename = template/boot-image-select-source-config.json.in
 md5sum = d53afe719e2cbfc2480277af340f8429
+
+[check-backup-directory.sh]
+filename = template/check-backup-directory.sh
+md5sum = e569494a941e1d585c2e0bbf070cf1c9

software/kvm/instance-kvm-export.cfg.jinja2

@@ -6,6 +6,7 @@ extends =
 parts +=
   cron-entry-backup
   ${instance-kvm-parts:parts}
+  check-backup-directory-promise
 
 [slap-parameter]
 {% for k, v in slapparameter_dict.items() -%}
@@ -16,13 +17,10 @@ parts +=
 {%-  endif %}
 {% endfor -%}
 
-# Create the exporter executable, which is a simple shell script
-[exporter]
-recipe = slapos.recipe.template:jinja2
-url = {{ template_kvm_export }}
-output = ${directory:bin}/${slap-parameter:namebase}-exporter
-# Resilient stack wants a "wrapper" parameter
-wrapper = ${:output}
+[directory]
+tmp = ${buildout:directory}/tmp
+
+[disk]
 {%- set disk_type = slapparameter_dict.get('disk-type', 'virtio') %}
 {%- if disk_type == "virtio" %}
 device = virtio0
@@ -32,12 +30,19 @@ device = ide0-hd0
 {%- else %}
 # unsupported disk-type {{ disk_type }}
 {%- endif %}
+
+[exporter]
+recipe = slapos.recipe.template:jinja2
+url = {{ template_kvm_export }}
+output = ${directory:bin}/${slap-parameter:namebase}-exporter
+# Resilient stack wants a "wrapper" parameter
+wrapper = ${:output}
 context =
     section directory directory
     section buildout buildout
     key socket_path kvm-instance:socket-path
-    key device :device
-    raw gzip_binary {{ gzip_binary }}
+    section disk disk
+    raw qmpbackup {{ qmpbackup }}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]
@@ -56,3 +61,10 @@ username = {{ slapparameter_dict['monitor-username'] }}
 password = {{ slapparameter_dict['monitor-password'] }}
 {% endif -%}
 
+[check-backup-directory-promise]
+# Check that disk image is not corrupted
+<= monitor-promise-base
+promise = check_command_execute
+name = check-backup-directory.py
+config-command = {{ check_backup_directory }} ${directory:backup}/${disk:device} ${directory:tmp}
+failure_amount = 5

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -48,6 +48,7 @@ bin = ${buildout:directory}/bin
 srv = ${buildout:directory}/srv
 var = ${buildout:directory}/var
 log = ${:var}/log
+tmp = ${buildout:directory}/tmp
 scripts = ${:etc}/run
 services = ${:etc}/service
 novnc-conf = ${:etc}/novnc
@@ -65,10 +66,7 @@ output = ${directory:bin}/${slap-parameter:namebase}-importer
 wrapper = ${:output}
 context =
     section directory directory
-    raw zcat_binary {{ zcat_binary }}
-    raw gzip_binary {{ gzip_binary }}
-
-backup-disk-path = ${directory:backup}/virtual.qcow2
+    raw qmprestore {{ qmprestore }}
 
 [kvm-disk-image-corruption-bin]
 recipe = collective.recipe.template

software/kvm/instance-kvm.cfg.jinja2

@@ -715,7 +715,6 @@ cron-entries = ${directory:cron-entries}
 crontabs = ${directory:crontabs}
 cronstamps = ${directory:cronstamps}
 catcher = ${cron-simplelogger:wrapper}
-binary = ${directory:bin}/crond_raw
 
 [cron-service]
 recipe = slapos.cookbook:wrapper

software/kvm/instance.cfg.in

@@ -122,6 +122,15 @@ template-replicated-destination = ${template-replicated:target}
 import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 
+[qmpbackup-binary]
+recipe = slapos.cookbook:wrapper
+environment =
+  PATH=${qemu:location}/bin
+wrapper-path =
+  $${buildout:bin-directory}/qmpbackup
+command-line =
+  ${buildout:bin-directory}/qmpbackup
+
 [dynamic-template-kvm-export]
 recipe = slapos.recipe.template:jinja2
 url = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
@@ -133,8 +142,18 @@ context =
     raw kvm_template $${dynamic-template-kvm:output}
     raw template_kvm_export ${template-kvm-export-script:target}
     key pbsready_export_template template-pbsready-export:output
-    raw gzip_binary ${gzip:location}/bin/gzip
     key slapparameter_dict slap-configuration:configuration
+    key qmpbackup qmpbackup-binary:wrapper-path
+    raw check_backup_directory ${check-backup-directory.sh:output}
+
+[qmprestore-binary]
+recipe = slapos.cookbook:wrapper
+environment =
+  PATH=${qemu:location}/bin
+wrapper-path =
+  $${buildout:bin-directory}/qmprestore
+command-line =
+  ${buildout:bin-directory}/qmprestore
 
 [dynamic-template-kvm-import]
 recipe = slapos.recipe.template:jinja2
@@ -148,5 +167,4 @@ context =
     raw template_kvm_import ${template-kvm-import-script:target}
     key pbsready_import_template template-pbsready-import:output
     key slapparameter_dict slap-configuration:configuration
-    raw zcat_binary ${gzip:location}/bin/zcat
-    raw gzip_binary ${gzip:location}/bin/gzip
+    key qmprestore qmprestore-binary:wrapper-path

software/kvm/software.cfg

@@ -10,7 +10,6 @@ extends =
   ../../component/netcat/buildout.cfg
   ../../component/nginx/buildout.cfg
   ../../component/pycurl/buildout.cfg
-  ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/resilient/buildout.cfg
   buildout.hash.cfg
@@ -19,6 +18,7 @@ extends =
 # to avoid versioning issues
 common-parts =
   template
+  qmpbackup
 
 # XXX: we have to manually add this for resilience
   pbs-recipe-egg
@@ -32,6 +32,18 @@ parts = ${:common-parts}
 #             In qemu builtin vnc server, and make it available only for localhost
 #             so that only novnc can listen to it.
 
+[qmpbackup]
+recipe = zc.recipe.egg
+eggs =
+  qemu.qmp
+  qmpbackup
+find-links +=
+  https://github.com/abbbi/qmpbackup/releases/download/v0.37/qmpbackup-0.37.tar.gz
+
+[versions]
+qemu.qmp = 0.0.3:whl
+qmpbackup = 0.37
+
 [python-with-eggs]
 recipe = zc.recipe.egg
 interpreter = ${:_buildout_section_name_}
@@ -129,3 +141,7 @@ context =
 
 [boot-image-select-source-config]
 <= download-base
+
+[check-backup-directory.sh]
+<= template-base
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/check-backup-directory.sh

software/kvm/template/check-backup-directory.sh

+#!/bin/bash
+
+directory=$1
+tmp=$2
+
+# support a case of not ready yet directory
+if [ ! -d $directory ] ; then
+  exit 0
+fi
+
+tmpfile=$(mktemp -p $tmp)
+
+trap "rm -fr $tmpfile" EXIT TERM INT
+
+find $directory -type f -name 'FULL*qcow2' -printf '%f\n' > $tmpfile
+full_amount=$(wc -l $tmpfile | cut -d ' ' -f 1)
+if [ $full_amount -gt 1 ]; then
+  echo "Too many FULL backups"
+  cat $tmpfile
+  exit 1
+fi
+
+find $directory -type f -name 'INC*qcow2' -printf '%f\n' > $tmpfile
+if [ $(wc -l $tmpfile | cut -d ' ' -f 1) -gt 0 ] && [ $full_amount -eq 0 ] ; then
+  echo "INC present but no FULL backup"
+  cat $tmpfile
+  exit 1
+fi
+
+find $directory -type f -name '*.partial' -printf '%f\n' > $tmpfile
+if [ $(wc -l $tmpfile | cut -d ' ' -f 1) -ne 0 ]; then
+  echo "Partial file present"
+  cat $tmpfile
+  exit 1
+fi
+
+exit 0

software/kvm/template/kvm-export.sh.jinja2

@@ -6,25 +6,42 @@ set -e
 LC_ALL=C
 export LC_ALL
 BACKUP_DIR={{ directory['backup'] }}
-BACKUP_FILE=virtual.qcow2
 
-QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
-
-$QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE {{ device }}
-
-# Due to the way qmp works, the VM file cannot be compressed on the fly.
-# Although the compression step is optional, the importer uses the .gz file
-# if present. So, remove it if you are disabling the compression.
-
-# The downside of compression, here, is the temporary usage of more disk space
-# in the exporter node. The goal is to minimize disk usage on the PBS server.
+log=$(mktemp --tmpdir={{ directory['tmp'] }})
+trap "rm -f $log" EXIT TERM INT
+
+set +e
+qmpbackup="{{ qmpbackup }} --socket {{ socket_path }} backup --compress --target $BACKUP_DIR --include {{ disk['device'] }}"
+$qmpbackup --level auto > $log
+RESULT=$?
+cat $log
+if [ $RESULT -ne 0 ] ; then
+  # recover from unfinished previous backup
+  if egrep -q 'Partial backup found in.*{{ disk['device']}}.*possible broken backup chain. Execute new full backup' $log ; then
+    find $BACKUP_DIR/{{ disk['device'] }} -name '*.partial' -delete
+    $qmpbackup --level auto || exit $?
+    echo "Recovered from partial backup by removing partial"
+  elif egrep -q 'Incremental backup requested but no active bitmap has been found' $log ; then
+    find $BACKUP_DIR/{{ disk['device'] }} -name '*.qcow2' -delete
+    $qmpbackup --level full || exit $?
+    echo "Post take-over cleanup"
+  else
+    exit $RESULT
+  fi
+fi
+set -e
 
-# If you want to compress the file in-place:
-# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
-# but the importer script would have to be adapted.
+# as new style backup went fine delete potential old style backup
+rm -f $BACKUP_DIR/virtual.qcow2{,.gz}
 
-echo "Compressing backup..."
-{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
+# cleanup the backup directory from too old backups, especially important after take-over
+recent_full=$(find $BACKUP_DIR -type f -name 'FULL-*.qcow2' -exec ls -t1 {} + | head -n1)
+if [ x"$recent_full" != x"" ] ; then
+  for f in $(find $BACKUP_DIR -type f -name '*qcow2' \! -newer $recent_full) ; do
+    if [ "$f" != "$recent_full" ] ; then
+      rm -vf $f
+    fi
+  done
+fi
 
 cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
-

software/kvm/template/kvm-import.sh.jinja2

@@ -4,7 +4,6 @@ set -e
 
 VM_DIR={{ directory['srv'] }}
 BACKUP_DIR={{ directory['backup'] }}
-VM_FILE=virtual.qcow2
 LC_ALL=C
 export LC_ALL
 umask 077
@@ -18,11 +17,19 @@ write_backup_proof () {
 # For now we just make the diff before 
 write_backup_proof
 
-if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
-    {{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
-    {{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
+tmpfile=$(mktemp --tmpdir={{ directory['tmp'] }})
+# use temporary space inside of the partition, as it can be quite big
+# assure the temporary directory is cleaned up
+trap "rm -rf $tmpdir $tmpfile" EXIT TERM INT
+
+if [ -d $BACKUP_DIR/ide0-hd0 ] ; then
+  disk_type="ide0-hd0"
+elif [ -d $BACKUP_DIR/virtio0 ] ; then
+  disk_type=virtio0
 else
-    rm $VM_DIR/$VM_FILE
-    cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+  echo "Unsupported disk type"
+  exit 1
 fi
 
+rm -f $VM_DIR/virtual.qcow2
+{{ qmprestore }} merge --dir $BACKUP_DIR/$disk_type --target $VM_DIR/virtual.qcow2

software/kvm/test/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template-test]
+filename = test-instance.cfg.in
+md5sum = 466c7324a44e0cd8ae688699aab62d5f
+
+[template-kvm-export-mock]
+filename = test-instance-kvm-export-mock.cfg.jinja2
+md5sum = acf0b9a916e1411f184eb4f94c7c2abb

software/kvm/test/test-instance-kvm-export-mock.cfg.jinja2

+[buildout]
+extends = {{ template_kvm_export }}
+
+parts +=
+  cron-entry-environment
+
+[directory]
+cron-d-mock = ${:var}/cron-d-mock
+
+[cron-entry-environment]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = environment
+frequency = * * * * *
+command = ${buildout:executable} -c 'import os ; import json ; print(json.dumps(dict(os.environ)))' > ${directory:var}/cron-environment.json
+
+[cron-entry-backup]
+recipe = slapos.recipe.template:jinja2
+inline =
+  ${:command}
+output = ${directory:cron-d-mock}/${:name}

software/kvm/test/test-instance.cfg.in

+[buildout]
+extends = ${template:output}
+
+[dynamic-template-kvm-export-mock]
+recipe = slapos.recipe.template:jinja2
+url = ${template-kvm-export-mock:location}/${template-kvm-export-mock:filename}
+output = $${buildout:directory}/template-kvm-export-mock.cfg
+context =
+  key template_kvm_export dynamic-template-kvm-export:output
+
+[switch_softwaretype]
+kvm-export = dynamic-template-kvm-export-mock:output

software/kvm/test/test-software.cfg

+[buildout]
+extends =
+  ../software.cfg
+  buildout.hash.cfg
+
+parts +=
+  template-test
+
+[template]
+output = ${buildout:directory}/template-original.cfg
+
+[template-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+
+[template-kvm-export-mock]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}

software/kvm/test/test.py

@@ -59,8 +59,7 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
 
 if has_kvm:
   setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    os.path.abspath(
-      os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+    os.path.join(os.path.dirname(__file__), 'test-software.cfg'))
   # XXX Keep using slapos node instance --all, because of missing promises
   InstanceTestCase.slap._force_slapos_node_instance_all = True
 else:
@@ -217,6 +216,35 @@ class KvmMixin:
         '--run-only', promise])
     )
 
+  @classmethod
+  def getPartitionId(cls, instance_type):
+    software_url = cls.getSoftwareURL()
+    for computer_partition in cls.slap.computer.getComputerPartitionList():
+      try:
+        partition_url = computer_partition.\
+          getSoftwareRelease()._software_release
+        partition_type = computer_partition.getType()
+      except (
+        slapos.slap.exception.NotFoundError,
+        slapos.slap.exception.ResourceNotReady
+      ):
+        partition_url = 'NA'
+        partition_type = 'NA'
+      if partition_url == software_url and partition_type == instance_type:
+        return computer_partition.getId()
+    raise Exception("Partition type %s not found" % instance_type)
+
+  @classmethod
+  def getPartitionPath(cls, instance_type='kvm-export', *paths):
+    return os.path.join(
+      cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
+
+  @classmethod
+  def getBackupPartitionPath(cls, *paths):
+    return cls.getPartitionPath(
+      'kvm-export', 'srv', 'backup', 'kvm',
+      cls.disk_type_backup_mapping[cls.disk_type], *paths)
+
   def getConnectionParameterDictJson(self):
     return json.loads(
       self.computer_partition.getConnectionParameterDict()['_'])
@@ -744,6 +772,190 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, KVMTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
+class CronMixin(object):
+  def setUp(self):
+    super().setUp()
+    # wait until all mocked partition have var/cron-environment.json
+    for i in range(20):
+      missing_list = []
+      for mocked in glob.glob(os.path.join(
+        self.slap._instance_root, '*', 'var', 'cron-d-mock')):
+        cron_environment = os.path.join(
+          '/', *mocked.split('/')[:-2], 'var', 'cron-environment.json')
+        if not os.path.exists(cron_environment):
+          missing_list.append(cron_environment)
+      if len(missing_list) == 0:
+        break
+      time.sleep(1)
+    else:
+      raise ValueError('Missing cron environment', ' '.join(missing_list))
+
+  @classmethod
+  def executeCronDMockJob(cls, instance_type, cron):
+    jobpath = cls.getPartitionPath(
+      instance_type, 'var', 'cron-d-mock', cron)
+    with open(
+      cls.getPartitionPath(
+          'kvm-export', 'var', 'cron-environment.json')) as fh:
+      cron_environment = json.load(fh)
+    job_list = []
+    with open(jobpath, 'r') as fh:
+      for job in fh.readlines():
+        job = job.strip()
+        job_list.append(job)
+    job_list_output = []
+    for job in job_list:
+      job_list_output.append(subprocess.run(
+        job, env=cron_environment, shell=True, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT))
+    return job_list_output
+
+
+class TestInstanceResilientBackupMixin(CronMixin, KvmMixin):
+  __partition_reference__ = 'irb'
+  instance_max_retry = 20
+
+  disk_type = 'virtio'
+  disk_type_backup_mapping = {
+    'virtio': 'virtio0',
+    'ide': 'ide0-hd0',
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    parameter_dict = {}
+    if cls.disk_type != 'virtio':
+      parameter_dict['disk-type'] = cls.disk_type
+    return parameter_dict
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'kvm-resilient'
+
+  def setUp(self):
+    super().setUp()
+    importer_partition = glob.glob(os.path.join(
+      self.slap.instance_directory, '*', 'template-kvm-import.cfg'))
+    self.assertEqual(1, len(importer_partition))
+    self.importer_partition = os.path.dirname(importer_partition[0])
+
+  def call_exporter(self):
+    result = self.executeCronDMockJob('kvm-export', 'backup')
+    self.assertEqual(len(result), 1)
+    self.assertEqual(
+      0,
+      result[0].returncode,
+      result[0].stdout.decode('utf-8'))
+    return result[0].stdout.decode('utf-8')
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupImporter(
+  TestInstanceResilientBackupMixin, KVMTestCase):
+  def test(self):
+    equeue_file = os.path.join(
+      self.importer_partition, 'var', 'log', 'equeue.log')
+    destination_qcow2 = os.path.join(
+      self.importer_partition, 'srv', 'virtual.qcow2')
+    destination_backup = os.path.join(
+      self.importer_partition, 'srv', 'backup', 'kvm',
+      self.disk_type_backup_mapping[self.disk_type])
+    # sanity check - no export/import happened yet
+    self.assertFalse(os.path.exists(self.getBackupPartitionPath()))
+    self.call_exporter()
+
+    def awaitBackup(equeue_file):
+      for f in range(30):
+        with open(equeue_file, 'r') as fh:
+          equeue_log = fh.read()
+          if 'finished successfully' in equeue_log:
+            break
+        time.sleep(1)
+      else:
+        self.fail('Backup not finished: %s' % (equeue_log))
+      return equeue_log
+    equeue_log = awaitBackup(equeue_file)
+    self.assertNotIn('qemu-img rebase', equeue_log)
+    self.assertEqual(
+      os.listdir(self.getBackupPartitionPath()),
+      os.listdir(destination_backup)
+    )
+    self.assertTrue(os.path.exists(destination_qcow2))
+    # clean up equeue file for precise assertion
+    with open(equeue_file, 'w') as fh:
+      fh.write('')
+    # drop backup destination to assert its recreation
+    os.unlink(destination_qcow2)
+    self.call_exporter()
+    equeue_log = awaitBackup(equeue_file)
+    self.assertIn('qemu-img rebase', equeue_log)
+    self.assertEqual(
+      os.listdir(self.getBackupPartitionPath()),
+      os.listdir(destination_backup)
+    )
+    self.assertTrue(os.path.exists(destination_qcow2))
+    # takeover
+    connection_parameter = self.computer_partition.getConnectionParameterDict()
+    takeover_result = requests.post(
+      connection_parameter['takeover-kvm-1-url'],
+      data={
+        'password': connection_parameter['takeover-kvm-1-password']})
+    self.assertEqual(httplib.OK, takeover_result.status_code)
+    self.assertTrue(takeover_result.text.startswith('Success.'))
+    # the real assertions comes from re-stabilizing the instance tree
+    self.slap.waitForInstance(max_retry=10)
+    # check that all stabilizes after backup after takeover
+    self.call_exporter()
+    self.slap.waitForInstance(max_retry=10)
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupImporterIde(
+  TestInstanceResilientBackupImporter):
+  disk_type = 'ide'
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporter(
+  TestInstanceResilientBackupMixin, KVMTestCase):
+  def test(self):
+    status_text = self.call_exporter()
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('FULL-*.qcow2'))),
+      1)
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('INC-*.qcow2'))),
+      0)
+    self.assertNotIn(
+      'Recovered from partial backup by removing partial',
+      status_text
+    )
+    # cover .partial file in the backup directory with fallback to full
+    current_backup = glob.glob(self.getBackupPartitionPath('FULL-*'))[0]
+    with open(current_backup + '.partial', 'w') as fh:
+      fh.write('')
+    status_text = self.call_exporter()
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('FULL-*.qcow2'))),
+      1)
+    self.assertEqual(
+      len(glob.glob(self.getBackupPartitionPath('INC-*.qcow2'))),
+      1)
+    self.assertIn(
+      'Recovered from partial backup by removing partial',
+      status_text
+    )
+    self.assertTrue(os.path.exists(os.path.join(
+      self.getPartitionPath(
+        'kvm-export', 'etc', 'plugin', 'check-backup-directory.py'))))
+
+
+@skipUnlessKvm
+class TestInstanceResilientBackupExporterIde(
+  TestInstanceResilientBackupExporter):
+  disk_type = 'ide'
+
+
 @skipUnlessKvm
 class TestInstanceResilient(KVMTestCase, KvmMixin):
   __partition_reference__ = 'ir'
@@ -762,25 +974,6 @@ class TestInstanceResilient(KVMTestCase, KvmMixin):
     cls.kvm0_ipv6 = cls.getPartitionIPv6(cls.kvm_instance_partition_reference)
     cls.kvm1_ipv6 = cls.getPartitionIPv6(cls.getPartitionId('kvm1'))
 
-  def test_kvm_exporter(self):
-    exporter_partition = os.path.join(
-      self.slap.instance_directory,
-      self.__partition_reference__ + '2')
-    backup_path = os.path.join(
-      exporter_partition, 'srv', 'backup', 'kvm', 'virtual.qcow2.gz')
-    exporter = os.path.join(exporter_partition, 'bin', 'exporter')
-    if os.path.exists(backup_path):
-      os.unlink(backup_path)
-
-    def call_exporter():
-      try:
-        return (0, subprocess.check_output(
-          [exporter], stderr=subprocess.STDOUT).decode('utf-8'))
-      except subprocess.CalledProcessError as e:
-        return (e.returncode, e.output.decode('utf-8'))
-    status_code, status_text = call_exporter()
-    self.assertEqual(0, status_code, status_text)
-
   def test(self):
     connection_parameter_dict = self\
       .computer_partition.getConnectionParameterDict()
@@ -2319,10 +2512,11 @@ class ExternalDiskModernMixin(object):
     # find qemu_img from the tested SR via it's partition parameter, as
     # otherwise qemu-kvm would be dependency of test suite
     with open(
-      os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
+      glob.glob(os.path.join(
+          self.slap._instance_root, '*', 'bin', 'kvm_raw'))[0]) as fh:
       self.qemu_img = [
         q for q in fh.readlines()
-        if 'raw qemu_img_executable_location' in q][0].split()[-1]
+        if 'qemu_img_path = ' in q][0].split()[-1].replace("'", "")
     self.first_disk = os.path.join(self.working_directory, 'first_disk')
     subprocess.check_call([
       self.qemu_img, "create", "-f", "qcow", self.first_disk, "1M"])
@@ -2469,10 +2663,11 @@ class TestExternalDiskModernIndexRequired(KVMTestCase, ExternalDiskMixin):
     # find qemu_img from the tested SR via it's partition parameter, as
     # otherwise qemu-kvm would be dependency of test suite
     with open(
-      os.path.join(self.computer_partition_root_path, 'buildout.cfg')) as fh:
+      glob.glob(os.path.join(
+          self.slap._instance_root, '*', 'bin', 'kvm_raw'))[0]) as fh:
       qemu_img = [
         q for q in fh.readlines()
-        if 'raw qemu_img_executable_location' in q][0].split()[-1]
+        if 'qemu_img_path = ' in q][0].split()[-1].replace("'", "")
 
     self.first_disk = os.path.join(self.working_directory, 'first_disk')
     subprocess.check_call([

stack/erp5/buildout.hash.cfg

@@ -90,7 +90,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9547bacad0635b0f64cac48f15c4e9ae
+md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -87,7 +87,7 @@ environment +=
   JUPYTER_PATH=${directory:jupyter-dir}
   JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir}
   JUPYTER_RUNTIME_DIR=${directory:jupyter-runtime-dir}
-{% if slapparameter_dict.get('python-hash-seed') %}
+{% if slapparameter_dict.get('python-hash-seed') != '' %}
   PYTHONHASHSEED={{ slapparameter_dict['python-hash-seed'] }}
 {% endif %}
 {% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
@@ -476,7 +476,7 @@ wrapper-path = ${buildout:bin-directory}/runUnitTest
 recipe = slapos.recipe.template
 inline =
   #!/bin/sh
-  {% if slapparameter_dict.get('python-hash-seed') %}
+  {% if slapparameter_dict.get('python-hash-seed') != '' %}
   PYTHONHASHSEED={{ slapparameter_dict['python-hash-seed'] }}
   {% endif %}
   if [ -z "$PYTHONHASHSEED" ]; then