Update Release Candidate

software/beremiz-runtime/software.cfg

@@ -40,7 +40,7 @@ post-install =
 [beremiz-source]
 recipe = slapos.recipe.build:gitclone
 repository = https://github.com/beremiz/beremiz.git
-revision = 2d061a6
+revision = 9eb53af
 git-executable = ${git:location}/bin/git
 
 [nevow-py3-source]

software/grafana/buildout.hash.cfg

@@ -19,11 +19,11 @@ md5sum = 32c772c593d2c3c38c26186b91b78cf8
 
 [instance-default]
 filename = instance-default.cfg.in
-md5sum = b4330fbe0c9c3631f4f477c06d3460b3
+md5sum = 86e379ac6ba789d65369ca1e0d4dfa3f
 
 [instance-agent]
 filename = instance-agent.cfg.in
-md5sum = 9fc368a3ac16bee297a4f2ad87b3bbb4
+md5sum = 89fd1b010351cea9968360054e0c4aa9
 
 
 [influxdb-config-file]

software/grafana/instance-agent.cfg.in

@@ -214,13 +214,13 @@ init =
                 },
                 {
                   "query": """
-                    select 'message' as cmf_activity_queue, count(*) as failed_message_count
-                      from message where processing_node between -2 and -10
-                    union all select 'message_queue' as cmf_activity_queue, count(*) as failed_message_count
-                      from message_queue where processing_node between -2 and -10
+                    select 'message' as cmf_activity_queue, method_id, count(*) as failed_message_count
+                      from message where processing_node between -10 and -2 group by method_id
+                    union all select 'message_queue' as cmf_activity_queue, method_id, count(*) as failed_message_count
+                      from message_queue where processing_node between -10 and -2 group by method_id
                   """,
                   "field_columns_include": ["failed_message_count"],
-                  "tag_columns_include": ["cmf_activity_queue"],
+                  "tag_columns_include": ["cmf_activity_queue", "method_id"],
                 },
                 # TODO: these queries are slow and maybe not correct
                 # {
@@ -263,7 +263,8 @@ init =
     urls = application.get("urls", [])
     if urls:
       inputs["http_response"].append({
-        "interval": "5m",
+        "interval": "2m",
+        "response_timeout": "1m",
         "urls": urls,
         "tags": {"app": application["name"]},
       })

software/grafana/instance-default.cfg.in

@@ -637,7 +637,7 @@ init =
       "type": "system",
       "partitions": [
         {
-          "name": "grafana",
+          "name": "influxdb",
           "static-tags": {
             "partition_reference": slap_connection['partition-id'],
           },
@@ -646,7 +646,7 @@ init =
           ]
         },
         {
-          "name": "influxdb",
+          "name": "grafana",
           "static-tags": {
             "partition_reference": slap_connection['partition-id'],
           },

software/kvm/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 9ae66fb63a3cdd8072582622aa1bb36c
+md5sum = bf0a70140e4775efaacdd5fe03ac3b62
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = bd3a7229e4fdfa9372ee61b6054acf78
+md5sum = 20554e40f9fe666839ef86ff8216cebf
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -31,19 +31,19 @@ md5sum = 839fc16c112d3b87e2dbd2e382e326de
 
 [template-kvm-import]
 filename = instance-kvm-import.cfg.jinja2.in
-md5sum = a463a5e3cd2287d275d6943c2a11b7e4
+md5sum = 0829d08337a09b8ef017e71fbb9384fb
 
 [template-kvm-import-script]
 filename = template/kvm-import.sh.jinja2
-md5sum = 013725987114c82ca3fd11097d0a7f9f
+md5sum = a6e6abb3b17fae37b84233d63014b885
 
 [template-kvm-export]
 filename = instance-kvm-export.cfg.jinja2
-md5sum = 34d1b7cc8ca62bfdfce759a1dfbbaccd
+md5sum = a02f0694dcb944c18d99f7f79afa2384
 
 [template-kvm-export-script]
 filename = template/kvm-export.sh.jinja2
-md5sum = 64aa1ce8785f6b94aabd787fa3443082
+md5sum = a1da7809d547b4c61e7c6337bf9f8b8a
 
 [template-nginx]
 filename = template/nginx_conf.in
@@ -96,3 +96,7 @@ md5sum = e9d40162ba77472775256637a2617d14
 [boot-image-select-source-config]
 filename = template/boot-image-select-source-config.json.in
 md5sum = d53afe719e2cbfc2480277af340f8429
+
+[check-backup-directory.sh]
+filename = template/check-backup-directory.sh
+md5sum = e569494a941e1d585c2e0bbf070cf1c9

software/kvm/instance-kvm-export.cfg.jinja2

@@ -6,6 +6,7 @@ extends =
 parts +=
   cron-entry-backup
   ${instance-kvm-parts:parts}
+  check-backup-directory-promise
 
 [slap-parameter]
 {% for k, v in slapparameter_dict.items() -%}
@@ -16,13 +17,10 @@ parts +=
 {%-  endif %}
 {% endfor -%}
 
-# Create the exporter executable, which is a simple shell script
-[exporter]
-recipe = slapos.recipe.template:jinja2
-url = {{ template_kvm_export }}
-output = ${directory:bin}/${slap-parameter:namebase}-exporter
-# Resilient stack wants a "wrapper" parameter
-wrapper = ${:output}
+[directory]
+tmp = ${buildout:directory}/tmp
+
+[disk]
 {%- set disk_type = slapparameter_dict.get('disk-type', 'virtio') %}
 {%- if disk_type == "virtio" %}
 device = virtio0
@@ -32,12 +30,19 @@ device = ide0-hd0
 {%- else %}
 # unsupported disk-type {{ disk_type }}
 {%- endif %}
+
+[exporter]
+recipe = slapos.recipe.template:jinja2
+url = {{ template_kvm_export }}
+output = ${directory:bin}/${slap-parameter:namebase}-exporter
+# Resilient stack wants a "wrapper" parameter
+wrapper = ${:output}
 context =
     section directory directory
     section buildout buildout
     key socket_path kvm-instance:socket-path
-    key device :device
-    raw gzip_binary {{ gzip_binary }}
+    section disk disk
+    raw qmpbackup {{ qmpbackup }}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]
@@ -56,3 +61,10 @@ username = {{ slapparameter_dict['monitor-username'] }}
 password = {{ slapparameter_dict['monitor-password'] }}
 {% endif -%}
 
+[check-backup-directory-promise]
+# Check that disk image is not corrupted
+<= monitor-promise-base
+promise = check_command_execute
+name = check-backup-directory.py
+config-command = {{ check_backup_directory }} ${directory:backup}/${disk:device} ${directory:tmp}
+failure_amount = 5

software/kvm/instance-kvm-import.cfg.jinja2.in

@@ -48,6 +48,7 @@ bin = ${buildout:directory}/bin
 srv = ${buildout:directory}/srv
 var = ${buildout:directory}/var
 log = ${:var}/log
+tmp = ${buildout:directory}/tmp
 scripts = ${:etc}/run
 services = ${:etc}/service
 novnc-conf = ${:etc}/novnc
@@ -65,10 +66,7 @@ output = ${directory:bin}/${slap-parameter:namebase}-importer
 wrapper = ${:output}
 context =
     section directory directory
-    raw zcat_binary {{ zcat_binary }}
-    raw gzip_binary {{ gzip_binary }}
-
-backup-disk-path = ${directory:backup}/virtual.qcow2
+    raw qmprestore {{ qmprestore }}
 
 [kvm-disk-image-corruption-bin]
 recipe = collective.recipe.template

software/kvm/instance-kvm.cfg.jinja2

@@ -715,7 +715,6 @@ cron-entries = ${directory:cron-entries}
 crontabs = ${directory:crontabs}
 cronstamps = ${directory:cronstamps}
 catcher = ${cron-simplelogger:wrapper}
-binary = ${directory:bin}/crond_raw
 
 [cron-service]
 recipe = slapos.cookbook:wrapper

software/kvm/instance.cfg.in

@@ -122,6 +122,15 @@ template-replicated-destination = ${template-replicated:target}
 import-list = file parts :template-parts-destination
               file replicated :template-replicated-destination
 
+[qmpbackup-binary]
+recipe = slapos.cookbook:wrapper
+environment =
+  PATH=${qemu:location}/bin
+wrapper-path =
+  $${buildout:bin-directory}/qmpbackup
+command-line =
+  ${buildout:bin-directory}/qmpbackup
+
 [dynamic-template-kvm-export]
 recipe = slapos.recipe.template:jinja2
 url = ${template-kvm-export:location}/instance-kvm-export.cfg.jinja2
@@ -133,8 +142,18 @@ context =
     raw kvm_template $${dynamic-template-kvm:output}
     raw template_kvm_export ${template-kvm-export-script:target}
     key pbsready_export_template template-pbsready-export:output
-    raw gzip_binary ${gzip:location}/bin/gzip
     key slapparameter_dict slap-configuration:configuration
+    key qmpbackup qmpbackup-binary:wrapper-path
+    raw check_backup_directory ${check-backup-directory.sh:output}
+
+[qmprestore-binary]
+recipe = slapos.cookbook:wrapper
+environment =
+  PATH=${qemu:location}/bin
+wrapper-path =
+  $${buildout:bin-directory}/qmprestore
+command-line =
+  ${buildout:bin-directory}/qmprestore
 
 [dynamic-template-kvm-import]
 recipe = slapos.recipe.template:jinja2
@@ -148,5 +167,4 @@ context =
     raw template_kvm_import ${template-kvm-import-script:target}
     key pbsready_import_template template-pbsready-import:output
     key slapparameter_dict slap-configuration:configuration
-    raw zcat_binary ${gzip:location}/bin/zcat
-    raw gzip_binary ${gzip:location}/bin/gzip
+    key qmprestore qmprestore-binary:wrapper-path

software/kvm/software.cfg

@@ -10,7 +10,6 @@ extends =
   ../../component/netcat/buildout.cfg
   ../../component/nginx/buildout.cfg
   ../../component/pycurl/buildout.cfg
-  ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/resilient/buildout.cfg
   buildout.hash.cfg
@@ -19,6 +18,7 @@ extends =
 # to avoid versioning issues
 common-parts =
   template
+  qmpbackup
 
 # XXX: we have to manually add this for resilience
   pbs-recipe-egg
@@ -32,6 +32,18 @@ parts = ${:common-parts}
 #             In qemu builtin vnc server, and make it available only for localhost
 #             so that only novnc can listen to it.
 
+[qmpbackup]
+recipe = zc.recipe.egg
+eggs =
+  qemu.qmp
+  qmpbackup
+find-links +=
+  https://github.com/abbbi/qmpbackup/releases/download/v0.37/qmpbackup-0.37.tar.gz
+
+[versions]
+qemu.qmp = 0.0.3:whl
+qmpbackup = 0.37
+
 [python-with-eggs]
 recipe = zc.recipe.egg
 interpreter = ${:_buildout_section_name_}
@@ -129,3 +141,7 @@ context =
 
 [boot-image-select-source-config]
 <= download-base
+
+[check-backup-directory.sh]
+<= template-base
+output = ${buildout:parts-directory}/${:_buildout_section_name_}/check-backup-directory.sh

software/kvm/template/check-backup-directory.sh

+#!/bin/bash
+
+directory=$1
+tmp=$2
+
+# support a case of not ready yet directory
+if [ ! -d $directory ] ; then
+  exit 0
+fi
+
+tmpfile=$(mktemp -p $tmp)
+
+trap "rm -fr $tmpfile" EXIT TERM INT
+
+find $directory -type f -name 'FULL*qcow2' -printf '%f\n' > $tmpfile
+full_amount=$(wc -l $tmpfile | cut -d ' ' -f 1)
+if [ $full_amount -gt 1 ]; then
+  echo "Too many FULL backups"
+  cat $tmpfile
+  exit 1
+fi
+
+find $directory -type f -name 'INC*qcow2' -printf '%f\n' > $tmpfile
+if [ $(wc -l $tmpfile | cut -d ' ' -f 1) -gt 0 ] && [ $full_amount -eq 0 ] ; then
+  echo "INC present but no FULL backup"
+  cat $tmpfile
+  exit 1
+fi
+
+find $directory -type f -name '*.partial' -printf '%f\n' > $tmpfile
+if [ $(wc -l $tmpfile | cut -d ' ' -f 1) -ne 0 ]; then
+  echo "Partial file present"
+  cat $tmpfile
+  exit 1
+fi
+
+exit 0

software/kvm/template/kvm-export.sh.jinja2

@@ -6,25 +6,42 @@ set -e
 LC_ALL=C
 export LC_ALL
 BACKUP_DIR={{ directory['backup'] }}
-BACKUP_FILE=virtual.qcow2
 
-QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
-
-$QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE {{ device }}
-
-# Due to the way qmp works, the VM file cannot be compressed on the fly.
-# Although the compression step is optional, the importer uses the .gz file
-# if present. So, remove it if you are disabling the compression.
-
-# The downside of compression, here, is the temporary usage of more disk space
-# in the exporter node. The goal is to minimize disk usage on the PBS server.
+log=$(mktemp --tmpdir={{ directory['tmp'] }})
+trap "rm -f $log" EXIT TERM INT
+
+set +e
+qmpbackup="{{ qmpbackup }} --socket {{ socket_path }} backup --compress --target $BACKUP_DIR --include {{ disk['device'] }}"
+$qmpbackup --level auto > $log
+RESULT=$?
+cat $log
+if [ $RESULT -ne 0 ] ; then
+  # recover from unfinished previous backup
+  if egrep -q 'Partial backup found in.*{{ disk['device']}}.*possible broken backup chain. Execute new full backup' $log ; then
+    find $BACKUP_DIR/{{ disk['device'] }} -name '*.partial' -delete
+    $qmpbackup --level auto || exit $?
+    echo "Recovered from partial backup by removing partial"
+  elif egrep -q 'Incremental backup requested but no active bitmap has been found' $log ; then
+    find $BACKUP_DIR/{{ disk['device'] }} -name '*.qcow2' -delete
+    $qmpbackup --level full || exit $?
+    echo "Post take-over cleanup"
+  else
+    exit $RESULT
+  fi
+fi
+set -e
 
-# If you want to compress the file in-place:
-# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
-# but the importer script would have to be adapted.
+# as new style backup went fine delete potential old style backup
+rm -f $BACKUP_DIR/virtual.qcow2{,.gz}
 
-echo "Compressing backup..."
-{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
+# cleanup the backup directory from too old backups, especially important after take-over
+recent_full=$(find $BACKUP_DIR -type f -name 'FULL-*.qcow2' -exec ls -t1 {} + | head -n1)
+if [ x"$recent_full" != x"" ] ; then
+  for f in $(find $BACKUP_DIR -type f -name '*qcow2' \! -newer $recent_full) ; do
+    if [ "$f" != "$recent_full" ] ; then
+      rm -vf $f
+    fi
+  done
+fi
 
 cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
-

software/kvm/template/kvm-import.sh.jinja2

@@ -4,7 +4,6 @@ set -e
 
 VM_DIR={{ directory['srv'] }}
 BACKUP_DIR={{ directory['backup'] }}
-VM_FILE=virtual.qcow2
 LC_ALL=C
 export LC_ALL
 umask 077
@@ -18,11 +17,19 @@ write_backup_proof () {
 # For now we just make the diff before 
 write_backup_proof
 
-if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
-    {{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
-    {{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
+tmpfile=$(mktemp --tmpdir={{ directory['tmp'] }})
+# use temporary space inside of the partition, as it can be quite big
+# assure the temporary directory is cleaned up
+trap "rm -rf $tmpdir $tmpfile" EXIT TERM INT
+
+if [ -d $BACKUP_DIR/ide0-hd0 ] ; then
+  disk_type="ide0-hd0"
+elif [ -d $BACKUP_DIR/virtio0 ] ; then
+  disk_type=virtio0
 else
-    rm $VM_DIR/$VM_FILE
-    cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+  echo "Unsupported disk type"
+  exit 1
 fi
 
+rm -f $VM_DIR/virtual.qcow2
+{{ qmprestore }} merge --dir $BACKUP_DIR/$disk_type --target $VM_DIR/virtual.qcow2

software/kvm/test/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template-test]
+filename = test-instance.cfg.in
+md5sum = 466c7324a44e0cd8ae688699aab62d5f
+
+[template-kvm-export-mock]
+filename = test-instance-kvm-export-mock.cfg.jinja2
+md5sum = acf0b9a916e1411f184eb4f94c7c2abb

software/kvm/test/test-instance-kvm-export-mock.cfg.jinja2

+[buildout]
+extends = {{ template_kvm_export }}
+
+parts +=
+  cron-entry-environment
+
+[directory]
+cron-d-mock = ${:var}/cron-d-mock
+
+[cron-entry-environment]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = environment
+frequency = * * * * *
+command = ${buildout:executable} -c 'import os ; import json ; print(json.dumps(dict(os.environ)))' > ${directory:var}/cron-environment.json
+
+[cron-entry-backup]
+recipe = slapos.recipe.template:jinja2
+inline =
+  ${:command}
+output = ${directory:cron-d-mock}/${:name}

software/kvm/test/test-instance.cfg.in

+[buildout]
+extends = ${template:output}
+
+[dynamic-template-kvm-export-mock]
+recipe = slapos.recipe.template:jinja2
+url = ${template-kvm-export-mock:location}/${template-kvm-export-mock:filename}
+output = $${buildout:directory}/template-kvm-export-mock.cfg
+context =
+  key template_kvm_export dynamic-template-kvm-export:output
+
+[switch_softwaretype]
+kvm-export = dynamic-template-kvm-export-mock:output

software/kvm/test/test-software.cfg

+[buildout]
+extends =
+  ../software.cfg
+  buildout.hash.cfg
+
+parts +=
+  template-test
+
+[template]
+output = ${buildout:directory}/template-original.cfg
+
+[template-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+
+[template-kvm-export-mock]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}

stack/erp5/buildout.hash.cfg

@@ -90,7 +90,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9547bacad0635b0f64cac48f15c4e9ae
+md5sum = 9c580be982d8c63ec06fc273ef3cb971
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -87,7 +87,7 @@ environment +=
   JUPYTER_PATH=${directory:jupyter-dir}
   JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir}
   JUPYTER_RUNTIME_DIR=${directory:jupyter-runtime-dir}
-{% if slapparameter_dict.get('python-hash-seed') %}
+{% if slapparameter_dict.get('python-hash-seed') != '' %}
   PYTHONHASHSEED={{ slapparameter_dict['python-hash-seed'] }}
 {% endif %}
 {% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
@@ -476,7 +476,7 @@ wrapper-path = ${buildout:bin-directory}/runUnitTest
 recipe = slapos.recipe.template
 inline =
   #!/bin/sh
-  {% if slapparameter_dict.get('python-hash-seed') %}
+  {% if slapparameter_dict.get('python-hash-seed') != '' %}
   PYTHONHASHSEED={{ slapparameter_dict['python-hash-seed'] }}
   {% endif %}
   if [ -z "$PYTHONHASHSEED" ]; then