gzip: added --rsyncable option; kvm: compress backup images

component/gzip/buildout.cfg

@@ -10,3 +10,12 @@ url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
 md5sum = da981f86677d58a106496e68de6f8995
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
+patch-options = -p1
+# The --rsyncable patch is from debian/ubuntu,
+# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
+# It is required to minimize the bandwidth used by rsync.
+# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
+# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
+patches =
+  ${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
+

software/kvm/common.cfg

@@ -84,7 +84,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bc5a986c7208d02d3284a897ea90b39d
+md5sum = 4c8f07da2217e54163c265fe6fe3d41d
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -115,7 +115,7 @@ on-update = true
 [template-kvm-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
-md5sum = 98b1acc438895c0fa2309144a9a52a62
+md5sum = 6835c9309ff4bf4a0efd1850e6c66b24
 output = ${buildout:directory}/template-kvm-import.cfg
 mode = 0644
 
@@ -123,7 +123,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2
 filename = kvm-import.sh.jinja2
-md5sum = 2178e7ef7f1b9ea6639bfbaf045e1feb
+md5sum = 926a11421921c29f91fae8240bbcf585
 download-only = true
 mode = 0755
 
@@ -131,7 +131,7 @@ mode = 0755
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = 4972bc6f6a4cb41c1820ac13a32915a8
+md5sum = c9f13c1f481ed08c75089aef1d3c6981
 download-only = true
 on-update = true
 
@@ -139,7 +139,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
 filename = kvm-export.sh.jinja2
-md5sum = 2e2a99e33e5cdde35833d164da1e325b
+md5sum = 22bd2e0c8fdb39a764a14c403a3bd752
 download-only = true
 mode = 0755
 

software/kvm/instance-kvm-export.cfg.jinja2

@@ -27,6 +27,7 @@ context =
     section directory directory
     section buildout buildout
     key socket_path kvm-instance:socket-path
+    raw gzip_binary {{ gzip_binary }}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]

software/kvm/instance-kvm-import.cfg.in

@@ -33,5 +33,7 @@ mode = 0700
 wrapper = $${:rendered}
 context =
     section directory directory
+    raw zcat_binary ${gzip:location}/bin/zcat
+    raw gzip_binary ${gzip:location}/bin/gzip
 
 backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance.cfg.in

@@ -82,6 +82,7 @@ context =
     raw kvm_template $${dynamic-template-kvm:rendered}
     raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
     raw pbsready_export_template ${pbsready-export:output}
+    raw gzip_binary ${gzip:location}/bin/gzip
 mode = 0644
 
 [dynamic-template-kvm-resilient-test]

software/kvm/template/kvm-export.sh.jinja2

@@ -11,5 +11,19 @@ QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
 
 $QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE
 
+# Due to the way qmp works, the VM file cannot be compressed on the fly.
+# Although the compression step is optional, the importer uses the .gz file
+# if present. So, remove it if you are disabling the compression.
+
+# The downside of compression, here, is the temporary usage of more disk space
+# in the exporter node. The goal is to minimize disk usage on the PBS server.
+
+# If you want to compress the file in-place:
+# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
+# but the importer script would have to be adapted.
+
+echo "Compressing backup..."
+{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
+
 cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
 

software/kvm/template/kvm-import.sh.jinja2

@@ -16,7 +16,11 @@ write_backup_proof () {
 # For now we just make the diff before 
 write_backup_proof
 
-# TODO: Use rdiff
-rm $VM_DIR/$VM_FILE
-cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
+    {{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
+    {{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
+else
+    rm $VM_DIR/$VM_FILE
+    cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+fi