erp5: add webrunner resiliency after-import (restore) scripts.

stack/erp5/buildout.cfg

@@ -147,10 +147,16 @@ context =
     key slapos_core_version versions:slapos.core
     ${:extra-context}
 
+[mariadb-resiliency-after-import-script]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
+#md5sum = e5fc0b14ed01fc66ef276ebb044c49f1
+mode = 755
+
 [template-mariadb]
 < = template-jinja2-base
 filename = instance-mariadb.cfg
-md5sum = 7fda20a3f39c425c52631b97eeb4e471
+#md5sum = 2116e3557447c2b30031b9fc220118bf
 extra-context =
     key coreutils_location coreutils:location
     key dcron_location dcron:location
@@ -160,6 +166,7 @@ extra-context =
     key gzip_location gzip:location
     key logrotate_location logrotate:location
     key mariadb_location mariadb:location
+    key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
     key perl_location perl:location
     key perl_siteprefix perl:siteprefix
     key sed_location sed:location
@@ -185,7 +192,7 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 453cbef56fc2cb0ecab813ecace2d7dd
+#md5sum = 025b8b5cd4bf162a976af192e5734b81
 mode = 640
 
 [template-cloudooo]
@@ -602,6 +609,7 @@ WSGIUtils = 0.7
 apache-libcloud = 0.16.0
 astroid = 1.3.2
 chardet = 2.3.0
+collective.recipe.template = 1.11
 csp-eventlet = 0.7.0
 ecdsa = 0.11
 elementtree = 1.2.6.post20050316
@@ -617,6 +625,7 @@ huBarcode = 1.0.0
 interval = 1.0.0
 ipdb = 0.8
 ipython = 2.3.1
+jsonschema = 2.4.0
 logilab-common = 0.63.2
 numpy = 1.9.1
 plone.recipe.command = 1.1

stack/erp5/instance-mariadb.cfg.in

@@ -10,6 +10,7 @@ parts =
   cron-entry-mariadb-backup-expire
   binary-link
   resiliency-exclude-file
+  resiliency-after-import-script
   promise
 
 eggs-directory = {{ eggs_directory }}
@@ -197,6 +198,13 @@ recipe = collective.recipe.template
 input = inline: srv/mariadb/**
 output = ${rootdirectory:srv}/exporter.exclude
 
+[resiliency-after-import-script]
+# Generate after import script used by importer instance of webrunner
+recipe = collective.recipe.template
+input = {{ mariadb_resiliency_after_import_script }}
+output = ${rootdirectory:srv}/runner-import-restore
+mode = 755
+
 [rootdirectory]
 recipe = slapos.cookbook:mkdirectory
 etc = ${buildout:directory}/etc

stack/erp5/instance-tidstorage.cfg.in

@@ -105,6 +105,7 @@ instance-tests = ${:instance}/tests
 log = ${:var}/log
 logrotate-backup = ${:backup}/logrotate
 logrotate-entries = ${:etc}/logrotate.d
+promise = ${:etc}/promise
 run = ${:var}/run
 services = ${:etc}/run
 srv = ${buildout:directory}/srv
@@ -113,6 +114,7 @@ tmp = ${buildout:directory}/tmp
 var = ${buildout:directory}/var
 zodb = ${:srv}/zodb
 zodb-backup = ${:backup}/zodb
+zodb-recovery-test = ${:srv}/zodbrecoverytest
 
 #############################
 # Binary symlinking
@@ -148,12 +150,57 @@ link-binary =
 #############################
 # Resiliency
 #############################
+# Used for ERP5 resiliency or (more probably)
+# webrunner resiliency with erp5 inside.
 [resiliency-exclude-file]
 # Generate rdiff exclude file in case of resiliency
 recipe = collective.recipe.template
 input = inline: srv/zodb/**
 output = ${directory:srv}/exporter.exclude
 
+[resiliency-after-import-script]
+# Generate after import script used by importer instance of webrunner
+recipe = collective.recipe.template
+input = inline: #!/bin/sh
+  # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+  # OR ZODB DATA WILL BE ERASED.
+
+  # This script will restore the repozo backup to the real
+  # zodb location. It is launched by the clone (importer) instance of webrunner
+  # in the end of the import script.
+
+  # Depending on the output, it will create a file containing
+  # the status of the restoration (success or failure).
+  zodb_directory="${directory:zodb}"
+  zodb_backup_directory="${directory:zodb-backup}"
+  repozo="${tidstorage:repozo-binary}"
+  EXIT_CODE=0
+  {% for zeo_id, zeo_configuration_list in json['zeo'].iteritems() -%}
+  {%   for zeo_slave in zeo_configuration_list -%}
+  storage_name="{{ zeo_slave['storage-name'] }}"
+  zodb_path="$storage_name.fs"
+  pid_file=${zeo-instance-{{ zeo_id }}:pid-path}
+  if [ -e "$pid_file" ]; then
+    pid=$(cat $pid_file) > /dev/null 2>&1
+    if kill -0 "$pid"; then
+      echo "Zeo is already running with pid $pid. Aborting."
+      exit 1
+    fi
+  fi
+  echo "Removing $zodb_path..."
+  echo "Restoring $storage_name into $zodb_path..."
+  $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name"
+  CURRENT_EXIT_CODE=$?
+  if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
+    EXIT_CODE="$CURRENT_EXIT_CODE"
+    echo "$storage_name Backup restoration failed."
+  fi
+  {%   endfor -%}
+  {% endfor -%}
+  exit $EXIT_CODE
+output = ${directory:srv}/runner-import-restore
+mode = 755
+
 #############################
 # CA
 #############################
@@ -534,6 +581,7 @@ parts =
   certificate-authority
   erp5-certificate-authority
   resiliency-exclude-file
+  resiliency-after-import-script
   tidstorage
   cron-entry-tidstorage-backup
   logrotate-entry-tidstorage

stack/erp5/template/instance-mariadb-resiliency-after-import-script.sh.in

+#!/bin/sh
+
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# OR MYSQL DATA WILL BE ERASED.
+
+# This script will import the dump of the mysql database to the real
+# database. It is launched by the clone (importer) instance of webrunner
+# in the end of the import script.
+
+# Depending on the output, it will create a file containing
+# the status of the restoration (success or failure)
+
+mysql_executable="${mariadb-instance:mysql-binary}"
+mysqldump_executable="${binary-wrap-mysqldump:wrapper-path}"
+mariadb_data_directory="${directory:mariadb-data}"
+mariadb_backup_directory="${directory:mariadb-backup-full}"
+instance_directory="${buildout:directory}"
+pid_file="${mariadb-instance:pid-file}"
+binlog_path="${mariadb-instance:binlog-path}"
+
+# Make sure mariadb is not already running
+if [ -e "$pid_file" ]; then
+  pid=$(cat $pid_file) > /dev/null 2>&1
+  if kill -0 "$pid"; then
+    echo "Mariadb is already running with pid $pid. Aborting."
+    exit 1
+  fi
+fi
+
+echo "Deleting existing database..."
+rm -r $mariadb_data_directory/*
+
+echo "Adapting binlog database to new paths..."
+new_binlog_directory="$(dirname $binlog_path)"
+binlog_index_file="$new_binlog_directory/binlog.index"
+old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
+sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
+
+echo "Starting mariadb..."
+# XXX hardcoded
+$instance_directory/etc/run/mariadb &
+mysqld_pid=$!
+$instance_directory/etc/run/mariadb_update > /dev/null 2>&1
+
+echo "Importing data..."
+# Use latest dump XXX can contain funny characters
+dump=$(ls -r $mariadb_backup_directory | head -1)
+zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
+RESTORE_EXIT_CODE=$?
+
+kill "$mysqld_pid"
+
+if [ $RESTORE_EXIT_CODE -eq 0 ]; then
+  echo 'Backup restoration successfully completed.'
+else
+  echo 'Backup restoration failed.'
+fi
+
+exit $RESTORE_EXIT_CODE