#!{{ shell_binary }} LC_ALL=C export LC_ALL umask 077 # Exit on any error, to prevent inconsistent backup # Error on unset variable expansion set -eu # Redirect output to log exec > >(tee -ai '{{ output_log_file }}') exec 2>&1 echo -e "\n\n$0 run at : $(date)" srv_directory='{{ directory["srv"] }}' backup_directory='{{ directory["backup"] }}' etc_directory='{{ directory["etc"] }}' RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}' RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}' ERROR_MESSAGE="" fail_with_exit_code () { echo 1 > $RESTORE_EXIT_CODE_FILE echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE exit 1 } trap fail_with_exit_code ERR log_message () { ERROR_MESSAGE=$1 echo -e $1 } # Delete the error message file, to not keep it even after a successful build rm "$RESTORE_ERROR_MESSAGE_FILE" || true rsync () { set -x '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@" set +x } log_message "Restoring WebRunner content..." ( # XXX: code duplication with runner-export.sh.jinja2 path=$srv_directory/runner backup_path=$backup_directory/runner/ cd "$backup_path" if [ -d instance ]; then # Concatenate the exclude file of each partition of webrunner # to create a global exclude file. # Also, ignore all buildout-managed files. exclude=$({{ sys.executable }} - "$path" <<EOF if 1: import glob, errno, os, sys sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }} from zc.buildout.configparser import parse path = sys.argv[1] def print_relative(path_list): for p in path_list: p = p.strip() if p: print(os.path.relpath(p, path)) print("*.sock") print("*.socket") print("*.pid") print(".installed*.cfg") for partition in glob.glob(path + "/instance/slappart*"): try: os.chdir(partition) except OSError as e: if e.errno != errno.ENOTDIR: raise continue try: with open("srv/exporter.exclude") as f: exclude = f.readlines() except IOError as e: if e.errno != errno.ENOENT: raise else: print_relative(exclude) for installed in glob.glob(".installed*.cfg"): try: with open(installed) as f: installed = parse(f, installed) except IOError as e: if e.errno != errno.ENOENT: raise else: for section in installed.itervalues(): print_relative(section.get( '__buildout_installed__', '').splitlines()) EOF ) echo "$exclude" |rsync --exclude-from=- instance "$path" fi test -d project && rsync project "$path" test -d public && rsync public "$path" test -f proxy.db && rsync proxy.db "$path" ) log_message "Restoring WebRunner config (etc directory)..." ( cd "$backup_directory"/etc/ rsync config.json "$etc_directory" # Hidden files are related to the webrunner's internals cp -r .??* "$etc_directory" ) # Invoke arbitrary script to perform specific restoration # procedure. runner_import_restore=$srv_directory/runner-import-restore if [ -x "$runner_import_restore" ]; then log_message "Running $runner_import_restore..." "$srv_directory/runner-import-restore" fi # If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume # that there is no instance deployed on runner0 if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then log_message "No Software Requested... Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi log_message "Updating slapproxy database..." HOME='{{ directory["home"] }}' # XXX Hardcoded export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export MAKEFLAGS=-j4 SLAPOS='{{ directory["bin"] }}'/slapos SQLITE3="{{ sqlite3_binary }}" DATABASE="$HOME/srv/runner/proxy.db" db_query () { # Try opening locked tables for 5 seconds to prevent "database is locked" error "$SQLITE3" "$DATABASE" <<EOF .timeout 5000 $@ EOF } # If slapproxy database is empty then no software release was opened if [ ! -s "$DATABASE" ]; then log_message "Slapproxy database empty, no Software Requested... Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi # Slap proxy table contain version number, find the table name dynamically. # This is known to work with version 11 or 12 of tables, but it will probably # work with earlier versions as well. DB_PARTITION_TABLE=$(db_query ".table partition__") DB_PARTITION_NETWORK_TABLE=$(db_query ".table partition\_network__") DB_SOFTWARE_TABLE=$(db_query ".table software__") # Change slapproxy database to point instances to new software release # XXX hardcoded PARTITION=$(basename $HOME) OLD_SOFTWARE_RELEASE=$(db_query "select software_release from $DB_PARTITION_TABLE where reference='slappart0';") if [ "$OLD_SOFTWARE_RELEASE" == "" ]; then echo "No instance configured" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi SOFTWARE_RELEASE=$({{ sys.executable }} - $OLD_SOFTWARE_RELEASE $PARTITION <<EOF if 1: import os, re, sys # We want to replace the last occurence only old_software_release, partition = sys.argv[1], sys.argv[2] for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release): start, end = match.start(), match.end() print old_software_release[:start] + partition + old_software_release[end:] EOF ) db_query "update $DB_PARTITION_TABLE set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;" db_query "update $DB_SOFTWARE_TABLE set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || db_query "delete from $DB_SOFTWARE_TABLE where url='$OLD_SOFTWARE_RELEASE';" # Change slapproxy database to have all instances stopped db_query "update $DB_PARTITION_TABLE set requested_state='stopped';" # Change slapproxy database to get correct IPs IPV4='{{ ipv4 }}' IPV6='{{ ipv6 }}' db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV4' where netmask='255.255.255.255';" db_query "update $DB_PARTITION_NETWORK_TABLE set address='$IPV6' where netmask='ffff:ffff:ffff::';" MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}' log_message "Removing old supervisord service description files..." # XXX: Path hardcoded in slapos.core rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true SLAPOSCFG='{{ supervisord["slapos-cfg"] }}' SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}' SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}' contain_software_release=0 SOFTWARE_RELEASES_COUNT=$(db_query "SELECT count(1) FROM $DB_SOFTWARE_TABLE WHERE url != '';") if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then contain_software_release=1 fi if [ $contain_software_release -eq 0 ]; then log_message "No Software Release were deployed, so skip to continue..." echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi log_message "Building newest Software Release..." "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDSRLOG" && false) contain_instance=0 for folder in $srv_directory/runner/instance/slappart*/; do if [ -f "$folder/buildout.cfg" ]; then contain_instance=1 fi done # If instance do not contains template.cfg it means the user contains no instance. # so it is safer to assume that he is using slaprunner for develop buildout rather them slapos. if [ $contain_instance -eq 0 ]; then log_message "None Instance were deployed with this software release, so skip to continue..." echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi # Remove defined scripts to force buildout to recreate them to have updated paths rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true log_message "Fixing Instances as needed after import..." # XXX hardcoded "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDCPLOG" && false) # Invoke defined scripts for each partition inside of slaprunner log_message "Invoke custom import scripts defined by each instances..." for partition in "$srv_directory"/runner/instance/slappart*/ do script=$partition/srv/runner-import-restore if [ -x "$script" ]; then log_message "Running custom instance script : $script..." "$script" fi done # Change back slapproxy database to have all instances started DB_PARTITION_TABLE=$(db_query ".table partition__") log_message "Set instances as to start after takeover..." db_query "update $DB_PARTITION_TABLE set requested_state='started';" # Write exit code to an arbitrary file that will be checked by promise/monitor log_message "Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0