#!{{ shell_binary }} LC_ALL=C export LC_ALL umask 077 # Exit on any error, to prevent inconsistent backup # Error on unset variable expansion set -eu # Redirect output to log exec > >(tee -ai '{{ output_log_file }}') exec 2>&1 echo -e "\n\n$0 run at : $(date)" srv_directory='{{ directory["srv"] }}' backup_directory='{{ directory["backup"] }}' etc_directory='{{ directory["etc"] }}' RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}' RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}' ERROR_MESSAGE="" fail_with_exit_code () { echo 1 > $RESTORE_EXIT_CODE_FILE echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE exit 1 } trap fail_with_exit_code ERR log_message () { ERROR_MESSAGE=$1 echo -e $1 } # Delete the error message file, to not keep it even after a successful build rm "$RESTORE_ERROR_MESSAGE_FILE" || true rsync () { set -x '{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@" set +x } log_message "Restoring WebRunner content..." ( # XXX: code duplication with runner-export.sh.jinja2 path=$srv_directory/runner backup_path=$backup_directory/runner/ cd "$backup_path" if [ -d instance ]; then # Concatenate the exclude file of each partition of webrunner # to create a global exclude file. # Also, ignore all buildout-managed files. exclude=$({{ sys.executable }} - "$path" <<EOF if 1: import glob, errno, os, sys sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }} from zc.buildout.configparser import parse path = sys.argv[1] def print_relative(path_list): for p in path_list: p = p.strip() if p: print(os.path.relpath(p, path)) print("*.sock") print("*.socket") print("*.pid") print(".installed*.cfg") for partition in glob.glob(path + "/instance/slappart*"): os.chdir(partition) try: with open("srv/exporter.exclude") as f: exclude = f.readlines() except IOError as e: if e.errno != errno.ENOENT: raise else: print_relative(exclude) for installed in glob.glob(".installed*.cfg"): try: with open(installed) as f: installed = parse(f, installed) except IOError as e: if e.errno != errno.ENOENT: raise else: for section in installed.itervalues(): print_relative(section.get( '__buildout_installed__', '').splitlines()) EOF ) echo "$exclude" |rsync --exclude-from=- instance "$path" fi test -d project && rsync project "$path" test -d public && rsync public "$path" test -f proxy.db && rsync proxy.db "$path" ) log_message "Restoring WebRunner config (etc directory)..." ( cd "$backup_directory"/etc/ rsync config.json "$etc_directory" # Hidden files are related to the webrunner's internals cp -r .??* "$etc_directory" ) # Invoke arbitrary script to perform specific restoration # procedure. runner_import_restore=$srv_directory/runner-import-restore if [ -x "$runner_import_restore" ]; then log_message "Running $runner_import_restore..." "$srv_directory/runner-import-restore" fi # If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume # that there is no instance deployed on runner0 if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then log_message "No Software Requested... Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi log_message "Updating slapproxy database..." HOME='{{ directory["home"] }}' # XXX Hardcoded export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export MAKEFLAGS=-j4 SLAPOS='{{ directory["bin"] }}'/slapos # XXX hardcoded SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3" DATABASE="$HOME/srv/runner/proxy.db" db_query () { # Try opening locked tables for 5 seconds to prevent "database is locked" error "$SQLITE3" "$DATABASE" <<EOF .timeout 5000 $@ EOF } # If slapproxy database is empty then no software release was opened if [ ! -s "$DATABASE" ]; then log_message "Slapproxy database empty, no Software Requested... Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi # Change slapproxy database to point instances to new software release # XXX hardcoded PARTITION=$(basename $HOME) OLD_SOFTWARE_RELEASE=$(db_query "select software_release from partition11 where reference='slappart0';") SOFTWARE_RELEASE=$(echo "$OLD_SOFTWARE_RELEASE" | sed 's/\/\(slappart\|test0-\)[0-9][0-9]*\//\/'"$PARTITION"'\//') db_query "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;" db_query "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || db_query "delete from software11 where url='$OLD_SOFTWARE_RELEASE';" # Change slapproxy database to have all instances stopped db_query "update partition11 set requested_state='stopped';" # Change slapproxy database to get correct IPs IPV4='{{ ipv4 }}' IPV6='{{ ipv6 }}' db_query "update partition_network11 set address='$IPV4' where netmask='255.255.255.255';" db_query "update partition_network11 set address='$IPV6' where netmask='ffff:ffff:ffff::';" MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}' log_message "Removing old supervisord service description files..." # XXX: Path hardcoded in slapos.core rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true SLAPOSCFG='{{ supervisord["slapos-cfg"] }}' SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}' SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}' contain_software_release=0 SOFTWARE_RELEASES_COUNT=$("$SQLITE3" "$DATABASE" 'SELECT count(1) FROM software11 WHERE url != "";') if [ $SOFTWARE_RELEASES_COUNT -gt 0 ]; then contain_software_release=1 fi if [ $contain_software_release -eq 0 ]; then log_message "No Software Release were deployed, so skip to continue..." echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi log_message "Building newest Software Release..." "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || "$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDSRLOG" && false) contain_instance=0 for folder in $srv_directory/runner/instance/slappart*/; do if [ -f "$folder/buildout.cfg" ]; then contain_instance=1 fi done # If instance do not contains template.cfg it means the user contains no instance. # so it is safer to assume that he is using slaprunner for develop buildout rather them slapos. if [ $contain_instance -eq 0 ]; then log_message "None Instance were deployed with this software release, so skip to continue..." echo 0 > $RESTORE_EXIT_CODE_FILE exit 0 fi # Remove defined scripts to force buildout to recreate them to have updated paths rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true log_message "Fixing Instances as needed after import..." # XXX hardcoded "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || "$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" >/dev/null 2>&1 || (tail -n 200 "$SLAPGRIDCPLOG" && false) # Invoke defined scripts for each partition inside of slaprunner log_message "Invoke custom import scripts defined by each instances..." for partition in "$srv_directory"/runner/instance/slappart*/ do script=$partition/srv/runner-import-restore if [ -x "$script" ]; then log_message "Running custom instance script : $script..." "$script" fi done # Change back slapproxy database to have all instances started log_message "Set instances as to start after takeover..." db_query "update partition11 set requested_state='started';" # Write exit code to an arbitrary file that will be checked by promise/monitor log_message "Writing status file... End" echo 0 > $RESTORE_EXIT_CODE_FILE exit 0