gitlab: add support for resiliency backup/inport from webrunner

software/gitlab/instance-gitlab-export.cfg.in

@@ -5,6 +5,8 @@ extends = {{ instance_gitlab_cfg }}
 
 parts +=
     cron-entry-gitlab-backup
+    resiliency-exclude-file
+    gitlab-resiliency-restore-script
 
 # -export specific instance parameters
 [instance-parameter]
@@ -29,10 +31,49 @@ environment   =
 # XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
 # `which` command in gitlab-backup, `chrt` in xnice, ...
 # and `/bin` for `sed` command in gitlab-backup restore
-  PATH=/bin:/usr/bin:${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin
+  PATH=${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin:/bin:/usr/bin
 
 [cron-entry-gitlab-backup]
 <= cron-entry
 # run backup script on a regular basis (given as instance parameter)
 frequency = ${instance-parameter:configuration.backup_frequency}
 command	= ${exporter:wrapper-path}
+
+[resiliency-exclude-file]
+# Generate rdiff exclude file in case of resiliency
+recipe = collective.recipe.template
+input = inline: gitlab-shell-work*
+  gitlab-work*
+  var/backup/**
+  var/repositories*
+  var/repositories/**
+  srv/postgresql/**
+  srv/postgresql
+  etc/service/postgres-start
+  srv/redis/**
+  srv/unicorn/unicorn.socket
+output = ${directory:srv}/exporter.exclude
+
+[gitlab-resiliency-restore-script]
+# script run by resilient stack to restore gitlab instance
+# this section should be added only on runner import instance
+recipe  = slapos.recipe.template:jinja2
+mode    = 0700
+template= {{ gitlab_restore_sh_in }}
+rendered= ${directory:srv}/runner-import-restore
+context =
+  raw     bash_bin                {{ bash_bin }}
+  raw     go_work_bin             {{ gopath_bin }}
+  raw     git_location            {{ git_location }}
+  raw     bin_directory           ${directory:bin}
+  raw     etc_directory           ${directory:etc}
+  raw     run_directory           ${directory:run}
+  raw     postgress_script        ${service-postgresql:services}/postgres-start
+  raw     redis_script            ${service-redis:wrapper}
+  raw     unicorn_script          ${service-unicorn:wrapper-path}
+  raw     sidekiq_script          ${service-sidekiq:wrapper-path}
+  raw     gitlab_backup_dir       ${gitlab-backup-directory:backup-gitlab.git}
+  raw     redis_pid_file          ${service-redis:pid_file}
+  raw     postgres_pid_file       ${service-postgresql:pgdata-directory}/postmaster.pid
+  raw     gitlab_work_location    ${gitlab-work:location}
+  raw     promise_lab_location    ${directory:promise.slow}

software/gitlab/instance.cfg.in

@@ -71,6 +71,7 @@ context =
     raw rack_attack_rb_in           ${rack_attack.rb.in:target}
     raw resque_yml_in               ${resque.yml.in:target}
     raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
+    raw gitlab_restore_sh_in        ${template-gitlab-resiliency-restore.sh.in:target}
     raw unicorn_rb_in               ${unicorn.rb.in:target}
 
     $${:context-extra}

software/gitlab/software.cfg

@@ -252,13 +252,14 @@ recipe  = zc.recipe.egg
 eggs    =
     plone.recipe.command
     cns.recipe.symlink
+    collective.recipe.template
 
 
 [instance.cfg]
 recipe  = slapos.recipe.template
 url     = ${:_profile_base_location_}/instance.cfg.in
 output  = ${buildout:directory}/instance.cfg
-md5sum  = 2329ddc4934e900785aa669adc214c23
+md5sum  = b2af6c8df8039ca7415ce433b6091919
 
 # macro: download a shell script and put it rendered into <software>/bin/
 [binsh]
@@ -325,7 +326,7 @@ md5sum  = b05fad928ffbb689b4415837525c62d1
 
 [instance-gitlab-export.cfg.in]
 <= download-file
-md5sum  = 00bd4ddf75a40a9d18d8243289f68ee9
+md5sum  = 319d7dbe3ad9b260c1e292cfc0d13b11
 
 [macrolib.cfg.in]
 <= download-file
@@ -351,6 +352,10 @@ md5sum  = 7c89a730889e3224548d9abe51a2d719
 <= download-template
 md5sum  = c67ea492e17f774d0e18f1217338a55f
 
+[template-gitlab-resiliency-restore.sh.in]
+<= download-template
+md5sum  = 99c4101b1fcc9a099cf94d09c33d70d0
+
 [unicorn.rb.in]
 <= download-template
 md5sum  = 83921db1835d9e81cbbe808631cc40a9

software/gitlab/template/template-gitlab-resiliency-restore.sh.in

+#!{{ bash_bin }}
+
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# DaTA WILL BE ERASED
+
+set -e
+
+echo "###################################################################################"
+echo "#                                                                                 #"
+echo "#  Warning: DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE DaTA WILL BE ERASED !!! #"
+echo "#                                                                                 #"
+echo "###################################################################################"
+echo -e "\nWill start in 10 seconds, cancel execution if you didn't want to run this script."
+
+sleep 10
+
+postgres_executable="{{ postgress_script }}"
+redis_executable="{{ redis_script }}"
+git_backup_directory="{{ gitlab_backup_dir }}"
+redis_pid_file="{{ redis_pid_file }}"
+postgres_pid_file="{{ postgres_pid_file }}"
+
+bin_location="{{ bin_directory }}"
+run_location="{{ run_directory }}"
+git_location="{{ git_location }}"
+go_work_bin="{{ go_work_bin }}"
+etc_location="{{ etc_directory }}"
+gitlab_work="{{ gitlab_work_location }}"
+promise_check="{{ promise_lab_location }}"
+unicorn_script="{{ unicorn_script }}"
+sidekiq_script="{{ sidekiq_script }}"
+
+# export GIT_EXEC_PATH=$git_location/libexec/git-core/
+
+check_process () {
+  pid_file=$1
+  pname=$2
+  if [ -e "$pid_file" ]; then
+    pid=$(head -n 1 $pid_file) > /dev/null 2>&1
+    if kill -0 "$pid"; then
+      echo "$pname is already running with pid $pid. Aborting."
+      exit 1
+    fi
+  fi
+}
+
+kill_process () {
+  pid=$1
+  R=0
+  kill -0 "$pid" > /dev/null 2>&1 || R=$?
+  if [ $R -eq 0 ]; then
+    kill -TERM $pid
+  fi
+}
+
+check_process $postgres_pid_file "Postgres"
+check_process $redis_pid_file "Redis"
+check_process $run_location/unicorn.pid "Unicorn"
+
+echo "Starting Postgres..."
+$postgres_executable &
+postgres_pid=$!
+trap "echo 'kill $postgres_pid" EXIT TERM INT
+
+echo "Starting Redis server..."
+$redis_executable &
+redis_pid=$!
+trap "kill $redis_pid" EXIT TERM INT
+echo "[OK]"
+echo "Restoring gitlab data..."
+
+# XXX - workaround until this problem is fixed on runner1
+sed -ie "s/connection.execute('TRUNCATE schema_migrations')\s*$/connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'/g" $gitlab_work/lib/tasks/gitlab/db.rake
+
+cd $git_backup_directory
+PATH=$bin_location:$go_work_bin:$git_location/bin:$PATH gitlab-backup restore -vupok -go HEAD
+
+echo "Checking gitlab promises..."
+echo "[info] Not all promises are checked!"
+$promise_check/gitlab-app
+
+echo "Starting Unicorn to check gitlab-shell promise..."
+$unicorn_script &
+unicorn_pid=$!
+trap "kill $unicorn_pid" EXIT TERM INT
+sleep 60
+if [ -s "$run_location/unicorn.pid" ]; then
+  unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
+  trap "kill $unicorn_ppid" EXIT TERM INT
+fi
+$promise_check/gitlab-shell
+
+#echo "starting Sidekiq to check sidekiq promise..."
+#$sidekiq_script &
+#sidekiq_pid=$!
+#trap "kill $sidekiq_pid" EXIT TERM INT
+#$promise_check/sidekiq
+
+kill_process $postgres_pid
+kill_process $redis_pid
+kill_process $unicorn_pid
+
+RESTORE_EXIT_CODE=$?
+
+if [ $RESTORE_EXIT_CODE -eq 0 ]; then
+  echo 'Backup restoration successfully completed.'
+else
+  echo 'Backup restoration failed.'
+fi
+
+exit $RESTORE_EXIT_CODE
\ No newline at end of file

[Vincent Pelletier]

Dangerous variable expansoin: if bash_bin contains one of the many buildout-magic characters, it will break this file.

Expant in a separate section, wrapping expanded value in a dumps() call (available as a global) to avoid this, and reference the section with section type keyword (requires rendered template change), or individual entries with the key type keyword (no rendered template change needed, but more verbose).

[EDIT]: and same applies of course for other lines in this file (and in many other places, sadly)

[Vincent Pelletier]

Dangerous variable expansion: if such value is multi-line, it will bread this file. Use the key type keyword instead, and leave ${ and } out of the value declaration.

[EDIT]: and same applies of course for other lines in this file (and in many other places, sadly)

[Vincent Pelletier]

Likewise.

[EDIT]: and same applies of course for other lines in this file (and in many other places, sadly)