Commit d3a66c85 by Alain Takoudjou

gitlab: add support for resiliency backup/inport from webrunner

1 parent 17d1f0fc
......@@ -5,6 +5,8 @@ extends = {{ instance_gitlab_cfg }}
parts +=
cron-entry-gitlab-backup
resiliency-exclude-file
gitlab-resiliency-restore-script
# -export specific instance parameters
[instance-parameter]
......@@ -29,10 +31,49 @@ environment =
# XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
# `which` command in gitlab-backup, `chrt` in xnice, ...
# and `/bin` for `sed` command in gitlab-backup restore
PATH=/bin:/usr/bin:${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin
PATH=${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin:/bin:/usr/bin
[cron-entry-gitlab-backup]
<= cron-entry
# run backup script on a regular basis (given as instance parameter)
frequency = ${instance-parameter:configuration.backup_frequency}
command = ${exporter:wrapper-path}
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: gitlab-shell-work*
gitlab-work*
var/backup/**
var/repositories*
var/repositories/**
srv/postgresql/**
srv/postgresql
etc/service/postgres-start
srv/redis/**
srv/unicorn/unicorn.socket
output = ${directory:srv}/exporter.exclude
[gitlab-resiliency-restore-script]
# script run by resilient stack to restore gitlab instance
# this section should be added only on runner import instance
recipe = slapos.recipe.template:jinja2
mode = 0700
template= {{ gitlab_restore_sh_in }}
rendered= ${directory:srv}/runner-import-restore
context =
raw bash_bin {{ bash_bin }}
  • Dangerous variable expansoin: if bash_bin contains one of the many buildout-magic characters, it will break this file.

    Expant in a separate section, wrapping expanded value in a dumps() call (available as a global) to avoid this, and reference the section with section type keyword (requires rendered template change), or individual entries with the key type keyword (no rendered template change needed, but more verbose).

    [EDIT]: and same applies of course for other lines in this file (and in many other places, sadly)

    Edited
raw go_work_bin {{ gopath_bin }}
raw git_location {{ git_location }}
raw bin_directory ${directory:bin}
  • Dangerous variable expansion: if such value is multi-line, it will bread this file. Use the key type keyword instead, and leave ${ and } out of the value declaration.

    [EDIT]: and same applies of course for other lines in this file (and in many other places, sadly)

    Edited
raw etc_directory ${directory:etc}
raw run_directory ${directory:run}
raw postgress_script ${service-postgresql:services}/postgres-start
raw redis_script ${service-redis:wrapper}
raw unicorn_script ${service-unicorn:wrapper-path}
raw sidekiq_script ${service-sidekiq:wrapper-path}
raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git}
raw redis_pid_file ${service-redis:pid_file}
raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid
raw gitlab_work_location ${gitlab-work:location}
raw promise_lab_location ${directory:promise.slow}
......@@ -71,6 +71,7 @@ context =
raw rack_attack_rb_in ${rack_attack.rb.in:target}
raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target}
$${:context-extra}
......
......@@ -252,13 +252,14 @@ recipe = zc.recipe.egg
eggs =
plone.recipe.command
cns.recipe.symlink
collective.recipe.template
[instance.cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
md5sum = 2329ddc4934e900785aa669adc214c23
md5sum = b2af6c8df8039ca7415ce433b6091919
# macro: download a shell script and put it rendered into <software>/bin/
[binsh]
......@@ -325,7 +326,7 @@ md5sum = b05fad928ffbb689b4415837525c62d1
[instance-gitlab-export.cfg.in]
<= download-file
md5sum = 00bd4ddf75a40a9d18d8243289f68ee9
md5sum = 319d7dbe3ad9b260c1e292cfc0d13b11
[macrolib.cfg.in]
<= download-file
......@@ -351,6 +352,10 @@ md5sum = 7c89a730889e3224548d9abe51a2d719
<= download-template
md5sum = c67ea492e17f774d0e18f1217338a55f
[template-gitlab-resiliency-restore.sh.in]
<= download-template
md5sum = 99c4101b1fcc9a099cf94d09c33d70d0
[unicorn.rb.in]
<= download-template
md5sum = 83921db1835d9e81cbbe808631cc40a9
......
#!{{ bash_bin }}
# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
# DaTA WILL BE ERASED
set -e
echo "###################################################################################"
echo "# #"
echo "# Warning: DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE DaTA WILL BE ERASED !!! #"
echo "# #"
echo "###################################################################################"
echo -e "\nWill start in 10 seconds, cancel execution if you didn't want to run this script."
sleep 10
postgres_executable="{{ postgress_script }}"
redis_executable="{{ redis_script }}"
git_backup_directory="{{ gitlab_backup_dir }}"
redis_pid_file="{{ redis_pid_file }}"
postgres_pid_file="{{ postgres_pid_file }}"
bin_location="{{ bin_directory }}"
run_location="{{ run_directory }}"
git_location="{{ git_location }}"
go_work_bin="{{ go_work_bin }}"
etc_location="{{ etc_directory }}"
gitlab_work="{{ gitlab_work_location }}"
promise_check="{{ promise_lab_location }}"
unicorn_script="{{ unicorn_script }}"
sidekiq_script="{{ sidekiq_script }}"
# export GIT_EXEC_PATH=$git_location/libexec/git-core/
check_process () {
pid_file=$1
pname=$2
if [ -e "$pid_file" ]; then
pid=$(head -n 1 $pid_file) > /dev/null 2>&1
if kill -0 "$pid"; then
echo "$pname is already running with pid $pid. Aborting."
exit 1
fi
fi
}
kill_process () {
pid=$1
R=0
kill -0 "$pid" > /dev/null 2>&1 || R=$?
if [ $R -eq 0 ]; then
kill -TERM $pid
fi
}
check_process $postgres_pid_file "Postgres"
check_process $redis_pid_file "Redis"
check_process $run_location/unicorn.pid "Unicorn"
echo "Starting Postgres..."
$postgres_executable &
postgres_pid=$!
trap "echo 'kill $postgres_pid" EXIT TERM INT
echo "Starting Redis server..."
$redis_executable &
redis_pid=$!
trap "kill $redis_pid" EXIT TERM INT
echo "[OK]"
echo "Restoring gitlab data..."
# XXX - workaround until this problem is fixed on runner1
sed -ie "s/connection.execute('TRUNCATE schema_migrations')\s*$/connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'/g" $gitlab_work/lib/tasks/gitlab/db.rake
cd $git_backup_directory
PATH=$bin_location:$go_work_bin:$git_location/bin:$PATH gitlab-backup restore -vupok -go HEAD
echo "Checking gitlab promises..."
echo "[info] Not all promises are checked!"
$promise_check/gitlab-app
echo "Starting Unicorn to check gitlab-shell promise..."
$unicorn_script &
unicorn_pid=$!
trap "kill $unicorn_pid" EXIT TERM INT
sleep 60
if [ -s "$run_location/unicorn.pid" ]; then
unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
trap "kill $unicorn_ppid" EXIT TERM INT
fi
$promise_check/gitlab-shell
#echo "starting Sidekiq to check sidekiq promise..."
#$sidekiq_script &
#sidekiq_pid=$!
#trap "kill $sidekiq_pid" EXIT TERM INT
#$promise_check/sidekiq
kill_process $postgres_pid
kill_process $redis_pid
kill_process $unicorn_pid
RESTORE_EXIT_CODE=$?
if [ $RESTORE_EXIT_CODE -eq 0 ]; then
echo 'Backup restoration successfully completed.'
else
echo 'Backup restoration failed.'
fi
exit $RESTORE_EXIT_CODE
\ No newline at end of file
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!