Commit ae7b4120 by iv Committed by Kirill Smelkov

gitlab: start automation of gitlab-backup

  - git-clone and build git-backup sources
  - add a cron job doing `gitlab-backup-pull` every 4 hours by default
  - add instance parameter to change backup frequency
  - add xnice for using less resources while backuping
  - add lock to avoid concurrent calls of backup script
  - add a specific software type `gitlab-export` doing all this instead of in
    `default`; this can be changed in Services > parameters (kirr's changes)

`coreutils`, `grep` and `tar` components are used by `gitlab-backup-pull`
script, as it is called from a cron and the PATH has to include these
binaries paths
1 parent 25acc473
#!{{ bash.location }}/bin/bash -e
# export gitlab site via `gitlab-backup pull` to a "for-export" git-backup repository
# gitlab-export <exportto-repo>
#
# NOTE gitlab-backup, gitlab-rails, ... all have to be on $PATH.
# which site to export is determined by which gitlab-rails is on $PATH.
if [ "$#" -ne 1 ]; then
echo "Usage: gitlab-export <exportto-repo>" 1>&2
exit 1
fi
exportto_repo="$1"
# create / setup export repository if it does not exist yet
mkdir -p "$exportto_repo"
cd "$exportto_repo"
# verify we are in a git repository
if ! git rev-parse --is-inside-git-dir ; then
git init --bare
fi
exec gitlab-backup pull
# GitLab instance + site export support
[buildout]
extends = {{ instance_gitlab_cfg }}
# TODO + ${pbsready-export:output}
parts +=
cron-entry-gitlab-backup
# -export specific instance parameters
[instance-parameter]
# cron frequency for gitlab backup (default: every 4h)
configuration.backup_frequency = 0 */4 * * *
[gitlab-backup-directory]
recipe = slapos.cookbook:mkdirectory
srv = ${buildout:directory}/srv
backup = ${:srv}/backup
backup-gitlab.git = ${:backup}/backup-gitlab.git
var = ${buildout:directory}/var
pid = ${:var}/pid
# instance exporter script
[exporter]
recipe = slapos.cookbook:wrapper
wrapper-path = ${buildout:directory}/bin/gitlab-exporter
command-line = {{ xnice_repository_location }}/bin/xnice {{ gitlab_export }} ${gitlab-backup-directory:backup-gitlab.git}
pidfile = ${gitlab-backup-directory:pid}/gitlab-exporter.pid
environment =
# XXX: `/usr/bin` has to be in the PATH environment variable to be able to use
# `which` command in gitlab-backup, `chrt` in xnice, ...
PATH=/usr/bin:${buildout:directory}/bin:{{ coreutils_location }}/bin:{{ grep_location }}/bin:{{ tar_location }}/bin:{{ gzip_location }}/bin:{{ gopath_bin }}:{{ git_location }}/bin
[cron-entry-gitlab-backup]
<= cron-entry
# run backup script on a regular basis (given as instance parameter)
frequency = ${instance-parameter:configuration.backup_frequency}
command = ${exporter:wrapper-path}
......@@ -10,15 +10,17 @@ offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-gitlab.cfg:rendered}
# TODO -export, -import, -pull-backup
gitlab = $${instance-gitlab.cfg:rendered}
gitlab-export = $${instance-gitlab-export.cfg:rendered}
default = $${:gitlab}
# TODO -import, -pull-backup
[instance-gitlab.cfg]
# macro: render instance-*.cfg from instance-*.cfg.in
[instance-cfg]
recipe = slapos.recipe.template:jinja2
mode = 0644
template= ${instance-gitlab.cfg.in:target}
rendered= $${buildout:directory}/instance-gitlab.cfg
rendered= $${buildout:directory}/$${:_buildout_section_name_}
context =
import os os
import pwd pwd
......@@ -33,12 +35,16 @@ context =
raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle}
raw coreutils_location ${coreutils:location}
raw curl_bin ${curl:location}/bin/curl
raw dcron_bin ${dcron-output:crond}
raw git ${git:location}/bin/git
raw git_location ${git:location}
raw gitlab_export ${gitlab-export:rendered}
raw gitlab_workhorse ${gitlab-workhorse:location}/gitlab-workhorse
raw gopath_bin ${gopath:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location}
raw gzip_bin ${gzip:location}/bin/gzip
raw gzip_location ${gzip:location}
raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate
......@@ -48,7 +54,9 @@ context =
raw postgresql_location ${postgresql92:location}
raw redis_binprefix ${redis28:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location}
raw watcher_sigkill ${watcher-sigkill:rendered}
raw xnice_repository_location ${xnice-repository:location}
# config files
raw database_yml_in ${database.yml.in:target}
......@@ -64,3 +72,16 @@ context =
raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target}
$${:context-extra}
context-extra =
[instance-gitlab.cfg]
<= instance-cfg
template= ${instance-gitlab.cfg.in:target}
[instance-gitlab-export.cfg]
<= instance-cfg
template= ${instance-gitlab-export.cfg.in:target}
context-extra =
raw instance_gitlab_cfg $${instance-gitlab.cfg:rendered}
......@@ -14,9 +14,12 @@ extends =
../../component/nginx/buildout.cfg
# for instance
../../component/coreutils/buildout.cfg
../../component/bash/buildout.cfg
../../component/grep/buildout.cfg
../../component/bzip2/buildout.cfg
../../component/curl/buildout.cfg
../../component/tar/buildout.cfg
../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
......@@ -36,6 +39,7 @@ parts =
gitlab-shell/vendor
gitlab/vendor/bundle
gitlab-workhorse
git-backup
# for instance
instance.cfg
......@@ -50,6 +54,7 @@ parts =
bash
curl
watcher-sigkill
gitlab-export
gzip
dcron-output
logrotate
......@@ -158,6 +163,52 @@ make-binary =
make-targets= cd ${:path} &&
${:bundle} install --deployment --without development test mysql kerberos
# directories and repositories required by gitlab-backup
[gopath]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
[go-git-repository]
<= git-repository
repository = https://${:go.importpath}.git
location = ${gopath:src}/${:go.importpath}
[git2go-repository]
<= go-git-repository
go.importpath = github.com/libgit2/git2go
# branch 'next' is required by git-backup
revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
[git-backup-repository]
<= go-git-repository
go.importpath = lab.nexedi.com/kirr/git-backup
revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
[git-backup]
# install git2go, git-backup, gitlab-backup in <gopath>/bin
recipe = slapos.recipe.cmmi
path = ${git-backup-repository:location}
configure-command = :
make-binary =
make-targets= cd ${git2go-repository:location}
&& git submodule update --init
&& make install
&& install -d ${gopath:bin}
&& go install ${git-backup-repository:go.importpath}
&& cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin}
environment =
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gopath:directory}
[xnice-repository]
# to get kirr's misc repo containing xnice script for executing processes
# with lower priority (used for backup script inside the cron)
<= git-repository
repository = https://lab.nexedi.com/kirr/misc.git
revision = 4073572ea700bf1b115f3a135aebebe5b3b824e4
location = ${buildout:parts-directory}/misc
# build needed-by-gitlab-shell gems via bundler
# ( there is not vendor/ dir in gitlab-shell, so to avoid having buildout error
......@@ -201,17 +252,26 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
md5sum = ef85f02c4f6070c586d773b859a2f4e2
md5sum = b99a99b161c0b292845002fc3fee50cd
[watcher-sigkill]
# macro: download a shell script and put it rendered into <software>/bin/
[binsh]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = 2986dcb006dc9e8508ff81f646656131
context =
section bash bash
[watcher-sigkill]
<= binsh
md5sum = 2986dcb006dc9e8508ff81f646656131
[gitlab-export]
<= binsh
md5sum = a7b32680e80f34276f0a32a5e22dad50
# macro: download a file named as section name
#
......@@ -257,6 +317,10 @@ md5sum = 176939a6428a7aca4767a36421b0af2b
<= download-file
md5sum = 89914e4a225f6cdebfa196d46359f6f2
[instance-gitlab-export.cfg.in]
<= download-file
md5sum = a295e524046a9e2af569ad4b55f210cd
[macrolib.cfg.in]
<= download-file
md5sum = a56a44e96f65f5ed20211bb6a54279f4
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!