Commit 4c127fdd by Kirill Smelkov

gitlab: Setup sidekiq service

Sidekiq[1] is used in GitLab as background jobs manager - i.e. if a
request handler needs to spawn some non-light job - it adds it to
sidekiq queue (in Redis) and relies on sidekiq service to later pick
this job up and execute it.

The service is setup with just to run bin/gitlab-sidekiq with
appropriate queues (extracted from omnibus-gitlab) and appropriate
settings to controlling GitLab's sidekiq Out-Of-Memory killer[2].

NOTE Unlike unicorn OOM killer, Sidekiq memory killer just makes sidekiq
    processes to be SIGKILL terminated and relies on managing service to
    restart it. In slapos we don't have mechanism to set autorestart=true,
    nor bang/watchdog currently work with slapproxy, so we setup to do
    such monitoring ourselves manually with here-introduced
    watcher-sigkill program.

NOTE2 sidekiq promise, because it is rake/gitlab based, is slow to
    load/run and thus is put into etc/promise.slow/

[1] http://sidekiq.org/
[2] https://gitlab.com/gitlab-org/gitlab-ce/blob/1322bd78/doc/operations/sidekiq_memory_killer.md

/cc @kazuhiko, @jerome
1 parent 76e371cd
......@@ -59,6 +59,12 @@ configuration.git_max_size =
configuration.git_timeout =
# sidekiq
configuration.sidekiq_shutdown_timeout = 4
configuration.sidekiq_concurrency = 25
configuration.sidekiq_memory_killer_max_rss = 1000000
# unicorn
configuration.unicorn_worker_timeout = 60
configuration.unicorn_worker_processes = 2
......
......@@ -22,6 +22,7 @@ parts =
service-gitlab-workhorse
service-unicorn
service-sidekiq
service-postgresql
service-redis
......@@ -235,6 +236,7 @@ environment =
BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile
HOME = ${directory:home}
RAILS_ENV = production
SIDEKIQ_MEMORY_KILLER_MAX_RSS = ${instance-parameter:configuration.sidekiq_memory_killer_max_rss}
# NOTE sys.argv[1:] implicitly appended
# (by slapos.recipe.librecipe.execute.generic_exec() at runtime)
......@@ -567,6 +569,62 @@ log = ${gitlab:log}/*.log
log = ${gitlab-shell:log}/*.log
#######################################
# sidekiq background jobs manager #
#######################################
[sidekiq-dir]
recipe = slapos.cookbook:mkdirectory
log = ${directory:log}/sidekiq
[sidekiq]
log = ${sidekiq-dir:log}
# NOTE see queue list here:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
# (last updated for ominbus-gitlab 8.2.3+ce.0-0-g8eda093)
[service-sidekiq]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/sidekiq
command-line =
# NOTE Sidekiq memory killer just makes sidekiq processes to be SIGKILL
# terminated and relies on managing service to restart it. In slapos we don't
# have mechanism to set autorestart=true, nor bang/watchdog currently work with
# slapproxy, so we do the monitoring ourselves.
{{ watcher_sigkill }}
${gitlab-sidekiq:wrapper-path}
# XXX -q runner ? (present in gitlab-ce/Procfile but not in omnibus)
# XXX -P ? (pidfile)
-e production
-r ${gitlab-work:location}
-t ${instance-parameter:configuration.sidekiq_shutdown_timeout}
-c ${instance-parameter:configuration.sidekiq_concurrency}
-L ${sidekiq:log}/sidekiq.log
-q post_receive
-q mailer
-q archive_repo
-q system_hook
-q project_web_hook
-q gitlab_shell
-q incoming_email
-q common
-q default
depend =
${promise-sidekiq:recipe}
${logrotate-entry-sidekiq:recipe}
[promise-sidekiq]
<= promise-rakebase
command-line = ${:rake} gitlab:sidekiq:check
[logrotate-entry-sidekiq]
<= logrotate-entry
log = ${sidekiq:log}/*.log
#############
# cron #
......
......@@ -43,6 +43,7 @@ context =
raw postgresql_location ${postgresql92:location}
raw redis_binprefix ${redis28:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
raw watcher_sigkill ${watcher-sigkill:rendered}
# config files
raw config_ru_in ${config.ru.in:target}
......
......@@ -43,6 +43,7 @@ parts =
bash
curl
watcher-sigkill
gzip
dcron-output
logrotate
......@@ -172,6 +173,15 @@ url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
# md5sum = TODO
[watcher-sigkill]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
# md5sum = TODO
context =
section bash bash
# macro: download a file named as section name
#
......
#!{{ bash.location }}/bin/bash
# run program under SIGKILL watchdog
# watcher-sigkill <prog> [<progargs> ...]
#
# if the program terminates with SIGKILL - it is restarted after grace period.
# if the program terminates otherwise - whole process terminates.
if [ "$#" -lt 1 ]; then
echo "Usage: watcher-sigkill <prog> [<progargs> ...]" 1>&2
exit 1
fi
prog="$@"
progpid=""
killexit="137" # = 128 + 9 (exit code of process terminated by SIGKILL)
# make sure to terminate children, when we exit.
# needed for e.g. when `slapos node stop ...` kills us.
trap 'atexit' EXIT
atexit() {
jobs="$(jobs -p)"
test -n "$jobs" && kill $jobs
}
# run prog under monitoring
while true; do
echo "run $prog"
$prog &
progpid=$!
echo "wait $progpid"
wait $progpid
status=$?
echo "-> $status"
# if program terminated not by SIGKILL - exit
if [ "$status" != "$killexit" ] ; then
echo "exit $status"
exit "$status"
fi
# otherwise sleep a bit and restart
sleep 1
done
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!