gitlab: Setup sidekiq service

Sidekiq[1] is used in GitLab as background jobs manager - i.e. if a
request handler needs to spawn some non-light job - it adds it to
sidekiq queue (in Redis) and relies on sidekiq service to later pick
this job up and execute it.

The service is setup with just to run bin/gitlab-sidekiq with
appropriate queues (extracted from omnibus-gitlab) and appropriate
settings to controlling GitLab's sidekiq Out-Of-Memory killer[2].

NOTE Unlike unicorn OOM killer, Sidekiq memory killer just makes sidekiq
    processes to be SIGKILL terminated and relies on managing service to
    restart it. In slapos we don't have mechanism to set autorestart=true,
    nor bang/watchdog currently work with slapproxy, so we setup to do
    such monitoring ourselves manually with here-introduced
    watcher-sigkill program.

NOTE2 sidekiq promise, because it is rake/gitlab based, is slow to
    load/run and thus is put into etc/promise.slow/

[1] http://sidekiq.org/
[2] https://gitlab.com/gitlab-org/gitlab-ce/blob/1322bd78/doc/operations/sidekiq_memory_killer.md

/cc @kazuhiko, @jerome

software/gitlab/gitlab-parameters.cfg

@@ -59,6 +59,12 @@ configuration.git_max_size              =
 configuration.git_timeout               =
 
 
+# sidekiq
+configuration.sidekiq_shutdown_timeout  = 4
+configuration.sidekiq_concurrency       = 25
+configuration.sidekiq_memory_killer_max_rss = 1000000
+
+
 # unicorn
 configuration.unicorn_worker_timeout    = 60
 configuration.unicorn_worker_processes  = 2

software/gitlab/instance-gitlab.cfg.in

@@ -22,6 +22,7 @@ parts =
 
     service-gitlab-workhorse
     service-unicorn
+    service-sidekiq
 
     service-postgresql
     service-redis
@@ -235,6 +236,7 @@ environment  =
     BUNDLE_GEMFILE = {{ gitlab_repository_location }}/Gemfile
     HOME = ${directory:home}
     RAILS_ENV = production
+    SIDEKIQ_MEMORY_KILLER_MAX_RSS = ${instance-parameter:configuration.sidekiq_memory_killer_max_rss}
 
 # NOTE sys.argv[1:] implicitly appended
 # (by slapos.recipe.librecipe.execute.generic_exec() at runtime)
@@ -567,6 +569,62 @@ log     = ${gitlab:log}/*.log
 log     = ${gitlab-shell:log}/*.log
 
 
+#######################################
+#   sidekiq background jobs manager   #
+#######################################
+[sidekiq-dir]
+recipe  = slapos.cookbook:mkdirectory
+log     = ${directory:log}/sidekiq
+
+[sidekiq]
+log     = ${sidekiq-dir:log}
+
+# NOTE see queue list here:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
+# (last updated for ominbus-gitlab 8.2.3+ce.0-0-g8eda093)
+[service-sidekiq]
+recipe  = slapos.cookbook:wrapper
+wrapper-path    = ${directory:service}/sidekiq
+command-line    =
+# NOTE Sidekiq memory killer just makes sidekiq processes to be SIGKILL
+# terminated and relies on managing service to restart it. In slapos we don't
+# have mechanism to set autorestart=true, nor bang/watchdog currently work with
+# slapproxy, so we do the monitoring ourselves.
+    {{ watcher_sigkill }}
+
+    ${gitlab-sidekiq:wrapper-path}
+# XXX -q runner ?  (present in gitlab-ce/Procfile  but not in omnibus)
+# XXX -P ?  (pidfile)
+    -e production
+    -r ${gitlab-work:location}
+    -t ${instance-parameter:configuration.sidekiq_shutdown_timeout}
+    -c ${instance-parameter:configuration.sidekiq_concurrency}
+    -L ${sidekiq:log}/sidekiq.log
+
+    -q post_receive
+    -q mailer
+    -q archive_repo
+    -q system_hook
+    -q project_web_hook
+    -q gitlab_shell
+    -q incoming_email
+    -q common
+    -q default
+
+depend  =
+    ${promise-sidekiq:recipe}
+    ${logrotate-entry-sidekiq:recipe}
+
+[promise-sidekiq]
+<= promise-rakebase
+command-line    = ${:rake} gitlab:sidekiq:check
+
+[logrotate-entry-sidekiq]
+<= logrotate-entry
+log     = ${sidekiq:log}/*.log
+
+
 
 #############
 #   cron    #

software/gitlab/instance.cfg.in

@@ -43,6 +43,7 @@ context =
     raw postgresql_location         ${postgresql92:location}
     raw redis_binprefix             ${redis28:location}/bin
     raw ruby_location               ${bundler-4gitlab:ruby-location}
+    raw watcher_sigkill             ${watcher-sigkill:rendered}
 
 # config files
     raw config_ru_in                ${config.ru.in:target}

software/gitlab/software.cfg

@@ -43,6 +43,7 @@ parts =
 
     bash
     curl
+    watcher-sigkill
     gzip
     dcron-output
     logrotate
@@ -172,6 +173,15 @@ url     = ${:_profile_base_location_}/instance.cfg.in
 output  = ${buildout:directory}/instance.cfg
 # md5sum = TODO
 
+[watcher-sigkill]
+recipe  = slapos.recipe.template:jinja2
+template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
+rendered= ${buildout:bin-directory}/${:_buildout_section_name_}
+mode    = 0755
+# md5sum = TODO
+context =
+    section bash    bash
+
 
 # macro: download a file named as section name
 #

software/gitlab/watcher-sigkill.in

+#!{{ bash.location }}/bin/bash
+# run program under SIGKILL watchdog
+# watcher-sigkill <prog> [<progargs> ...]
+#
+# if the program terminates with SIGKILL - it is restarted after grace period.
+# if the program terminates otherwise - whole process terminates.
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: watcher-sigkill <prog> [<progargs> ...]" 1>&2
+    exit 1
+fi
+prog="$@"
+
+progpid=""
+killexit="137"  # = 128 + 9  (exit code of process terminated by SIGKILL)
+
+# make sure to terminate children, when we exit.
+# needed for e.g. when `slapos node stop ...` kills us.
+trap 'atexit' EXIT
+atexit() {
+    jobs="$(jobs -p)"
+    test -n "$jobs" && kill $jobs
+}
+
+# run prog under monitoring
+while true; do
+    echo "run $prog"
+    $prog &
+    progpid=$!
+    echo "wait $progpid"
+    wait $progpid
+    status=$?
+    echo "-> $status"
+
+    # if program terminated not by SIGKILL - exit
+    if [ "$status" != "$killexit" ] ; then
+        echo "exit $status"
+        exit "$status"
+    fi
+
+    # otherwise sleep a bit and restart
+    sleep 1
+done