Commit ab6d2f28 by Kirill Smelkov

gitlab: Add empty instance

Add stub instance configuration which just establishes a way to have
several software types(*), pass all needed info from software to
instance, organizes base directory and establishes log rotation base for
upcoming services.

Log rotation is done with the help of cron periodicallly calling
logrotate. The rotation is done in "copytruncate" mode - i.e. log file
is not moved away and signal sent for service to reopen it, but instead
log content is just copied to outside and there is no need for a service
to reopen it's log file.

The reason it is done this way, is that there is a chance of not
handling such "reopen-log-file" callbacks correctly on a service side,
and so the net is full of crashing reports, e.g. like this:

    http://serverfault.com/questions/627521/why-is-logrotate-causing-apache-to-seg-fault-each-time

That's why we take a safer approach instead, even if "copytruncate" mode
is risking to loose several log entries(**) on rotation.

NOTE services will organize log rotation with just

    [logrotate-entry-<service>]
    <= logrotate-entry
    log     = path/to/log/files/*.log

For this to work some "!py!" magic (our way to serialize object into
executable python and process it in buildout recipes) is used to process
section names.

The approach trick is also used for cron, e.g. logrotate registers to
cron this way:

    [cron-entry-logrotate]
    <= cron-entry
    time    = daily
    command = ${logrotate:wrapper}

NOTE2 instance md5 are not fixed yet - we'll fix them after applying all
    patches in gitlab series.

(*) for now there is only 1 - "gitlab", but we'll need to have "-export"
    and "-import" for resiliency in the future.

(**) ideally such things should be done with logfs - a filesystem
    specializeing in logging - for client services it will look like as
    they just continue to write to log file, and on log service side, the
    rotation can happen, all transparent to client service.

/cc @kazuhiko, @jerome
1 parent 5e971c58
# GitLab instance
[buildout]
parts =
directory
service-cron
# std stuff for slapos instance
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
##################################
# GitLab instance parameters #
##################################
[instance-parameter]
# std stuff to fetch slapos instance parameters
recipe = slapos.cookbook:slapconfiguration
computer= ${slap-connection:computer-id}
partition=${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
#############################
# GitLab instance setup #
#############################
# 1. directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
etc = ${:home}/etc
var = ${:home}/var
log = ${:var}/log
run = ${:var}/run
srv = ${:home}/srv
# slapos startup/service/promise scripts live here:
startup = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
# 2. services
#############
# cron #
#############
[cron-dir]
recipe = slapos.cookbook:mkdirectory
cron.d = ${directory:etc}/cron.d
crontabs= ${directory:srv}/cron/crontabs
cronstamps = ${directory:var}/cron/cronstamps
log = ${directory:log}/cron
[service-cron]
recipe = slapos.cookbook:cron
binary = ${directory:service}/crond
cron-entries = ${cron-dir:cron.d}
crontabs = ${cron-dir:crontabs}
cronstamps = ${cron-dir:cronstamps}
catcher = ${cron-simplelogger:wrapper}
dcrond-binary = {{ dcron_bin }}
depends =
${logrotate-entry-cron:recipe}
# "mailer" that cron uses to emit messages to logfile
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = ${directory:bin}/${:_buildout_section_name_}
log = ${cron-dir:log}/cron.log
# base entry for clients who registers to cron
[cron-entry]
recipe = slapos.cookbook:cron.d
# name = <section-name>.strip_prefix('cron-entry-')
# XXX len() is not available in !py! - 11 hardcoded
name = !py!'${:_buildout_section_name_}' [11:]
# NOTE _not_ ${service-cron:cron-entries} - though the value is the same we do
# not want service-cron to be instantiated just if a cron-entry is registered.
cron-entries = ${cron-dir:cron.d}
# cron logs are also rotated
[logrotate-entry-cron]
<= logrotate-entry
log = ${cron-dir:log}/*.log
#######################################
# logrotate base for all services #
#######################################
[logrotate-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/logrotate
entries = ${directory:etc}/logrotate.d
[logrotate]
recipe = slapos.cookbook:logrotate
wrapper = ${directory:bin}/${:_buildout_section_name_}
conf = ${directory:etc}/logrotate.conf
logrotate-entries = ${logrotate-dir:entries}
state-file = ${logrotate-dir:srv}/logrotate.status
logrotate-binary = {{ logrotate_bin }}
gzip-binary = {{ gzip_bin }}
gunzip-binary = {{ gunzip_bin }}
depend = ${cron-entry-logrotate:recipe}
# base entry for clients who registers to logrotate
[logrotate-entry]
recipe = slapos.cookbook:logrotate.d
logrotate-entries = ${logrotate:logrotate-entries}
# name = <section-name>.strip_prefix('logrotate-entry-')
# XXX len is not available in !py! - 16 hardcoded
name = !py!'${:_buildout_section_name_}'[16:]
# NOTE frequency is hardcoded to `daily` in slapos.cookbook:logrotate.d
# NOTE backup is also used to add custom logrotate options (hack)
backup = ...
# TODO settle whether we need/want olddir or not
noolddir
# override create emitted by slapos.cookbook:logrotate.d
nocreate
# do not move log file and this way we do not need to signal its program to
# reopen the log. There are a lot of bugs when on such reopen / restart /
# graceful-restart something bad happens. Even if copytruncate is a bit racy
# and can loose some data, it is better to keep the system the stable way.
copytruncate
# hook logrotate into cron
[cron-entry-logrotate]
<= cron-entry
time = daily
command = ${logrotate:wrapper}
# GitLab "switch-softwaretype" instance
[buildout]
parts = switch-softwaretype
# std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${instance-gitlab.cfg:rendered}
# TODO -export, -import, -pull-backup
[instance-gitlab.cfg]
recipe = slapos.recipe.template:jinja2
mode = 0644
template= ${instance-gitlab.cfg.in:target}
rendered= $${buildout:directory}/instance-gitlab.cfg
context =
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
# program binaries
raw dcron_bin ${dcron-output:crond}
raw gunzip_bin ${gzip:location}/bin/gunzip
raw gzip_bin ${gzip:location}/bin/gzip
raw logrotate_bin ${logrotate:location}/usr/sbin/logrotate
......@@ -13,6 +13,11 @@ extends =
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
# for instance
../../component/gzip/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
parts =
ruby2.1
golang15
......@@ -28,6 +33,16 @@ parts =
gitlab/vendor/bundle
gitlab-workhorse
# for instance
instance.cfg
slapos-cookbook
eggs
gzip
dcron-output
logrotate
############################
# Software compilation #
......@@ -135,7 +150,41 @@ environment =
PATH=${golang15:location}/bin:%(PATH)s
###############################
# Trampoline for instance #
###############################
# eggs for instance.cfg
[eggs]
recipe = zc.recipe.egg
eggs =
plone.recipe.command
[instance.cfg]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
# md5sum = TODO
# macro: download a file named as section name
#
# [filename]
# <= download-file
# md5sum = ...
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[instance-gitlab.cfg.in]
<= download-file
# md5sum = TODO
[versions]
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2
slapos.recipe.template = 2.9
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!