Skip to content

slapos
slapos
Commit af88b7c8 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

software/gitlab: upgrade to version 13.12.15

parent cd46fcf2
Expand all Show whitespace changes
Inline Side-by-side
Showing with 445 additions and 356 deletions
software/gitlab/buildout.hash.cfg
View file @ af88b7c8
......@@ -14,7 +14,7 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = d1ca30a1b910b6b775f4f95bd91123a6
md5sum = 956ae53af22b551fbb087415e835868b
[watcher]
_update_hash_filename_ = watcher.in
......@@ -30,35 +30,35 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
[gitconfig.in]
_update_hash_filename_ = template/gitconfig.in
md5sum = eb1230fee50067924ba89f4dc6e82fa9
md5sum = c559a24ab6281268b608ed3bccb8e4ce
[gitlab-parameters.cfg]
_update_hash_filename_ = gitlab-parameters.cfg
md5sum = cfda6d959bb90bf0b9c947383f45ce0a
md5sum = 95b18789111ed239146d243e39ffefbe
[gitlab-shell-config.yml.in]
_update_hash_filename_ = template/gitlab-shell-config.yml.in
md5sum = 69e8ed76b06233d11932a5c0ef16f03b
md5sum = 70d394305f4e1482a5c1a673b0762c6a
[gitlab-unicorn-startup.in]
_update_hash_filename_ = gitlab-unicorn-startup.in
md5sum = 705825e6d8c6b37699f1321805d09de3
[gitlab-puma-startup.in]
_update_hash_filename_ = gitlab-puma-startup.in
md5sum = 838209b9246fa86d6a21fef910f17e25
[gitlab.yml.in]
_update_hash_filename_ = template/gitlab.yml.in
md5sum = 673c393e6728a8d82e6b9a44886785a8
md5sum = 6f8df1467a6168bde7d2f5f42abb1012
[gitaly-config.toml.in]
_update_hash_filename_ = template/gitaly-config.toml.in
md5sum = 58e3d5bbda32583d00cd8f44ec0525b0
md5sum = d769ea27820e932c596c35bbbf3f2902
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = b913c4a1f199a87ad71da6d102adffa4
md5sum = 6d8d20ded84622339d49c60b0e61380c
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
md5sum = b8dea5ca4c6f9fc1ca54eb0265e1fdee
md5sum = c8231583d04bf0d3fe2d26230b94d78d
[macrolib.cfg.in]
_update_hash_filename_ = macrolib.cfg.in
......@@ -72,22 +72,18 @@ md5sum = 4980c1571a4dd7753aaa60d065270849
_update_hash_filename_ = template/nginx.conf.in
md5sum = 8c904510eb39dc212204f68f2b81b068
[rack_attack.rb.in]
_update_hash_filename_ = template/rack_attack.rb.in
md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
[resque.yml.in]
_update_hash_filename_ = template/resque.yml.in
md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in]
_update_hash_filename_ = template/smtp_settings.rb.in
md5sum = 4e1ced687a86e4cfff2dde91237e3942
md5sum = b1becd9ec4c2eeefe573af4bb53c9751
[template-gitlab-resiliency-restore.sh.in]
_update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
md5sum = 87f16b4f4a2370acada46b2751ef3366
md5sum = 8ce31a27e814e750dfd38c92a278fb9e
[unicorn.rb.in]
_update_hash_filename_ = template/unicorn.rb.in
md5sum = b4758129a8d0c47b2c3adb10fefb8275
[puma.rb.in]
_update_hash_filename_ = template/puma.rb.in
md5sum = 707c0c713af41518d21724c1be8efe22
software/gitlab/gitlab-parameters.cfg
View file @ af88b7c8
......@@ -15,8 +15,11 @@ configuration.external_url = https://lab.example.com
configuration.db_pool = 10
# rack-attack
configuration.rate_limit_requests_per_period = 10
configuration.rate_limit_period = 60
configuration.rack_attack_enable = true
configuration.rack_attack_max_retry = 10
configuration.rack_attack_find_time = 60
configuration.rack_attack_ban_time = 3600
configuration.rack_attack_ip_whitelist = 127.0.0.1
configuration.time_zone = UTC
......@@ -64,8 +67,10 @@ configuration.sidekiq_memory_killer_max_rss = 1000000
# unicorn
configuration.unicorn_worker_timeout = 60
configuration.unicorn_worker_processes = 2
configuration.puma_worker_timeout = 60
configuration.puma_worker_processes = 2
configuration.puma_min_threads = 1
configuration.puma_max_threads = 16
# unicorn advanced
configuration.unicorn_backlog_socket = 1024
......
software/gitlab/gitlab-unicorn-startup.in → software/gitlab/gitlab-puma-startup.in
View file @ af88b7c8
......@@ -38,6 +38,8 @@ echo "I: PostgreSQL ready." 1>&2
# make sure pg_trgm extension is enabled for gitlab db
psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' || die "pg_trgm setup failed"
psql -c 'CREATE EXTENSION IF NOT EXISTS btree_gist;' || die "btree_gist setup failed"
if echo "$pgtables" | grep -q '^Did not find any relations' ; then
$RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed"
fi
......@@ -70,8 +72,7 @@ $RAKE cache:clear || die "cache:clear failed"
force=yes $RAKE gitlab:shell:setup || die "gitlab:shell:setup failed"
# 3. finally exec to unicorn
exec {{ gitlab_unicorn }} \
-E production \
-c {{ unicorn_rb.output }} \
{{ gitlab_work.location }}/config.ru
# 3. finally exec to puma
exec {{ gitlab_puma }} \
-e production \
-C {{ puma_rb.output }}
software/gitlab/gowork.cfg
View file @ af88b7c8
......@@ -6,6 +6,7 @@ depends_gitfetch =
${go_github.com_pkg_errors:recipe}
${go_lab.nexedi.com_kirr_git-backup:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_golang.org_x_crypto:recipe}
[go_github.com_libgit2_git2go]
......@@ -13,7 +14,7 @@ depends_gitfetch =
go.importpath = github.com/libgit2/git2go
repository = https://github.com/libgit2/git2go.git
# branch 'next' is required by git-backup
revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8
revision = cbca5b82b8c22c08c183a1f44cad4b8b51ba6f25
[go_github.com_pkg_errors]
<= go-git-package
......@@ -25,10 +26,17 @@ revision = v0.8.0-12-g816c908556
<= go-git-package
go.importpath = lab.nexedi.com/kirr/git-backup
repository = https://lab.nexedi.com/kirr/git-backup.git
revision = da754af24da351291c99caa421a103db09e7a4c4
revision = 3327aedfbe903b210366105b8a4f9f85a313a0a3
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = 95433de34f
revision = 8299741f
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto.git
revision = 75b288015ac94e66e3d6715fb68a9b41bf046ec2
software/gitlab/instance-gitlab-export.cfg.in
View file @ af88b7c8
......@@ -54,7 +54,8 @@ input = inline: gitlab-shell-work*
srv/backup/logrotate/**
etc/service/postgres-start
srv/redis/**
srv/unicorn/unicorn.socket
srv/puma/puma.socket
.cache
output = ${directory:srv}/exporter.exclude
[gitlab-resiliency-restore-script]
......@@ -70,13 +71,14 @@ context =
raw git_location {{ git_location }}
raw bin_directory ${directory:bin}
raw etc_directory ${directory:etc}
raw run_directory ${directory:run}
raw var_directory ${directory:var}
raw postgress_script ${service-postgresql:services}/postgres-start
raw redis_script ${service-redis:wrapper}
raw unicorn_script ${service-unicorn:wrapper-path}
raw puma_script ${service-puma:wrapper-path}
raw sidekiq_script ${service-sidekiq:wrapper-path}
raw gitlab_backup_dir ${gitlab-backup-directory:backup-gitlab.git}
raw redis_pid_file ${service-redis:pid-file}
raw postgres_pid_file ${service-postgresql:pgdata-directory}/postmaster.pid
raw puma_pid_file ${puma:pid}/puma.pid
raw gitlab_work_location ${gitlab-work:location}
raw promise_lab_location ${directory:promise.slow}
software/gitlab/instance-gitlab.cfg.in
View file @ af88b7c8
......@@ -12,7 +12,7 @@ parts =
# gitlab-<prog>
# ? mailroom
{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %}
{% set gitlab_progv = 'rails rake puma sidekiq puma-startup' .split() %}
{% for prog in gitlab_progv %}
gitlab-{{ prog }}
{% endfor %}
......@@ -23,7 +23,7 @@ parts =
gitlab-shell-work
service-gitlab-workhorse
service-unicorn
service-puma
service-sidekiq
service-nginx
......@@ -51,29 +51,29 @@ offline = true
[worker-processes]
recipe = slapos.recipe.build
unicorn-worker-processes = {{ instance_parameter_dict['configuration.unicorn_worker_processes'] }}
puma-worker-processes = {{ instance_parameter_dict['configuration.puma_worker_processes'] }}
init =
import multiprocessing
worker_count = int(options['unicorn-worker-processes'])
worker_count = int(options['puma-worker-processes'])
if worker_count == 0:
# automatically load all available CPUs
worker_count = multiprocessing.cpu_count() + 1
worker_count = 2 if worker_count < 2 else worker_count
options['unicorn-worker-processes'] = worker_count
options['puma-worker-processes'] = worker_count
options['nginx-worker-processes'] = worker_count -1
[instance-parameter]
{#- There are dangerous keys like recipe, etc #}
{#- XXX: Some other approach would be useful #}
{%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert',
'configuration.unicorn_worker_processes', 'configuration.nginx_worker_processes'] %}
'configuration.puma_worker_processes', 'configuration.nginx_worker_processes'] %}
{%- for key, value in instance_parameter_dict.items() -%}
{%- if key not in DROP_KEY_LIST %}
{{ key }} = {{ value }}
{%- endif -%}
{%- endfor %}
# settings for worker processes:
configuration.unicorn_worker_processes = ${worker-processes:unicorn-worker-processes}
configuration.puma_worker_processes = ${worker-processes:puma-worker-processes}
configuration.nginx_worker_processes = ${worker-processes:nginx-worker-processes}
......@@ -186,16 +186,18 @@ mode = 0700
[gitaly-dir]
recipe = slapos.cookbook:mkdirectory
gitaly = ${directory:var}/gitaly
sockets = ${:gitaly}/sockets
internal = ${directory:var}/int
sockets = ${:gitaly}/s
internal = ${:sockets}/int
log = ${directory:log}/gitaly
[gitaly]
socket = ${directory:var}/gitaly.socket
log = ${gitaly-dir:log}
socket = ${gitaly-dir:sockets}/gitaly.socket
logdir = ${gitaly-dir:log}
location = {{ gitaly_location }}
pid = ${directory:run}/gitaly.pid
internal_socket = ${gitaly-dir:internal}
basedir = ${gitaly-dir:gitaly}
num_workers = 2
[gitaly-socket-listening-promise]
<= monitor-promise-base
......@@ -249,7 +251,7 @@ context-extra =
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section unicorn unicorn
section puma puma
section service_redis service-redis
raw redis_binprefix {{ redis_binprefix }}
......@@ -261,6 +263,7 @@ context-extra =
section gitlab gitlab
section gitlab_shell gitlab-shell
section gitlab_shell_work gitlab-shell-work
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly
[nginx.conf]
......@@ -288,12 +291,10 @@ context-extra =
import urllib urllib
section gitlab gitlab
section gitlab_shell_work gitlab-shell-work
section gitlab_shell gitlab-shell
section gitlab_workhorse gitlab-workhorse
section gitaly gitaly
[rack_attack.rb]
<= gitlab-etc-template
url = {{ rack_attack_rb_in }}
[resque.yml]
<= gitlab-etc-template
url = {{ resque_yml_in }}
......@@ -306,11 +307,11 @@ url = {{ smtp_settings_rb_in }}
# contains smtp password
mode = 0600
[unicorn.rb]
[puma.rb]
<= gitlab-etc-template
url = {{ unicorn_rb_in }}
url = {{ puma_rb_in }}
context-extra =
section unicorn unicorn
section puma puma
section directory directory
section gitlab_work gitlab-work
......@@ -340,20 +341,20 @@ prog = {{ prog }}
{% endfor %}
[gitlab-unicorn-startup]
[gitlab-puma-startup]
recipe = slapos.recipe.template:jinja2
mode = 0755
url = {{ gitlab_unicorn_startup_in }}
url = {{ gitlab_puma_startup_in }}
output= ${directory:bin}/${:_buildout_section_name_}
context =
raw bash_bin {{ bash_bin }}
raw gitlab_rake ${gitlab-rake:wrapper-path}
raw gitlab_unicorn ${gitlab-unicorn:wrapper-path}
raw gitlab_puma ${gitlab-puma:wrapper-path}
raw psql_bin {{ postgresql_location }}/bin/psql
section pgsql service-postgresql
raw log_dir ${gitlab:log}
raw var_dir ${directory:var}
section unicorn_rb unicorn.rb
section puma_rb puma.rb
section gitlab_work gitlab-work
......@@ -421,14 +422,13 @@ tune-command =
ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
# config/
cd config &&
ln -sf ${unicorn.rb:output} unicorn.rb &&
ln -sf ${puma.rb:output} puma.rb &&
ln -sf ${gitlab.yml:output} gitlab.yml &&
ln -sf ${database.yml:output} database.yml &&
ln -sf ${resque.yml:output} resque.yml &&
ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml &&
# config/initializers/
cd initializers &&
ln -sf ${rack_attack.rb:output} rack_attack.rb &&
ln -sf ${smtp_settings.rb:output} smtp_settings.rb &&
# public/
cd ../../public &&
......@@ -573,11 +573,12 @@ wrapper-path = ${directory:service}/gitlab-workhorse
command-line = {{ gitlab_workhorse }}
-listenNetwork unix
-listenAddr ${gitlab-workhorse:socket}
-authSocket ${unicorn:socket}
-authSocket ${puma:socket}
-documentRoot ${gitlab-work:location}/public
-secretPath ${gitlab-workhorse:secret}
-logFile ${gitlab-workhorse:log}
-repoPath ${gitlab-repo-dir:repositories}
# repoPath is for patched gitlab-workhorse
# -repoPath ${gitlab-repo-dir:repositories}
# NOTE for profiling
# -pprofListenAddr ...
......@@ -606,41 +607,47 @@ config-command = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket} ht
######################
# unicorn worker #
# puma worker #
######################
[unicorn-dir]
[puma-dir]
recipe = slapos.cookbook:mkdirectory
srv = ${directory:srv}/unicorn
log = ${directory:log}/unicorn
srv = ${directory:srv}/puma
log = ${directory:log}/puma
pid = ${directory:srv}/pids
[unicorn]
srv = ${unicorn-dir:srv}
log = ${unicorn-dir:log}
socket = ${directory:srv}/unicorn.socket
[puma]
srv = ${puma-dir:srv}
log = ${puma-dir:log}
socket = ${puma-dir:srv}/puma.socket
pid = ${puma-dir:pid}
[service-unicorn]
[service-puma]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/unicorn
# NOTE we perform db setup / migrations as part of unicorn startup.
wrapper-path = ${directory:service}/puma
# NOTE we perform db setup / migrations as part of puma startup.
# Those operations require PG and Redis to be up and running already, that's
# why we do it here. See gitlab-unicorn-startup for details.
command-line = ${gitlab-unicorn-startup:output}
# why we do it here. See gitlab-puma-startup for details.
command-line = ${gitlab-puma-startup:output}
{% if instance_parameter_dict.get('configuration.root-password') -%}
environment =
GITLAB_ROOT_PASSWORD={{ instance_parameter_dict['configuration.root-password'] }}
{% endif %}
depend =
${promise-unicorn:recipe}
${promise-puma:recipe}
${promise-gitlab-app:recipe}
${promise-gitlab-shell:recipe}
${logrotate-entry-unicorn:recipe}
# gitlab is a service "run" under unicorn
${logrotate-entry-puma:recipe}
# gitlab is a service "run" under puma
# gitlab-shell is called by gitlab
# -> associate their logs rotation to here
${logrotate-entry-gitlab:recipe}
[promise-unicorn]
[promise-puma]
<= promise-byurl
config-command = {{ curl_bin }} --unix-socket ${unicorn:socket} http://localhost/
config-command = {{ curl_bin }} --unix-socket ${puma:socket} http://localhost/
[promise-rakebase]
recipe = slapos.cookbook:wrapper
......@@ -662,10 +669,10 @@ command-line = ${:rake} gitlab:gitlab_shell:check
# rake gitlab:repo:check (fsck all repos)
[logrotate-entry-unicorn]
[logrotate-entry-puma]
<= logrotate-entry-base
log = ${unicorn:log}/*.log
name = unicorn
log = ${puma:log}/*.log
name = puma
copytruncate = true
[logrotate-entry-gitlab]
......@@ -682,8 +689,8 @@ copytruncate = true
[logrotate-entry-gitlab-workhorse]
<= logrotate-entry-base
log = ${gitlab-workhorse-dir:log}//*.log
name = gitlab-shell
log = ${gitlab-workhorse-dir:log}/*.log
name = gitlab-workhorse
copytruncate = true
#######################################
......@@ -828,18 +835,17 @@ cron-entries = ${cron:cron-entries}
[service-gitaly]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/gitaly
#command-line = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
command-line = {{ gitaly_location }}/gitaly ${gitaly-config.toml:output}
environment =
PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin
PATH={{ buildout_bin_directory }}:{{ ruby_location }}/bin:/bin:/usr/bin
# 6. on-reinstantiate actions
# NOTE here we only recompile assets. Other on-reinstantiate actions, which
# require pg and redis running, are performed as part of unicorn service -
# right before its startup (see gitlab-unicorn-startup).
# require pg and redis running, are performed as part of puma service -
# right before its startup (see gitlab-puma-startup).
[on-reinstantiate]
recipe = plone.recipe.command
stop-on-error = true
......
software/gitlab/instance.cfg.in
View file @ af88b7c8
......@@ -53,6 +53,7 @@ context =
section instance_parameter_dict slap-configuration
# program binaries
raw buildout_bin_directory ${buildout:bin-directory}
raw bash_bin ${bash:location}/bin/bash
raw bzip2_location ${bzip2:location}
raw bundler_4gitlab ${bundler-4gitlab:bundle}
......@@ -64,7 +65,7 @@ context =
raw git_location ${git:location}
raw gitaly_location ${gitaly-repository:location}
raw gitlab_export ${gitlab-export:output}
raw gitlab_workhorse ${gowork:bin}/gitlab-workhorse
raw gitlab_workhorse ${gitlab-workhorse:binary}
raw gopath_bin ${gowork:bin}
raw gunzip_bin ${gzip:location}/bin/gunzip
raw grep_location ${grep:location}
......@@ -75,8 +76,8 @@ context =
raw nginx_mime_types ${nginx-output:mime}
raw node_bin_location ${nodejs:location}/bin/
raw openssl_bin ${openssl-output:openssl}
raw postgresql_location ${postgresql10:location}
raw redis_binprefix ${redis28:location}/bin
raw postgresql_location ${postgresql:location}
raw redis_binprefix ${redis:location}/bin
raw ruby_location ${bundler-4gitlab:ruby-location}
raw tar_location ${tar:location}
raw watcher ${watcher:output}
......@@ -88,17 +89,16 @@ context =
raw gitconfig_in ${gitconfig.in:target}
raw monitor_template ${monitor2-template:output}
raw gitlab_shell_config_yml_in ${gitlab-shell-config.yml.in:target}
raw gitlab_unicorn_startup_in ${gitlab-unicorn-startup.in:target}
raw gitlab_puma_startup_in ${gitlab-puma-startup.in:target}
raw gitlab_yml_in ${gitlab.yml.in:target}
raw gitaly_config_toml_in ${gitaly-config.toml.in:target}
raw macrolib_cfg_in ${macrolib.cfg.in:target}
raw nginx_conf_in ${nginx.conf.in:target}
raw nginx_gitlab_http_conf_in ${nginx-gitlab-http.conf.in:target}
raw rack_attack_rb_in ${rack_attack.rb.in:target}
raw resque_yml_in ${resque.yml.in:target}
raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw gitlab_restore_sh_in ${template-gitlab-resiliency-restore.sh.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target}
raw puma_rb_in ${puma.rb.in:target}
$${:context-extra}
context-extra =
......
software/gitlab/software.cfg
View file @ af88b7c8
This diff is collapsed.
software/gitlab/template/gitaly-config.toml.in
View file @ af88b7c8
......@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}"
# The directory where Gitaly's executables are stored
bin_dir = "{{ gitaly.location }}"
# # Optional: listen on a TCP socket. This is insecure (no authentication)
# # Optional. The directory where Gitaly can create all files required to
# # properly operate at runtime. If not set, Gitaly will create a directory in
# # the global temporary directory. This directory must exist.
runtime_dir = "{{ gitaly.basedir }}"
# # Optional if socket_path is set. TCP address for Gitaly to listen on. This is insecure (unencrypted connection).
# listen_addr = "localhost:9999"
# tls_listen_addr = "localhost:8888
......@@ -33,6 +38,10 @@ internal_socket_dir = "{{ gitaly.internal_socket }}"
bin_path = "{{ git }}"
# catfile_cache_size = 100
# [[git.config]]
# key = fetch.fsckObjects
# value = true
[[storage]]
name = "default"
path = "{{ gitlab.repositories }}"
......@@ -47,9 +56,9 @@ path = "{{ gitlab.repositories }}"
# You can optionally configure Gitaly to output JSON-formatted log messages to stdout
[logging]
# The directory where Gitaly stores extra log files
dir = "{{ gitaly.log }}"
# format = "json"
# format = "json"
dir = "{{ gitaly.logdir }}"
# format = "text"
format = "json"
# # Optional: Set log level to only log entries with that severity or above
# # One of, in order: debug, info, warn, errror, fatal, panic
# # Defaults to "info"
......@@ -79,7 +88,7 @@ dir = "{{ gitaly.location }}/ruby"
# restart_delay = "5m"
#
# # Number of gitaly-ruby worker processes
# num_workers = 2
num_workers = {{ gitaly.num_workers }}
#
# # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc)
# # NOTE: This only affects RPCs that use Rugged.
......@@ -89,7 +98,47 @@ dir = "{{ gitaly.location }}/ruby"
# The directory where gitlab-shell is installed
dir = "{{ gitlab_shell_work.location }}"
[hooks]
custom_hooks_dir = "{{ gitlab_shell_work.location }}/hooks/"
[gitlab]
secret_file = "{{ gitlab_shell.secret }}"
url = "http+unix://{{ urllib.parse.unquote_plus(gitlab_workhorse.socket) }}"
# Only needed if a UNIX socket is used in `url` and GitLab is configured to
# use a relative path (e.g. /gitlab).
# relative_url_root = '/'
[gitlab.http-settings]
# read_timeout = 300
# user = someone
# password = somepass
# ca_file = /etc/ssl/cert.pem
# ca_path = /etc/pki/tls/certs
# self_signed_cert = false
# # You can adjust the concurrency of each RPC endpoint
# [[concurrency]]
# rpc = "/gitaly.RepositoryService/GarbageCollect"
# max_per_repo = 1
# Daily maintenance designates time slots to run daily to optimize and maintain
# enabled storages.
# [daily_maintenance]
# start_hour = 23
# start_minute = 30
# duration = "45m"
# storages = ["default"]
# disabled = false
# [cgroups]
# count = 10
# mountpoint = "/sys/fs/cgroup"
# hierarchy_root = "gitaly"
# [cgroups.memory]
# enabled = true
# limit = 1048576
# [cgroups.cpu]
# enabled = true
# shares = 512
software/gitlab/template/gitconfig.in
View file @ af88b7c8
......@@ -12,15 +12,24 @@
[pack]
threads = 1
# Enable packfile bitmaps
[repack]
writeBitmaps = true
# don't allow corrupt/broken objects to go in
# Enable push (advertisePushOptions) options
[receive]
fsckObjects = true
advertisePushOptions = true
[user]
name = {{ cfg('email_display_name') }}
email = {{ cfg('email_from') }}
# Enable fsyncObjectFiles to reduce risk of repository corruption if the server crashes
[core]
autocrlf = input
fsyncObjectFiles = true
[gc]
auto = 0
software/gitlab/template/gitlab-shell-config.yml.in
View file @ af88b7c8
......@@ -7,13 +7,15 @@
# GitLab user. git by default
user: {{ backend_info.user }}
# Url to gitlab instance. Used for api calls. Should end with a slash.
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(unicorn.socket) }}/"
# URL to GitLab instance, used for API calls. Default: http://localhost:8080.
# For relative URL support read http://doc.gitlab.com/ce/install/relative_url.html
gitlab_url: "http+unix://{{ urllib.parse.quote_plus(puma.socket) }}/"
http_settings:
{# we don't need any
<%= @http_settings.to_json if @http_settings %>
#}
# read_timeout: 300
# user: someone
# password: somepass
# ca_file: /etc/ssl/cert.pem
......@@ -34,35 +36,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
# Default is .gitlab_shell_secret in the root directory.
secret_file: "{{ gitlab_shell.secret }}"
# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
# Default is hooks in the gitlab-shell directory.
custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
# Redis settings used for pushing commit notices to gitlab
redis:
bin: {{ redis_binprefix }}/redis-cli
host: {# <%= @redis_host %> #}
port: {# <%= @redis_port %> #}
socket: {{ service_redis.unixsocket }}
database: {# <%= @redis_database %> #}
namespace: resque:gitlab
# Log file.
# Default is gitlab-shell.log in the root directory.
log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
# Log level. INFO by default
log_level:
log_level: INFO
# Log format. 'text' by default
log_format: text
# Audit usernames.
# Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
# incurs an extra API call on every gitlab-shell command.
audit_usernames:
# Enable git-annex support
# git-annex allows managing files with git, without checking the file contents into git
# See https://git-annex.branchable.com/ for documentation
# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
# For CentOS: sudo yum install epel-release && sudo yum install git-annex
git_annex_enabled:
audit_usernames: false
software/gitlab/template/gitlab.yml.in
View file @ af88b7c8
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg, cfg_https, external_url with context %}
# # # # # # # # # # # # # # # # # #
# GitLab application config file #
# # # # # # # # # # # # # # # # # #
#
########################### NOTE #####################################
# This file should not receive new settings. All configuration options #
# * are being moved to ApplicationSetting model! #
# If a setting requires an application restart say so in that screen. #
# If you change this file in a merge request, please also create #
# a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests. #
# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md #
########################################################################
#
#
# How to use:
# 1. Copy file as gitlab.yml
# 2. Update gitlab -> host with your fully qualified domain name
# 3. Update gitlab -> email_from
# 4. If you installed Git from source, change git -> bin_path to /usr/local/bin/git
# IMPORTANT: If Git was installed in a different location use that instead.
# You can check with `which git`. If a wrong path of Git is specified, it will
# result in various issues such as failures of GitLab CI builds.
# 5. Review this configuration file for other settings you may want to adjust
production: &base
#
# 1. GitLab app settings
......@@ -18,6 +40,9 @@ production: &base
host: {{ external_url.hostname }}
port: {{ external_url.port or default_port[external_url.scheme] }}
https: {{ cfg_https }}
# The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
# Default is 95% of the worker timeout
max_request_duration_seconds: 57
{# ssh is disabled completely in slapos version
# Uncommment this line below if your ssh host is different from HTTP/HTTPS one
......@@ -55,6 +80,8 @@ production: &base
worker_src: "'self' blob:"
report_uri:
allowed_hosts: []
# Trusted Proxies
# Customize if you have GitLab behind a reverse proxy which is running on a different machine.
# Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
......@@ -122,6 +149,15 @@ production: &base
repository_downloads_path: <%= @gitlab_repository_downloads_path %>
#}
## Impersonation settings
impersonation_enabled: true
## Disable jQuery and CSS animations
# disable_animations: true
## Application settings cache expiry in seconds (default: 60)
# application_settings_cache_seconds: 60
{# we do not support reply by email
## Reply by email
# Allow users to comment on issues and merge requests by replying to notification emails.
......@@ -414,7 +450,9 @@ production: &base
# Gitaly settings
gitaly:
# Default Gitaly authentication token. Can be overriden per storage. Can
# Path to the directory containing Gitaly client executables.
client_path: {{ gitaly.location }}
# Default Gitaly authentication token. Can be overridden per storage. Can
# be left blank when Gitaly is running locally on a Unix socket, which
# is the normal way to deploy Gitaly.
token:
......@@ -463,7 +501,6 @@ production: &base
authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
repos_path: {{ gitlab.repositories }}
hooks_path: {{ gitlab_shell_work.location }}/hooks/
secret_file: {{ gitlab_shell.secret }}
# Git over HTTP
......@@ -483,17 +520,16 @@ production: &base
# gitlab-shell needs to be set to true
git_annex_enabled: <%= @git_annex_enabled %>
workhorse:
# File that contains the secret key for verifying access for gitlab-workhorse.
secret_file: {{ gitlab_workhorse.secret }}
## Git settings
# CAUTION!
# Use the default values unless you really know what you are doing
git:
bin_path: {{ git }}
# The next value is the maximum memory size grit can use
# Given in number of bytes per git object (e.g. a commit)
# This value can be increased if you have very large commits
max_size: {{ cfg('git_max_size') }}
# Git timeout to read a commit, in seconds
timeout: {{ cfg('git_timeout') }}
#
# 5. Extra customization
......@@ -515,11 +551,6 @@ production: &base
<% end %>
#}
{# we are ok (for now) with default rack-attack git settings
rack_attack:
git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %>
#}
## Site ICP License
# XXX unquote needed only for slapos.core earlier than
......@@ -530,6 +561,22 @@ production: &base
{# ICP: '{{ cfg("icp_license") }}' #}
{% endif %}
rack_attack:
git_basic_auth:
# Rack Attack IP banning enabled
enabled: {{ cfg("rack_attack_enable") }}
#
# Whitelist requests from 127.0.0.1 for web proxies (NGINX/Apache) with incorrect headers
ip_whitelist: [{{ cfg("rack_attack_ip_whitelist")}}]
#
# Limit the number of Git HTTP authentication attempts per IP
maxretry: {{ cfg("rack_attack_max_retry") }}
#
# Reset the auth attempt counter per IP after 60 seconds
findtime: {{ cfg("rack_attack_find_time") }}
#
# Ban an IP for one hour (3600s) after too many auth attempts
bantime: {{ cfg("rack_attack_ban_time") }}
development:
<<: *base
......
software/gitlab/template/puma.rb.in 0 → 100644
View file @ af88b7c8
# see: https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-cookbooks/gitlab/templates/default/puma.rb.erb
{% from 'macrolib.cfg.in' import cfg with context %}
# frozen_string_literal: true
# Load "path" as a rackup file.
#
# The default is "config.ru".
#
rackup 'config.ru'
pidfile '{{ puma.pid }}/puma.pid'
state_path '{{ puma.pid }}/puma.state'
stdout_redirect '{{ puma.log }}/puma.stdout.log',
'{{ puma.log }}/puma.stderr.log',
true
# Configure "min" to be the minimum number of threads to use to answer
# requests and "max" the maximum.
#
# The default is "0, 16".
#
threads {{ cfg("puma_min_threads") }}, {{ cfg("puma_max_threads") }}
# By default, workers accept all requests and queue them to pass to handlers.
# When false, workers accept the number of simultaneous requests configured.
#
# Queueing requests generally improves performance, but can cause deadlocks if
# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
#
# When set to false this may require a reverse proxy to handle slow clients and
# queue requests before they reach puma. This is due to disabling HTTP keepalive
queue_requests false
# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
# accepted protocols.
bind 'unix://{{ puma.socket }}'
directory '{{ gitlab_work.location }}'
workers {{ cfg("puma_worker_processes") }}
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events"
require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/puma_worker_killer_initializer"
on_restart do
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
options = { workers: {{ cfg("puma_worker_processes") }} }
before_fork do
# Signal to the puma killer
Gitlab::Cluster::PumaWorkerKillerInitializer.start options unless ENV['DISABLE_PUMA_WORKER_KILLER']
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
end
Gitlab::Cluster::LifecycleEvents.set_puma_options options
on_worker_boot do
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
end
# Preload the application before starting the workers; this conflicts with
# phased restart feature. (off by default)
preload_app!
tag 'gitlab-puma-worker'
# Verifies that all workers have checked in to the master process within
# the given timeout. If not the worker process will be restarted. Default
# value is 60 seconds.
#
worker_timeout {{ cfg("puma_worker_timeout") }}
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#lower-latency-better-throughput
wait_for_less_busy_worker ENV.fetch('PUMA_WAIT_FOR_LESS_BUSY_WORKER', 0.001).to_f
# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#nakayoshi_fork
nakayoshi_fork unless ENV['DISABLE_PUMA_NAKAYOSHI_FORK'] == 'true'
# Use json formatter
require_relative "{{ gitlab_work.location }}/lib/gitlab/puma_logging/json_formatter"
json_formatter = Gitlab::PumaLogging::JSONFormatter.new
log_formatter do |str|
json_formatter.call(str)
end
software/gitlab/template/rack_attack.rb.in deleted 100644 → 0
View file @ cd46fcf2
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
{% from 'macrolib.cfg.in' import cfg with context %}
# 1. Rename this file to rack_attack.rb
# 2. Review the paths_to_be_protected and add any other path you need protecting
#
paths_to_be_protected = [
"#{Rails.application.config.relative_url_root}/users/password",
"#{Rails.application.config.relative_url_root}/users/sign_in",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
"#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
"#{Rails.application.config.relative_url_root}/users",
"#{Rails.application.config.relative_url_root}/users/confirmation",
"#{Rails.application.config.relative_url_root}/unsubscribes/"
]
# Create one big regular expression that matches strings starting with any of
# the paths_to_be_protected.
paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
unless Rails.env.test? || !rack_attack_enabled
Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
if req.post? && req.path =~ paths_regex
req.ip
end
end
end
software/gitlab/template/smtp_settings.rb.in
View file @ af88b7c8
......@@ -21,8 +21,6 @@ if Rails.env.production?
enable_starttls_auto: {{ cfg('smtp_enable_starttls_auto') }},
# ssl:
openssl_verify_mode: '{{ cfg("smtp_openssl_verify_mode") }}'
# ca_path:
# ca_file:
}
end
{% else %}
......
software/gitlab/template/template-gitlab-resiliency-restore.sh.in
View file @ af88b7c8
......@@ -21,15 +21,15 @@ redis_pid_file="{{ redis_pid_file }}"
postgres_pid_file="{{ postgres_pid_file }}"
bin_location="{{ bin_directory }}"
run_location="{{ run_directory }}"
git_location="{{ git_location }}"
go_work_bin="{{ go_work_bin }}"
etc_location="{{ etc_directory }}"
gitlab_work="{{ gitlab_work_location }}"
promise_check="{{ promise_lab_location }}"
unicorn_script="{{ unicorn_script }}"
puma_script="{{ puma_script }}"
puma_pid_file="{{ puma_pid_file }}"
sidekiq_script="{{ sidekiq_script }}"
var_location="{{ run_directory }}/.."
var_location="{{ var_directory }}"
# export GIT_EXEC_PATH=$git_location/libexec/git-core/
......@@ -56,7 +56,7 @@ kill_process () {
check_process $postgres_pid_file "Postgres"
check_process $redis_pid_file "Redis"
check_process $run_location/unicorn.pid "Unicorn"
check_process $puma_pid_file "Puma"
if [ -f "$postgres_pid_file" ]; then
rm $postgres_pid_file
......@@ -90,14 +90,14 @@ echo "Checking gitlab promises..."
echo "[info] Not all promises are checked!"
$promise_check/gitlab-app
echo "Starting Unicorn to check gitlab-shell promise..."
$unicorn_script &
unicorn_pid=$!
trap "kill $postgres_pid $redis_pid $unicorn_pid" EXIT TERM INT
echo "Starting Puma to check gitlab-shell promise..."
$puma_script &
puma_pid=$!
trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
sleep 60
if [ -s "$run_location/unicorn.pid" ]; then
unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $unicorn_ppid" EXIT TERM INT
if [ -s "$puma_pid_file" ]; then
puma_pid=$(head -n 1 $puma_pid_file) > /dev/null 2>&1
trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
fi
$promise_check/gitlab-shell
......@@ -109,7 +109,7 @@ $promise_check/gitlab-shell
kill_process $postgres_pid
kill_process $redis_pid
kill_process $unicorn_pid
kill_process $puma_pid
RESTORE_EXIT_CODE=$?
......
software/gitlab/template/unicorn.rb.in deleted 100644 → 0
View file @ cd46fcf2
{{ autogenerated }}
# see:
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
{% from 'macrolib.cfg.in' import cfg with context %}
# What ports/sockets to listen on, and what options for them.
# we listen only on unix socket
listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
#listen "127.0.0.1:8888", :tcp_nopush => true
working_directory '{{ gitlab_work.location }}'
# What the timeout for killing busy workers is, in seconds
timeout {{ cfg('unicorn_worker_timeout') }}
# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
preload_app true
# Enable this flag to have unicorn test client connections by writing the
# beginning of the HTTP headers before calling the application. This
# prevents calling the application for connections that have disconnected
# while queued. This is only guaranteed to detect clients on the same
# host unicorn runs on, and unlikely to detect disconnects even on a
# fast LAN.
check_client_connection false
require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
before_exec do |server|
# Signal application hooks that we're about to restart
Gitlab::Cluster::LifecycleEvents.do_before_master_restart
end
# How many worker processes
worker_processes {{ cfg('unicorn_worker_processes') }}
# about before_fork / after_fork - see:
# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
# http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
# What to do before we fork a worker
before_fork do |server, worker|
# Signal application hooks that we're about to fork
Gitlab::Cluster::LifecycleEvents.do_before_fork
# The following is only recommended for memory/DB-constrained
# installations. It is not needed if your system can house
# twice as many worker_processes as you have configured.
#
# This allows a new master process to incrementally
# phase out the old master process with SIGTTOU to avoid a
# thundering herd (especially in the "preload_app false" case)
# when doing a transparent upgrade. The last worker spawned
# will then kill off the old master process with a SIGQUIT.
old_pid = "#{server.config[:pid]}.oldbin"
if old_pid != server.pid
begin
sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
Process.kill(sig, File.read(old_pid).to_i)
rescue Errno::ENOENT, Errno::ESRCH
end
end
#
# Throttle the master from forking too quickly by sleeping. Due
# to the implementation of standard Unix signal handlers, this
# helps (but does not completely) prevent identical, repeated signals
# from being lost when the receiving process is busy.
# sleep 1
end
# What to do after we fork a worker
after_fork do |server, worker|
# Signal application hooks of worker start
Gitlab::Cluster::LifecycleEvents.do_worker_start
# per-process listener ports for debugging/admin/migrations
# addr = "127.0.0.1:#{9293 + worker.nr}"
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
end
# Where to drop a pidfile
pid '{{ directory.run }}/unicorn.pid'
# Where stderr gets logged
stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
# Where stdout gets logged
stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
{# we do not support Relative url
<%- if @relative_url %>
# Relative url from where GitLab is served
ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
<%- end %>
#}
# Min memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
# Max memory size (RSS) per worker
ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s
software/gitlab/test/setup.py
View file @ af88b7c8
......@@ -46,6 +46,7 @@ setup(
'erp5.util',
'supervisor',
'requests',
'beautifulsoup4'
],
zip_safe=True,
test_suite='test',
......
software/gitlab/test/test.py
View file @ af88b7c8
......@@ -26,10 +26,10 @@
##############################################################################
import os
import logging
from urllib.parse import urlparse
import requests
import functools
import bs4
from urllib.parse import urljoin
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -46,6 +46,10 @@ class TestGitlab(SlapOSInstanceTestCase):
def getInstanceSoftwareType(cls):
return 'gitlab'
@classmethod
def getInstanceParameterDict(cls):
return {'root-password': 'admin1234'}
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend_url']
......@@ -54,3 +58,33 @@ class TestGitlab(SlapOSInstanceTestCase):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
def test_rack_attack_sign_in_rate_limiting(self):
session = requests.session()
# Load the login page to get a CSRF token.
response = session.get(urljoin(self.backend_url, 'users/sign_in'), verify=False)
self.assertEqual(response.status_code, 200)
# Extract the CSRF token and param.
bsoup = bs4.BeautifulSoup(response.text, 'html.parser')
csrf_param = bsoup.find('meta', dict(name='csrf-param'))['content']
csrf_token = bsoup.find('meta', dict(name='csrf-token'))['content']
request_data = {
'user[login]': 'test',
'user[password]': 'random',
csrf_param: csrf_token}
sign_in = functools.partial(
session.post,
response.url,
data=request_data,
verify=False)
for _ in range(10):
sign_in(headers={'X-Forwarded-For': '1.2.3.4'})
# after 10 authentication failures, this client is rate limited
self.assertEqual(sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).status_code, 429)
# but other clients are not
self.assertNotEqual(sign_in(headers={'X-Forwarded-For': '5.6.7.8'}).status_code, 429)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7