gitlab: Upgrade to 8.5 + ...

- GitLab Software + patches ported to latest stable GitLab 8.5.1
  (including fix for raw downloading to work in browser for private
repositories);
- Sync-with-upstream procedure streamlined (now only 1 branch for
  tracking upstream configuration files);
- Base software upgraded: Ruby, Redis, Nginx, Git;
- misc fixes.

/cc @jerome, @jm
/reviewed-by @kazuhiko
/reviewed-on !55

component/git/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [git]
 recipe = slapos.recipe.cmmi
-url = https://www.kernel.org/pub/software/scm/git/git-2.7.0.tar.xz
-md5sum = 0214e04f7041f835c5c38f2b78eccced
+url = https://www.kernel.org/pub/software/scm/git/git-2.7.2.tar.xz
+md5sum = b14189d9d9ea32274e51c7a4240ffc15
 configure-options =
   --with-curl=${curl:location}
   --with-openssl=${openssl:location}

component/nginx/buildout.cfg

@@ -11,15 +11,15 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.9.4.tar.gz
-md5sum = 27322fbb4b265c0e0cc548f5e6b7f201
+url = http://nginx.org/download/nginx-1.9.12.tar.gz
+md5sum = 0afe4a7e589a0de43b7b54aa055a4351
 
 [nginx]
 <= nginx-common
 configure-options=
   --with-ipv6
   --with-http_ssl_module
-  --with-http_spdy_module
+  --with-http_v2_module
   --with-http_gzip_static_module
   --with-mail
   --with-mail_ssl_module
@@ -38,7 +38,7 @@ mode = 0644
 configure-options =
   --with-ipv6
   --with-http_ssl_module
-  --with-http_spdy_module
+  --with-http_v2_module
   --with-http_gzip_static_module
   --with-mail
   --with-mail_ssl_module

component/redis/buildout.cfg

@@ -10,8 +10,8 @@ extends = ../tcl/buildout.cfg
 
 [redis28]
 recipe = slapos.recipe.cmmi
-url = http://download.redis.io/releases/redis-2.8.23.tar.gz
-md5sum = ac7f43f845d0eedb8ae3e5e217b34c61
+url = http://download.redis.io/releases/redis-2.8.24.tar.gz
+md5sum = 7b6eb6e4ccc050c351df8ae83c55a035
 configure-command = true
 prefix = 
 make-options =

component/ruby/buildout.cfg

@@ -26,8 +26,8 @@ environment =
 
 [ruby2.1]
 <= ruby-common
-url = http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.7.tar.xz
-md5sum = 2e43a1d32cc16975a6b5d3ffce399199
+url = http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.8.tar.xz
+md5sum = f18ed96bd1d5890f97a17d0d17aaefdd
 
 
 [ruby2.2]

software/gitlab/gitlab-parameters.cfg

@@ -6,7 +6,7 @@
 #
 # TODO better autogenerate from ^^^ (?)
 #
-# (last updated for omnibus-gitlab 8.4.2+ce.0-3-g68d5ee8)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 [gitlab-parameters]
 configuration.external_url              = http://lab.example.com

software/gitlab/instance-gitlab.cfg.in

@@ -632,7 +632,7 @@ log     = ${sidekiq-dir:log}
 # NOTE see queue list here:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Procfile
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 [service-sidekiq]
 recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/sidekiq

software/gitlab/software.cfg

@@ -68,7 +68,7 @@ eggs    =
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 [rubygemsrecipe]
 recipe  = rubygemsrecipe
-url     = https://rubygems.org/rubygems/rubygems-2.5.1.zip
+url     = https://rubygems.org/rubygems/rubygems-2.5.2.zip
 
 
 # bundler, that we'll use to
@@ -93,10 +93,13 @@ bundle  = ${buildout:bin-directory}/bundle
 # execjs needs: nodejs
 # rails needs db client program on path: psql
 # gitlab wants to check redis version via running: redis-cli
+# gitlab wants git to be really on path ( it uses git from abspath defined in
+#   gitlab.yml, but there are not all cases like this, e.g. in
+#   https://gitlab.com/gitlab-org/gitlab_git/blob/2f0d3c1a/lib/gitlab_git/repository.rb#L259 )
 # gitlab (via github-markup) wants to convert rst -> html via running: python2 (with docutils egg)
 # (python-4gitlab puts interpreter into ${buildout:bin-directory})
 environment =
-  PATH    = ${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql92:location}/bin:${redis28:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH    = ${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql92:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
 
 
 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -109,25 +112,26 @@ git-executable = ${git:location}/bin/git
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
 repository = https://lab.nexedi.com/kirr/gitlab-ce.git
-# 8.4.X + NXD patches:
-revision = v8.4.4-18-g32fa1180aca0b5c119f9a42ecfc733db1fad1dc0
+# 8.5.X + NXD patches:
+revision = v8.5.1-13-g73c204d506a24ba0b09db5513628c8bf8863b76a
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
-repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-# gitlab 8.4 wants gitlab-shell 2.6.10
-# 2.6.10
-revision = v2.6.10-0-g82b3a4e8f70692ec679d880628fdb0f5844d42b9
+#repository = https://gitlab.com/gitlab-org/gitlab-shell.git
+repository = https://lab.nexedi.com/kirr/gitlab-shell.git
+# gitlab 8.5 wants gitlab-shell 2.6.10
+# 2.6.10 + no-hooks.old patch
+revision = v2.6.10-13-gcb8f331c955aa780efe10263d0d45b332f030f42
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitlab-workhorse-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-workhorse.git
 repository = https://lab.nexedi.com/kirr/gitlab-workhorse.git
-# 0.6.X + NXD patches for blob download speedup
+# 0.6.4 + NXD patches for blob download speedup
 # (https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/17)
-revision = 0.6.1-2-ga23a5e18486b0de6e3435711dc555c8bfe08fde2
+revision = 0.6.4-2-g86b2a17f24f6c369774bbb388560e4aa9a4321c1
 location = ${buildout:parts-directory}/gitlab-workhorse
 
 
@@ -219,23 +223,23 @@ url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 
 [config.ru.in]
 <= download-template
-md5sum  = 3ed3c439ac1b93f75121dabcea126078
+md5sum  = 91737a2067be80f3ae48157e5a0738b4
 
 [database.yml.in]
 <= download-template
-md5sum  = b33f4f2f49a5a3e3e6542357c555a3a3
+md5sum  = 76d8f20532d63282ecd3617a3937fcf1
 
 [gitconfig.in]
 <= download-template
-md5sum  = 75f620ea0751fc8d2dc717cf929d29f3
+md5sum  = d8b3611386c4982605edd9a31832ee28
 
 [gitlab-parameters.cfg]
 <= download-file
-md5sum  = 2cfd3bbf9da10627044ca3a9a149fdbb
+md5sum  = b04b72949f0b9b5da96870ffbf8c1ff4
 
 [gitlab-shell-config.yml.in]
 <= download-template
-md5sum  = f061d529b71241d58affbf7aec5c8af1
+md5sum  = 9c62aa1bf7396f207a528e39973aa135
 
 [gitlab-unicorn-startup.in]
 <= download-file
@@ -243,11 +247,11 @@ md5sum  = 14c5632182d830c03f7788c85d6f4da1
 
 [gitlab.yml.in]
 <= download-template
-md5sum  = cd7aaeeb1917fdedb7656943065c0a9c
+md5sum  = 8f1c52ed223ee2c2d07c6c79bcd2f6c0
 
 [instance-gitlab.cfg.in]
 <= download-file
-md5sum  = 5f0b7d51e921e0e37e40fb65552dde9c
+md5sum  = 7a0ba60d09e6a62ffc1da34cf79737a0
 
 [macrolib.cfg.in]
 <= download-file
@@ -255,27 +259,27 @@ md5sum  = a56a44e96f65f5ed20211bb6a54279f4
 
 [nginx-gitlab-http.conf.in]
 <= download-template
-md5sum  = 3b494fe8425a12e4a7fd3a9bb17f88f8
+md5sum  = be7ddd26b0b0196fde8d699e50e84b2f
 
 [nginx.conf.in]
 <= download-template
-md5sum  = dc16257d49d1fc1ae6e7d10865898201
+md5sum  = 71b4221cd91e0e1e20a72b97991f2665
 
 [rack_attack.rb.in]
 <= download-template
-md5sum  = fbea569a1ac9ee46e37d0b98b5441169
+md5sum  = 2cf56b210a906cee80f86cde7112f468
 
 [resque.yml.in]
 <= download-template
-md5sum  = 2cd97d9f5906d06e00774dd2e4e6af0e
+md5sum  = 1fa21655a845602f68a901f23a1da89a
 
 [smtp_settings.rb.in]
 <= download-template
-md5sum  = 75b9e0325737ca5ecbf938443a5d3321
+md5sum  = 07b640122b8c4840e853360f2479d0ac
 
 [unicorn.rb.in]
 <= download-template
-md5sum  = 1b55105a3de1ef13260ac3faa30d6e85
+md5sum  = 4f7c14147fd60b4cc9aefe7f64524abe
 
 
 [versions]

software/gitlab/template/config.ru.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 # This file is used by Rack-based servers to start the application.
 

software/gitlab/template/database.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
-# (last updated for 8.4.4+ce.0-0-g1680742)
+# (last updated for 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/gitconfig.in

@@ -3,7 +3,7 @@
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitconfig.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 #
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/gitlab-shell-config.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 # GitLab user. git by default
 user: {{ backend_info.user }}

software/gitlab/template/gitlab.yml.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url  with context %}
 
@@ -28,7 +28,9 @@ production: &base
 
     # WARNING: See config/application.rb under "Relative url support" for the list of
     # other files that need to be changed for relative url support
-    # relative_url_root: /gitlab
+    {# we do not support relative URL
+    relative_url_root: <%= @gitlab_relative_url %>
+    #}
 
     # Uncomment and customize if you can't use the default user to run GitLab (default: 'git')
     user: {{ backend_info.user }}
@@ -141,6 +143,8 @@ production: &base
     host: <%= @pages_host %>
     port: <%= @pages_port %>
     https: <%= @pages_https %>
+    external_http: <%= @pages_external_http %>
+    external_https: <%= @pages_external_https %>
   #}
 
   {# we do not support Elasticsearch
@@ -282,15 +286,22 @@ production: &base
     auto_sign_in_with_provider: <%= @omniauth_auto_sign_in_with_provider %>
 
     # CAUTION!
-    # This allows users to login without having a user account first (default: false).
+    # This allows users to login without having a user account first. Define the allowed
+    # providers using an array, e.g. ["saml", "twitter"]
     # User accounts will be created automatically when authentication was successful.
-    allow_single_sign_on: <%= @omniauth_allow_single_sign_on %>
+    allow_single_sign_on: <%= @omniauth_allow_single_sign_on.to_json %>
+
     # Locks down those users until they have been cleared by the admin (default: true).
     block_auto_created_users: <%= @omniauth_block_auto_created_users %>
     # Look up new users in LDAP servers. If a match is found (same uid), automatically
     # link the omniauth identity with the LDAP account. (default: false)
     auto_link_ldap_user: <%= @omniauth_auto_link_ldap_user %>
 
+    # Allow users with existing accounts to login and auto link their account via SAML
+    # login, without having to do a manual login first and manually add SAML
+    # (default: false)
+    auto_link_saml_user: <%= @omniauth_auto_link_saml_user.to_json %>
+
 
     ## Auth providers
     # Uncomment the following lines and fill in the data of the auth provider you want to use

software/gitlab/template/nginx-gitlab-http.conf.in

 {{ autogenerated }}
 # see:
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn  with context %}
 
@@ -56,7 +56,7 @@ server {
 #}
 
 server {
-  listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl spdy{% endif %};
+  listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl http2{% endif %};
 
   {# we don't use: kerbeeros
   <% if @kerberos_enabled && @kerberos_use_dedicated_port %>
@@ -104,7 +104,9 @@ server {
   access_log  {{ nginx.log }}/gitlab_access.log gitlab_access;
   error_log   {{ nginx.log }}/gitlab_error.log;
 
-  location / {
+  {# we do not support relative URL - path is always "/" #}
+  {% set path = "/" %}
+  location {{ path }} {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
     {{ 'gzip off;' if cfg_https else ''}}
@@ -117,6 +119,11 @@ server {
 
     proxy_http_version 1.1;
 
+    # NOTE(slapos) proxy headers are defined upstream in omnibus-gitlab in:
+    #   - files/gitlab-config-template/gitlab.rb.template       nginx['proxy_set_headers']
+    #   - files/gitlab-cookbooks/gitlab/attributes/default.rb   default['gitlab']['nginx']['proxy_set_headers']
+    #   - files/gitlab-cookbooks/gitlab/libraries/gitlab.rb     parse_nginx_proxy_headers()
+    # (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
     proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
     {% if cfg_https %}

software/gitlab/template/nginx.conf.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/rack_attack.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 

software/gitlab/template/resque.yml.in

@@ -2,6 +2,6 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
-# (last udpdated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last udpdated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 production: unix://{{ redis.unixsocket }}

software/gitlab/template/smtp_settings.rb.in

@@ -2,7 +2,7 @@
 # see:
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_bool  with context %}
 

software/gitlab/template/unicorn.rb.in

@@ -3,7 +3,7 @@
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
 # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
-# (last updated for omnibus-gitlab 8.4.4+ce.0-0-g1680742)
+# (last updated for omnibus-gitlab 8.5.1+ce.0-1-ge732b39)
 
 {% from 'macrolib.cfg.in' import cfg  with context %}
 
@@ -71,3 +71,11 @@ stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
 
 # Where stdout gets logged
 stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
+
+{# we do not support Relative url
+<%- if @relative_url %>
+# Relative url from where GitLab is served
+ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
+
+<%- end %>
+#}