software/gitlab: use authenticate-to-backend for client IP

Introduces a new instance parameter, frontend-caucase-url-list which is
a space separated list of IP addresses (this software still uses xml
serialisation and does not have a parameter schema yet).

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 3ffdd78aeb77ab581c51ce419176dd37
+md5sum = 3607ea995293975a736be136f0cdf675
 
 [watcher]
 _update_hash_filename_ = watcher.in
@@ -34,7 +34,7 @@ md5sum = c559a24ab6281268b608ed3bccb8e4ce
 
 [gitlab-parameters.cfg]
 _update_hash_filename_ = gitlab-parameters.cfg
-md5sum = 95b18789111ed239146d243e39ffefbe
+md5sum = f4a308d52527536b3d5874f615145b66
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
@@ -54,7 +54,7 @@ md5sum = d769ea27820e932c596c35bbbf3f2902
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 6d8d20ded84622339d49c60b0e61380c
+md5sum = 95b318570fa22dfbe2925ffd671135c2
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
@@ -66,7 +66,7 @@ md5sum = 70612697434bf4fbe838fdf4fd867ed8
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = 4980c1571a4dd7753aaa60d065270849
+md5sum = a53d179168259f4045640b8461b7fd5c
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in

software/gitlab/gitlab-parameters.cfg

@@ -96,7 +96,7 @@ configuration.nginx_proxy_connect_timeout   = 300
 # nginx advanced
 configuration.nginx_worker_processes    = 4
 configuration.nginx_worker_connections  = 10240
-configuration.nginx_log_format          = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
+configuration.nginx_log_format          = $trusted_remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
 configuration.nginx_sendfile            = on
 configuration.nginx_tcp_nopush          = on
 configuration.nginx_tcp_nodelay         = on

software/gitlab/instance-gitlab.cfg.in

@@ -748,6 +748,65 @@ copytruncate = true
 #   Nginx frontend   #
 ######################
 
+[frontend-caucase-ssl]
+ca =
+crl =
+
+{% import "caucase" as caucase with context %}
+{% set frontend_caucase_url_list = instance_parameter_dict.get('configuration.frontend-caucase-url-list', '').split() -%}
+
+{% set frontend_caucase_url_hash_list = [] -%}
+{% set frontend_caucase_updater_section_list = [] -%}
+{% for frontend_caucase_url in frontend_caucase_url_list -%}
+{%   set hash = hashlib.md5(frontend_caucase_url.encode()).hexdigest() -%}
+{%   do frontend_caucase_url_hash_list.append(hash) -%}
+{%   set data_dir = '${nginx-ssl-dir:ssl}/%s' % hash -%}
+{{   caucase.updater(
+       prefix='frontend-caucase-updater-%s' % hash,
+       buildout_bin_directory=buildout_bin_directory,
+       updater_path='${directory:service}/frontend-caucase-updater-%s' % hash,
+       url=frontend_caucase_url,
+       data_dir=data_dir,
+       ca_path='%s/ca.crt' % data_dir,
+       crl_path='%s/crl.pem' % data_dir,
+       on_renew='${frontend-caucase-updater-housekeeper:output}',
+       max_sleep=1,
+       openssl=openssl_bin,
+     )}}
+{%   do frontend_caucase_updater_section_list.append('frontend-caucase-updater-%s' % hash) -%}
+{% endfor -%}
+
+{% if frontend_caucase_url_hash_list %}
+{%   do frontend_caucase_updater_section_list.append('frontend-caucase-updater-housekeeper') -%}
+[frontend-caucase-ssl]
+ca = ${nginx-ssl-dir:ssl}/frontend_ca.crt
+crl = ${nginx-ssl-dir:ssl}/frontend_crl.pem
+
+[frontend-caucase-updater-housekeeper]
+recipe = slapos.recipe.template
+output = ${directory:bin}/frontend-caucase-updater-housekeeper
+mode = 700
+inline =
+  #!/bin/sh -e
+  # assemble all frontends CA and CRL in one file
+  CA=${frontend-caucase-ssl:ca}
+  CA_TMP=$CA.tmp
+  :> $ CA_TMP
+  CRL=${frontend-caucase-ssl:crl}
+  CRL_TMP=$CRL.tmp
+  :> $ CRL_TMP
+{% for hash in frontend_caucase_url_hash_list %}
+{%   set data_dir = '${nginx-ssl-dir:ssl}/%s' % hash %}
+  echo "# {{ data_dir }}/ca.crt" >> $CA_TMP
+  cat "{{ data_dir }}/ca.crt" >> $CA_TMP
+  echo "# {{ data_dir }}/crl.pem" >> $CRL_TMP
+  cat "{{ data_dir }}/crl.pem" >> $CRL_TMP
+{% endfor %}
+  mv $CA_TMP $CA
+  mv $CRL_TMP $CRL
+  kill -HUP $(cat ${directory:run}/nginx.pid)
+{% endif %}
+
 # srv/nginx/ prefix  +  etc/ log/ ...
 [nginx-dir]
 recipe  = slapos.cookbook:mkdirectory
@@ -787,6 +846,9 @@ ssl     = ${nginx-ssl-dir:ssl}
 cert_file   = ${nginx-generate-certificate:cert_file}
 key_file    = ${nginx-generate-certificate:key_file}
 
+client_ca_file = ${frontend-caucase-ssl:ca}
+client_crl_file = ${frontend-caucase-ssl:crl}
+
 
 [nginx-symlinks]
 # (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log)
@@ -801,6 +863,9 @@ depend  =
     ${nginx-symlinks:recipe}
     ${promise-nginx:recipe}
     ${logrotate-entry-nginx:recipe}
+{% for section in frontend_caucase_updater_section_list %}
+    {{ '${' ~  section ~ ':recipe}' }}
+{% endfor %}
 
 
 [promise-nginx]

software/gitlab/instance.cfg.in

@@ -41,8 +41,12 @@ configuration.icp_license  =
 recipe  = slapos.recipe.template:jinja2
 mode    = 0644
 output= $${buildout:directory}/$${:_buildout_section_name_}
+extensions = jinja2.ext.do
+import-list =
+    rawfile caucase ${caucase-jinja2-library:target}
 context =
     import os os
+    import hashlib hashlib
     import pwd pwd
 
     key bin_directory           buildout:bin-directory

software/gitlab/software.cfg

@@ -3,6 +3,7 @@
 extends =
     buildout.hash.cfg
     ../../stack/slapos.cfg
+    ../../stack/caucase/buildout.cfg
     ../../stack/nodejs.cfg
     ../../stack/monitor/buildout.cfg
     ../../component/libgit2/buildout.cfg
@@ -54,6 +55,7 @@ parts =
 
     slapos-cookbook
     eggs
+    caucase-eggs
 
     bash
     curl

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -76,11 +76,12 @@ server {
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
   ssl_certificate {{ nginx.cert_file }};
   ssl_certificate_key {{ nginx.key_file }};
-  {# we don't need - most root CA will be included by default
-  <% if @ssl_client_certificate %>
-  ssl_client_certificate <%= @ssl_client_certificate%>;
- 	<% end %>
-  #}
+
+  {%   if nginx.client_ca_file %}
+  ssl_client_certificate {{ nginx.client_ca_file }};
+  ssl_crl {{ nginx.client_crl_file }};
+  ssl_verify_client optional_no_ca;
+  {%   endif %}
 
   # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
   # NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
@@ -110,6 +111,18 @@ server {
   set_real_ip_from {{ trusted_address }};
   {% endfor %}
 
+  ## SlapOS: For Real IP, instead of trusting the frontends through their IP addresses,
+  ## we expect the frontends to use a client certificate and we trust frontends only if
+  ## we can validate that certificate.
+  set $trusted_remote_addr $remote_addr;
+  {% if nginx.client_ca_file %}
+  set_real_ip_from 0.0.0.0/0;
+  set_real_ip_from ::/0;
+  if ($ssl_client_verify != SUCCESS) {
+    set $trusted_remote_addr $realip_remote_addr;
+  }
+  {% endif %}
+
   ## HSTS Config
   ## https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/
   {% if int(cfg("nginx_hsts_max_age")) > 0 -%}
@@ -163,7 +176,7 @@ server {
     {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
     {% endif %}
-    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-For     $trusted_remote_addr;
     proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
 
     proxy_pass http://gitlab-workhorse;
@@ -188,7 +201,7 @@ server {
     {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
     {% endif %}
-    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-For     $trusted_remote_addr;
     proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
 
     proxy_pass  http://gitlab-workhorse;

software/gitlab/test/test.py

@@ -26,12 +26,16 @@
 ##############################################################################
 
 import os
-import requests
 import functools
+import urllib.parse
+import time
+from typing import Optional, Tuple
+
 import bs4
-from urllib.parse import urljoin
+import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.caucase import CaucaseCertificate, CaucaseService
 
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -49,7 +53,13 @@ class TestGitlab(SlapOSInstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
-    return {'root-password': 'admin1234'}
+    frontend_caucase = cls.getManagedResource('frontend_caucase', CaucaseService)
+    certificate = cls.getManagedResource('client_certificate', CaucaseCertificate)
+    certificate.request('shared frontend', frontend_caucase)
+    return {
+      'root-password': 'admin1234',
+      'frontend-caucase-url-list': frontend_caucase.url
+    }
 
   def setUp(self):
     self.backend_url = self.computer_partition.getConnectionParameterDict(
@@ -61,10 +71,14 @@ class TestGitlab(SlapOSInstanceTestCase):
       resp.status_code in [requests.codes.ok, requests.codes.found])
 
   def test_rack_attack_sign_in_rate_limiting(self):
-    session = requests.session()
+    client_certificate = self.getManagedResource('client_certificate', CaucaseCertificate)
+    session = requests.Session()
+    session.cert = (client_certificate.cert_file, client_certificate.key_file)
 
     # Load the login page to get a CSRF token.
-    response = session.get(urljoin(self.backend_url, 'users/sign_in'), verify=False)
+    response = session.get(
+      urllib.parse.urljoin(self.backend_url, 'users/sign_in'),
+      verify=False)
     self.assertEqual(response.status_code, 200)
 
     # Extract the CSRF token and param.
@@ -84,8 +98,50 @@ class TestGitlab(SlapOSInstanceTestCase):
        verify=False)
 
     for _ in range(10):
-      sign_in(headers={'X-Forwarded-For': '1.2.3.4'})
+      sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).raise_for_status()
     # after 10 authentication failures, this client is rate limited
     self.assertEqual(sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).status_code, 429)
     # but other clients are not
     self.assertNotEqual(sign_in(headers={'X-Forwarded-For': '5.6.7.8'}).status_code, 429)
+
+  def _get_client_ip_address_from_nginx_log(self, cert: Optional[Tuple[str, str]]) -> str:
+    requests.get(
+      urllib.parse.urljoin(
+        self.backend_url,
+        f'/users/sign_in?request_id={self.id()}',
+      ),
+      verify=False,
+      cert=cert,
+      headers={'X-Forwarded-For': '1.2.3.4'}
+    ).raise_for_status()
+    nginx_log_file = self.computer_partition_root_path / 'var' / 'log'/ 'nginx' / 'gitlab_access.log'
+    for _ in range(100):
+      last_log_line = nginx_log_file.read_text().splitlines()[-1]
+      if self.id() in last_log_line:
+        return last_log_line.split('-')[0].strip()
+      time.sleep(1)
+    raise RuntimeError(f"Could not find {self.id()} in {last_log_line=}")
+
+  def test_client_ip_in_nginx_log_with_certificate(self):
+    client_certificate = self.getManagedResource('client_certificate', CaucaseCertificate)
+    self.assertEqual(
+      self._get_client_ip_address_from_nginx_log(
+        cert=(client_certificate.cert_file, client_certificate.key_file)),
+      '1.2.3.4',
+    )
+
+  def test_client_ip_in_nginx_log_without_certificate(self):
+    self.assertNotEqual(
+      self._get_client_ip_address_from_nginx_log(cert=None),
+        '1.2.3.4',
+    )
+
+  def test_client_ip_in_nginx_log_with_not_verified_certificate(self):
+    another_unrelated_caucase = self.getManagedResource('another_unrelated_caucase', CaucaseService)
+    unknown_client_certificate = self.getManagedResource('unknown_client_certificate', CaucaseCertificate)
+    unknown_client_certificate.request('unknown client certificate', another_unrelated_caucase)
+    self.assertNotEqual(
+      self._get_client_ip_address_from_nginx_log(
+        cert=(unknown_client_certificate.cert_file, unknown_client_certificate.key_file)),
+        '1.2.3.4',
+    )