software/gitlab: minimal rate-limiting of archive downloads with nginx

nginx is not really flexible for this, but since gitlab does not make
download of archive configurable, this adds a rate limit of 1 request
per minute per source IP for archive downloads.

software/gitlab/buildout.hash.cfg

@@ -34,7 +34,7 @@ md5sum = c559a24ab6281268b608ed3bccb8e4ce
 
 [gitlab-parameters.cfg]
 _update_hash_filename_ = gitlab-parameters.cfg
-md5sum = 311f3b06ba5026b6aa958a86c58b815c
+md5sum = 16b25d654fe1f219a78d8a3da16b07dd
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
@@ -66,7 +66,7 @@ md5sum = 70612697434bf4fbe838fdf4fd867ed8
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = 093f8b1472294fd97213272f3fe1411a
+md5sum = b40b6d7948f4a54c45f2ecbb7e3d7a36
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in

software/gitlab/gitlab-parameters.cfg

@@ -123,3 +123,5 @@ configuration.nginx_real_ip_recursive           = off
 # space separated URLs of caucase service providing CA to validate frontends client
 # certificate and trust the frontend if they provide a valid certificate.
 configuration.frontend-caucase-url-list         =
+# rate limit of git projects archive download, in requests per minutes.
+configuration.nginx_download_archive_rate_limit = 1

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -37,6 +37,8 @@ upstream gitlab-workhorse {
   server unix:{{ gitlab_workhorse.socket }};
 }
 
+limit_req_zone $trusted_remote_addr zone=downloadarchive:10m rate={{ cfg('nginx_download_archive_rate_limit') }}r/m;
+
 {# not needed for us - the frontend can do the redirection and also
    gitlab/nginx speaks HSTS on https port so when we access https port via http
    protocol, it gets redirected to https
@@ -161,6 +163,8 @@ server {
 
   proxy_http_version 1.1;
 
+  limit_req_status 429;
+
   {# we do not support relative URL - path is always "/" #}
   {% set path = "/" %}
 
@@ -182,6 +186,20 @@ server {
     proxy_pass http://gitlab-workhorse;
   }
 
+  ## archive downloads are rate limited.
+  location ~ /[^/]+/[^/]+/-/archive/.* {
+    limit_req zone=downloadarchive;
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    {% if cfg_https %}
+    proxy_set_header    X-Forwarded-Ssl     on;
+    {% endif %}
+    proxy_set_header    X-Forwarded-For     $trusted_remote_addr;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
+
+    proxy_pass http://gitlab-workhorse;
+  }
+
   location {{ path }} {
     # NOTE(slapos) proxy headers are defined upstream in omnibus-gitlab in:
     #   - files/gitlab-config-template/gitlab.rb.template       nginx['proxy_set_headers']

software/gitlab/test/test.py

@@ -28,6 +28,7 @@
 import os
 import functools
 import urllib.parse
+import subprocess
 import time
 from typing import Optional, Tuple
 
@@ -162,3 +163,69 @@ class TestGitlab(SlapOSInstanceTestCase):
       ),
       "1.2.3.4",
     )
+
+  def test_download_archive_rate_limiting(self):
+    gitlab_rails_bin = self.computer_partition_root_path / 'bin' / 'gitlab-rails'
+
+    subprocess.check_call(
+      (gitlab_rails_bin,
+      'runner',
+      "user = User.find(1);" \
+      "token = user.personal_access_tokens.create(scopes: [:api], name: 'Root token');" \
+      "token.set_token('SLurtnxPscPsU-SDm4oN');" \
+      "token.save!"),
+    )
+
+    client_certificate = self.getManagedResource('client_certificate', CaucaseCertificate)
+    with requests.Session() as session:
+      session.cert = (client_certificate.cert_file, client_certificate.key_file)
+      session.verify = False
+
+      ret = session.post(
+        urllib.parse.urljoin(self.backend_url, '/api/v4/projects'),
+        data={
+          'name': 'sample-test',
+          'visibility': 'public',
+        },
+        headers={"PRIVATE-TOKEN" : 'SLurtnxPscPsU-SDm4oN'},
+      )
+      ret.raise_for_status()
+      project_id = ret.json()['id']
+
+      session.post(
+        urllib.parse.urljoin(
+          self.backend_url, f"/api/v4/projects/{project_id}/repository/commits"
+        ),
+        json={
+          "branch": "main",
+          "commit_message": "Add a file to test download archive",
+          "actions": [
+            {"action": "create", "file_path": "README.md", "content": "file content"}
+          ],
+        },
+        headers={"PRIVATE-TOKEN": "SLurtnxPscPsU-SDm4oN"},
+      ).raise_for_status()
+
+      for i, ext in enumerate(("zip", "tar.gz", "tar.bz2", "tar")):
+        headers = {"X-Forwarded-For": f"{i}.{i}.{i}.{i}"}
+        get = functools.partial(
+          session.get,
+          urllib.parse.urljoin(
+            self.backend_url,
+            f"/root/sample-test/-/archive/main/sample-test-main.{ext}",
+          ),
+          headers=headers,
+        )
+        with self.subTest(ext):
+          get().raise_for_status()
+          self.assertEqual(get().status_code, 429)
+
+      self.assertEqual(
+        session.get(
+          urllib.parse.urljoin(
+            self.backend_url,
+            f"/root/sample-test/-/archive/invalidref/sample-test-invalidref.zip",
+          ),
+        ).status_code,
+        404,
+      )