Merge branch '330629-introduce-a-new-cache-to-store-latest-gitaly-artifacts' into 'master'

ci: Introduce a cache for Gitaly binaries [UPDATE CACHE]

See merge request gitlab-org/gitlab!62214

.gitlab/ci/global.gitlab-ci.yml

@@ -36,6 +36,23 @@
   <<: *gitaly-ruby-gems-cache
   policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
+.gitaly-binaries-cache: &gitaly-binaries-cache
+  key:
+    files:
+      - GITALY_SERVER_VERSION
+    prefix: "gitaly-binaries"
+  paths:
+    - tmp/tests/gitaly/_build/bin/
+    - tmp/tests/gitaly/config.toml
+    - tmp/tests/gitaly/gitaly2.config.toml
+    - tmp/tests/gitaly/internal/
+    - tmp/tests/gitaly/internal_gitaly2/
+    - tmp/tests/gitaly/internal_sockets/
+    - tmp/tests/gitaly/Makefile
+    - tmp/tests/gitaly/praefect.config.toml
+    - tmp/tests/gitaly/ruby/
+  policy: pull
+
 .go-pkg-cache: &go-pkg-cache
   key: "go-pkg-v1"
   paths:
@@ -97,6 +114,7 @@
   cache:
     - *ruby-gems-cache
     - *gitaly-ruby-gems-cache
+    - *gitaly-binaries-cache
     - *go-pkg-cache
 
 .setup-test-env-cache-push:
@@ -105,6 +123,11 @@
     - *gitaly-ruby-gems-cache-push
     - *go-pkg-cache-push
 
+.gitaly-binaries-cache-push:
+  cache:
+    - <<: *gitaly-binaries-cache
+      policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
 .rails-cache:
   cache:
     - *ruby-gems-cache

.gitlab/ci/rails.gitlab-ci.yml

@@ -177,6 +177,14 @@ update-setup-test-env-cache:
   artifacts:
     paths: []  # This job's purpose is only to update the cache.
 
+update-gitaly-binaries-cache:
+  extends:
+    - setup-test-env
+    - .gitaly-binaries-cache-push
+    - .shared:rules:update-gitaly-binaries-cache
+  artifacts:
+    paths: []  # This job's purpose is only to update the cache.
+
 .coverage-base:
   extends:
     - .default-retry

.gitlab/ci/rules.gitlab-ci.yml

@@ -111,6 +111,9 @@
   - ".gitlab/ci/build-images.gitlab-ci.yml"
   - ".gitlab/ci/qa.gitlab-ci.yml"
 
+.gitaly-patterns: &gitaly-patterns
+  - "GITALY_SERVER_VERSION"
+
 .workhorse-patterns: &workhorse-patterns
   - "GITLAB_WORKHORSE_VERSION"
   - "workhorse/**/*"
@@ -310,6 +313,11 @@
     - <<: *if-security-schedule
     - <<: *if-merge-request-title-update-caches
 
+.shared:rules:update-gitaly-binaries-cache:
+  rules:
+    - <<: *if-merge-request-title-update-caches
+    - changes: *gitaly-patterns
+
 ######################
 # Build images rules #
 ######################

doc/development/pipelines.md

@@ -557,6 +557,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
 1. These cache definitions are composed of [multiple atomic caches](../ci/yaml/README.md#multiple-caches).
 1. Only 6 specific jobs, running in 2-hourly scheduled pipelines, are pushing (i.e. updating) to the caches:
    - `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
+   - `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
    - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).

scripts/gitaly-test-build

@@ -15,7 +15,12 @@ class GitalyTestBuild
   def run
     set_bundler_config
 
-    abort 'gitaly build failed' unless build_gitaly
+    # If we have the binaries from the cache, we can skip building them again
+    if File.exist?(tmp_tests_gitaly_bin_dir)
+      GitalySetup::LOGGER.debug "Gitaly binary already built. Skip building...\n"
+    else
+      abort 'gitaly build failed' unless build_gitaly
+    end
 
     ensure_gitlab_shell_secret!
     check_gitaly_config!