Merge branch '218274-fix-gitaly-dependent-jobs-missing-gems' into 'master'

Ensure Gitaly is installed properly even when cache is stale

Closes #218274

See merge request gitlab-org/gitlab!32508

.gitlab/ci/dev-fixtures.gitlab-ci.yml

 .run-dev-fixtures:
   extends:
     - .default-retry
-    - .default-cache
+    - .rails-cache
     - .default-before_script
     - .use-pg11
   stage: test
@@ -19,8 +19,9 @@ run-dev-fixtures:
     - .run-dev-fixtures
     - .dev-fixtures:rules:ee-and-foss
   script:
-    - scripts/gitaly-test-spawn
-    - RAILS_ENV=test bundle exec rake db:seed_fu
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu"
 
 run-dev-fixtures-ee:
   extends:
@@ -28,6 +29,7 @@ run-dev-fixtures-ee:
     - .dev-fixtures:rules:ee-only
     - .use-pg11-ee
   script:
-    - scripts/gitaly-test-spawn
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
     - cp ee/db/fixtures/development/* $FIXTURE_PATH
-    - RAILS_ENV=test bundle exec rake db:seed_fu
+    - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu"

.gitlab/ci/frontend.gitlab-ci.yml

@@ -136,16 +136,15 @@ compile-assets pull-cache as-if-foss:
 .frontend-fixtures-base:
   extends:
     - .default-retry
-    - .default-cache
+    - .rails-cache
     - .default-before_script
     - .use-pg11
   stage: fixtures
   needs: ["setup-test-env", "compile-assets pull-cache"]
   script:
-    - date
-    - scripts/gitaly-test-spawn
-    - date
-    - bundle exec rake frontend:fixtures
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - run_timed_command "bundle exec rake frontend:fixtures"
   artifacts:
     name: frontend-fixtures
     expire_in: 31d

.gitlab/ci/global.gitlab-ci.yml

@@ -29,6 +29,19 @@
       - vendor/gitaly-ruby
     policy: pull
 
+.rails-cache:
+  cache:
+    key:
+      files:
+        - Gemfile.lock
+        - GITALY_SERVER_VERSION
+      prefix: "ruby-go-cache-v1"
+    paths:
+      - vendor/ruby
+      - vendor/gitaly-ruby
+      - .go/pkg/mod
+    policy: pull
+
 .yarn-cache:
   cache:
     key:

.gitlab/ci/rails.gitlab-ci.yml

 .rails:needs:setup-and-assets:
   needs: ["setup-test-env", "compile-assets pull-cache"]
 
-.rails-cache:
-  cache:
-    key: "ruby-go-cache-v1"
-    paths:
-      - vendor/ruby
-      - vendor/gitaly-ruby
-      - .go/pkg/mod
-    policy: pull
-
 .rails-job-base:
   extends:
     - .default-retry
@@ -18,15 +9,18 @@
 
 #######################################################
 # EE/FOSS: default refs (MRs, master, schedules) jobs #
-.base-setup-test-env:
+setup-test-env:
   extends:
     - .rails-job-base
+    - .rails:rules:default-refs-code-backstage-qa
+    - .use-pg11
   stage: prepare
   variables:
     GITLAB_TEST_EAGER_LOAD: "0"
   script:
     - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
     - run_timed_command "scripts/gitaly-test-build"  # Do not use 'bundle exec' here
+    - rm tmp/tests/gitaly/.ruby-bundle  # This file prevents gems from being installed even if vendor/gitaly-ruby is missing
   artifacts:
     expire_in: 7d
     paths:
@@ -44,12 +38,6 @@
   cache:
     policy: pull-push
 
-setup-test-env:
-  extends:
-    - .base-setup-test-env
-    - .rails:rules:default-refs-code-backstage-qa
-    - .use-pg11
-
 static-analysis:
   extends:
     - .rails-job-base
@@ -84,6 +72,8 @@ downtime_check:
   stage: test
   needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
   script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
     - source scripts/rspec_helpers.sh
     - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
   artifacts:
@@ -108,6 +98,8 @@ downtime_check:
 
 .rspec-base-migration:
   script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
     - source scripts/rspec_helpers.sh
     - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
 
@@ -193,6 +185,7 @@ gitlab:setup:
     # db/fixtures/development/04_project.rb thanks to SIZE=1 below
     - git clone https://gitlab.com/gitlab-org/gitlab-test.git
        /home/git/repositories/gitlab-org/gitlab-test.git
+    - run_timed_command "scripts/gitaly-test-build"
     - run_timed_command "scripts/gitaly-test-spawn"
     - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
   artifacts:
@@ -300,6 +293,8 @@ rspec-ee system pg11:
 .rspec-ee-base-geo:
   extends: .rspec-base-ee
   script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
     - source scripts/rspec_helpers.sh
     - scripts/prepare_postgres_fdw.sh
     - rspec_paralellized_job "--tag ~quarantine --tag geo"

scripts/rspec_helpers.sh

@@ -53,8 +53,6 @@ function rspec_simple_job() {
 
   export NO_KNAPSACK="1"
 
-  scripts/gitaly-test-spawn
-
   bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
 }
 
@@ -104,8 +102,6 @@ function rspec_paralellized_job() {
     fi
   fi
 
-  scripts/gitaly-test-spawn
-
   mkdir -p tmp/memory_test
 
   export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"