Merge branch 'improve-tests-metadata-jobs' into 'master'

Improve the 'retrieve-tests-metadata' and 'update-tests-metadata' jobs

See merge request gitlab-org/gitlab!17077

.gitlab-ci.yml

@@ -17,8 +17,7 @@ variables:
   GIT_DEPTH: "20"
   GIT_SUBMODULE_STRATEGY: "none"
   GET_SOURCES_ATTEMPTS: "3"
-  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
-  EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master-ee.json
+  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json
   FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
   BUILD_ASSETS_IMAGE: "false"
   ES_JAVA_OPTS: "-Xms256m -Xmx256m"

.gitlab/ci/frontend.gitlab-ci.yml

@@ -77,7 +77,7 @@ gitlab:assets:compile pull-cache:
     - .default-before_script
     - .assets-compile-cache
     - .only-code-qa-changes
-    - .use-pg
+    - .use-pg9
   stage: prepare
   script:
     - node --version
@@ -120,7 +120,7 @@ compile-assets pull-cache:
     - .default-only
     - .default-before_script
     - .only-code-changes
-    - .use-pg
+    - .use-pg9
   dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
 
 karma:

.gitlab/ci/global.gitlab-ci.yml

@@ -110,19 +110,34 @@
       - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
     kubernetes: active
 
-.use-pg:
+.use-pg9:
   services:
-    - name: postgres:9.6.14
+    - name: postgres:9.6
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
 
-.use-pg-10:
+.use-pg10:
   image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
   services:
     - name: postgres:10.9
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
 
+.use-pg9-ee:
+  services:
+    - name: postgres:9.6
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+
+.use-pg10-ee:
+  image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33"
+  services:
+    - name: postgres:10.9
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+
 .only-ee:
   only:
     variables:

.gitlab/ci/memory.gitlab-ci.yml

@@ -36,7 +36,7 @@ memory-static:
 memory-on-boot:
   extends:
     - .only-code-memory-job-base
-    - .use-pg-10
+    - .use-pg10
   variables:
     NODE_ENV: "production"
     RAILS_ENV: "production"

.gitlab/ci/rails.gitlab-ci.yml

@@ -33,31 +33,29 @@
     - .default-before_script
     - .only-code-qa-changes
 
+setup-test-env:
+  extends:
+    - .only-code-qa-rails-job-base
+    - .use-pg9
+  stage: prepare
+  script:
+    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
+    - scripts/gitaly-test-build  # Do not use 'bundle exec' here
+  artifacts:
+    expire_in: 7d
+    paths:
+      - tmp/tests
+      - config/secrets.yml
+      - vendor/gitaly-ruby
+  cache:
+    policy: pull-push
+
 .rspec-base:
   extends: .only-code-rails-job-base
   stage: test
   script:
-    - JOB_NAME=( $CI_JOB_NAME )
-    - TEST_TOOL=${JOB_NAME[0]}
-    - TEST_LEVEL=${JOB_NAME[1]}
-    - DATABASE=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec
-    - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH}
-    - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - export FLAKY_RSPEC_GENERATE_REPORT=true
-    - export CACHE_CLASSES=true
-    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
-    - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
-    - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
-    - scripts/gitaly-test-spawn
-    - date
-    - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")'
-    - mkdir -p tmp/memory_test
-    - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
-    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
-    - date
+    - source scripts/rspec_helpers.sh
+    - rspec_paralellized_job "--tag ~quarantine --tag ~geo"
   artifacts:
     expire_in: 31d
     when: always
@@ -71,78 +69,145 @@
     reports:
       junit: junit_rspec.xml
 
-.rspec-base-pg:
+.rspec-base-pg9:
   extends:
     - .rspec-base
-    - .use-pg
+    - .use-pg9
 
-.rspec-base-pg-10:
+.rspec-base-pg10:
   extends:
     - .rspec-base
-    - .use-pg-10
+    - .use-pg10
+    - .only-master
 
-setup-test-env:
-  extends:
-    - .only-code-qa-rails-job-base
-    - .use-pg
-  stage: prepare
-  script:
-    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
-    - scripts/gitaly-test-build  # Do not use 'bundle exec' here
-  artifacts:
-    expire_in: 7d
-    paths:
-      - tmp/tests
-      - config/secrets.yml
-      - vendor/gitaly-ruby
-  cache:
-    policy: pull-push
+rspec unit pg9:
+  extends: .rspec-base-pg9
+  parallel: 20
+
+rspec integration pg9:
+  extends: .rspec-base-pg9
+  parallel: 6
 
-rspec unit pg:
-  extends: .rspec-base-pg
+rspec system pg9:
+  extends: .rspec-base-pg9
+  parallel: 24
+
+rspec unit pg10:
+  extends: .rspec-base-pg10
   parallel: 20
 
-rspec integration pg:
-  extends: .rspec-base-pg
+rspec integration pg10:
+  extends: .rspec-base-pg10
   parallel: 6
 
-rspec system pg:
-  extends: .rspec-base-pg
+rspec system pg10:
+  extends: .rspec-base-pg10
   parallel: 24
 
-rspec unit pg-10:
+.rspec-ee-base-pg9:
+  extends:
+    - .rspec-base
+    - .only-ee
+    - .use-pg9-ee
+
+.rspec-ee-base-pg10:
+  extends:
+    - .rspec-base
+    - .only-ee
+    - .use-pg10-ee
+
+rspec-ee unit pg9:
+  extends: .rspec-ee-base-pg9
+  parallel: 7
+
+rspec-ee integration pg9:
+  extends: .rspec-ee-base-pg9
+  parallel: 3
+
+rspec-ee system pg9:
+  extends: .rspec-ee-base-pg9
+  parallel: 5
+
+rspec-ee unit pg10:
   extends:
-    - .rspec-base-pg-10
+    - .rspec-ee-base-pg10
     - .only-master
-  parallel: 20
+  parallel: 7
 
-rspec integration pg-10:
+rspec-ee integration pg10:
   extends:
-    - .rspec-base-pg-10
+    - .rspec-ee-base-pg10
     - .only-master
-  parallel: 6
+  parallel: 3
 
-rspec system pg-10:
+rspec-ee system pg10:
   extends:
-    - .rspec-base-pg-10
+    - .rspec-ee-base-pg10
     - .only-master
-  parallel: 24
+  parallel: 5
 
-rspec-fast-spec-helper:
-  extends: .rspec-base-pg
+.rspec-ee-base-geo:
+  extends:
+    - .rspec-base
+    - .only-ee
   script:
-    - bundle exec rspec spec/fast_spec_helper.rb
+    - source scripts/rspec_helpers.sh
+    - scripts/prepare_postgres_fdw.sh
+    - rspec_paralellized_job "--tag ~quarantine --tag geo"
+
+.rspec-ee-base-geo-pg9:
+  extends:
+    - .rspec-ee-base-geo
+    - .use-pg9-ee
+
+.rspec-ee-base-geo-pg10:
+  extends:
+    - .rspec-ee-base-geo
+    - .use-pg10-ee
+
+rspec-ee unit pg9 geo:
+  extends: .rspec-ee-base-geo-pg9
+  parallel: 2
+
+rspec-ee integration pg9 geo:
+  extends: .rspec-ee-base-geo-pg9
 
-rspec quarantine pg:
+rspec-ee system pg9 geo:
+  extends: .rspec-ee-base-geo-pg9
+
+rspec-ee unit pg10 geo:
+  extends: .rspec-ee-base-geo-pg10
+  parallel: 2
+
+rspec-ee integration pg10 geo:
+  extends: .rspec-ee-base-geo-pg10
+
+rspec-ee system pg10 geo:
+  extends: .rspec-ee-base-geo-pg10
+
+rspec quarantine pg9:
   extends:
-    - .rspec-base-pg
+    - .rspec-base-pg9
     - .only-master
+  variables:
+    RSPEC_OPTS: "--tag quarantine -- spec/"
   script:
-    - export NO_KNAPSACK=1 CACHE_CLASSES=true
-    - scripts/gitaly-test-spawn
-    - bin/rspec --color --format documentation --tag quarantine -- spec/
+    - source scripts/rspec_helpers.sh
+    - rspec_simple_job "${RSPEC_OPTS}"
   allow_failure: true
 
+rspec-ee quarantine pg9:
+  extends:
+    - rspec quarantine pg9
+    - .only-ee
+  variables:
+    RSPEC_OPTS: "--tag quarantine -- ee/spec/"
+
+rspec fast_spec_helper:
+  extends: .rspec-base-pg9
+  script:
+    - bin/rspec spec/fast_spec_helper.rb
+
 static-analysis:
   extends: .only-code-qa-rails-job-base
   dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
@@ -174,7 +239,7 @@ downtime_check:
 .db-job-base:
   extends:
     - .only-code-rails-job-base
-    - .use-pg
+    - .use-pg9
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
@@ -258,108 +323,6 @@ coverage:
       - coverage/assets/
       - tmp/memory_test/
 
-## EE-specific content
-.rspec-base-ee:
-  extends:
-    - .rspec-base
-    - .only-ee
-  script:
-    - JOB_NAME=( $CI_JOB_NAME )
-    - TEST_TOOL=${JOB_NAME[0]}
-    - TEST_LEVEL=${JOB_NAME[1]}
-    - DATABASE=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_ee_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec
-    - export CACHE_CLASSES=true
-    - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
-    - scripts/gitaly-test-spawn
-    - date
-    - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(ee/)).pattern(:${TEST_LEVEL})")'
-    - mkdir -p tmp/memory_test
-    - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
-    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
-    - date
-
-.rspec-base-pg-ee:
-  extends: .rspec-base-ee
-  services:
-    - name: postgres:9.6
-      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
-    - name: redis:alpine
-    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
-
-rspec unit pg ee:
-  extends: .rspec-base-pg-ee
-  parallel: 7
-
-rspec integration pg ee:
-  extends: .rspec-base-pg-ee
-  parallel: 3
-
-rspec system pg ee:
-  extends: .rspec-base-pg-ee
-  parallel: 5
-
-.rspec-base-geo:
-  extends: .rspec-base-ee
-  parallel: 3
-  script:
-    - JOB_NAME=( $CI_JOB_NAME )
-    - TEST_TOOL=${JOB_NAME[0]}
-    - TEST_LEVEL=${JOB_NAME[1]}
-    - DATABASE=${JOB_NAME[2]}
-    - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
-    - source scripts/prepare_postgres_fdw.sh
-    - scripts/gitaly-test-spawn
-    - mkdir -p tmp/memory_test
-    - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
-    - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo"
-
-rspec geo pg ee:
-  extends:
-    - .rspec-base-geo
-    - .use-pg
-  except:
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
-
-rspec geo pg-10 ee:
-  extends:
-    - .rspec-base-geo
-    - .use-pg-10
-  except:
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
-
-quick-rspec geo pg ee:
-  extends:
-    - .rspec-base-geo
-    - .use-pg
-  stage: quick-test
-  only:
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
-
-quick-rspec geo pg-10 ee:
-  extends:
-    - .rspec-base-geo
-    - .use-pg-10
-  stage: quick-test
-  only:
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
-
-rspec quarantine pg ee:
-  extends:
-    - rspec quarantine pg
-    - .only-ee
-  script:
-    - export NO_KNAPSACK=1 CACHE_CLASSES=true
-    - scripts/gitaly-test-spawn
-    - bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag quarantine -- ee/spec/
-
 db:rollback geo:
   extends:
     - db:rollback
@@ -367,5 +330,3 @@ db:rollback geo:
   script:
     - bundle exec rake geo:db:migrate VERSION=20170627195211
     - bundle exec rake geo:db:migrate
-
-## END of EE-specific content

.gitlab/ci/test-metadata.gitlab-ci.yml

 .tests-metadata-state:
   extends:
     - .default-only
+    - .only-code-changes
   variables:
     TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
     - source scripts/utils.sh
+  cache:
+    key: tests_metadata
+    paths:
+      - knapsack/
+      - rspec_flaky/
   artifacts:
     expire_in: 31d
     paths:
@@ -13,55 +19,29 @@
       - rspec_profiling/
 
 retrieve-tests-metadata:
-  extends:
-    - .tests-metadata-state
-    - .only-code-changes
+  extends: .tests-metadata-state
   stage: prepare
   cache:
-    key: tests_metadata
     policy: pull
   script:
-    - mkdir -p knapsack/${CI_PROJECT_NAME}/
-    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
-    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
-    - mkdir -p rspec_flaky/
-    - mkdir -p rspec_profiling/
-    - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
-    - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
-    - '[[ ! -d "ee/" ]] || wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
-    - '[[ ! -d "ee/" ]] || [[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+    - source scripts/rspec_helpers.sh
+    - retrieve_tests_metadata
 
 update-tests-metadata:
-  extends:
-    - .tests-metadata-state
-    - .only-code-changes
+  extends: .tests-metadata-state
   stage: post-test
   cache:
-    key: tests_metadata
-    paths:
-      - knapsack/
-      - rspec_flaky/
     policy: push
   script:
     - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
-    - echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}
-    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_node_*.json
-    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
-    - '[[ ! -d "ee/" ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
-    - '[[ ! -d "ee/" ]] || scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json'
-    - '[[ ! -d "ee/" ]] || [[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
-    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
-    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
-    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
-    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
-    - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
-    - scripts/insert-rspec-profiling-data
+    - source scripts/rspec_helpers.sh
+    - update_tests_metadata
   only:
     refs:
-      - master
+      - schedules
     variables:
-      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
-      - $CI_SERVER_HOST == "dev.gitlab.org"
+      # Only update the Knapsack metadata on GitLab.com/gitlab-org/gitlab
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab"
 
 flaky-examples-check:
   extends:

doc/development/testing_guide/ci.md

@@ -4,27 +4,24 @@
 
 Our current CI parallelization setup is as follows:
 
-1. The `knapsack` job in the prepare stage that is supposed to ensure we have a
-   `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file:
-   - The `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file is fetched
-     from S3, if it's not here we initialize the file with `{}`.
-1. Each `rspec x y` job are run with `knapsack rspec` and should have an evenly
-   distributed share of tests:
-   - It works because the jobs have access to the
-     `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` since the "artifacts
-     from all previous stages are passed by default".
+1. The `retrieve-tests-metadata` job in the `prepare` stage ensures we have a
+   `knapsack/report-master.json` file:
+   - The `knapsack/report-master.json` file is fetched from S3, if it's not here
+     we initialize the file with `{}`.
+1. Each `[rspec|rspec-ee] [unit|integration|system|geo] n m` job are run with
+   `knapsack rspec` and should have an evenly distributed share of tests:
+   - It works because the jobs have access to the `knapsack/report-master.json`
+     since the "artifacts from all previous stages are passed by default".
    - the jobs set their own report path to
-     `KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json`.
+     `"knapsack/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"`.
    - if knapsack is doing its job, test files that are run should be listed under
      `Report specs`, not under `Leftover specs`.
-1. The `update-knapsack` job takes all the
-   `knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json`
-   files from the `rspec x y` jobs and merge them all together into a single
-   `knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file that is then
-   uploaded to S3.
-
-After that, the next pipeline will use the up-to-date
-`knapsack/${CI_PROJECT_NAME}/rspec_report-master.json` file.
+1. The `update-tests-metadata` job (which only runs on scheduled pipelines for
+   [the canonical project](https://gitlab.com/gitlab-org/gitlab) takes all the
+   `knapsack/rspec*_pg_*.json` files and merge them all together into a single
+   `knapsack/report-master.json` file that is then uploaded to S3.
+
+After that, the next pipeline will use the up-to-date `knapsack/report-master.json` file.
 
 ## Monitoring
 

lib/quality/test_level.rb

@@ -53,11 +53,11 @@ module Quality
     end
 
     def pattern(level)
-      @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb"
+      @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*_spec.rb"
     end
 
     def regexp(level)
-      @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze
+      @regexps[level] ||= Regexp.new("#{prefix}spec/#{folders_regex(level)}").freeze
     end
 
     def level_for(file_path)
@@ -72,5 +72,27 @@ module Quality
         raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}."
       end
     end
+
+    private
+
+    def folders_pattern(level)
+      case level
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        '**'
+      else
+        "{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}"
+      end
+    end
+
+    def folders_regex(level)
+      case level
+      # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
+      when :all, :geo
+        ''
+      else
+        "(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})"
+      end
+    end
   end
 end

scripts/rspec_helpers.sh

+#!/bin/bash
+
+function retrieve_tests_metadata() {
+  mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
+
+  if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+    wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
+    wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+  fi
+}
+
+function update_tests_metadata() {
+  echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+
+  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "knapsack/rspec*_pg9_*.json"
+  if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
+    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  rm -f "knapsack/rspec*.json"
+
+  scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "rspec_flaky/all_*.json"
+
+  export FLAKY_RSPEC_GENERATE_REPORT="1"
+  scripts/prune-old-flaky-specs "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+
+  if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
+    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+  fi
+
+  rm -f "rspec_flaky/all_*.json" "rspec_flaky/new_*.json"
+
+  scripts/insert-rspec-profiling-data
+}
+
+function rspec_simple_job() {
+  local rspec_opts="${1}"
+
+  export NO_KNAPSACK="1"
+  export CACHE_CLASSES="true"
+
+  scripts/gitaly-test-spawn
+
+  bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml "${rspec_opts}"
+}
+
+function rspec_paralellized_job() {
+  read -ra job_name <<< "$CI_JOB_NAME"
+  local test_tool="${job_name[0]}"
+  local test_level="${job_name[1]}"
+  local database="${job_name[2]}"
+  local rspec_opts="${1}"
+  local spec_folder_prefix=""
+
+  if [[ "${test_tool}" =~ "-ee" ]]; then
+    spec_folder_prefix="ee/"
+  fi
+
+  export CACHE_CLASSES="true"
+  export KNAPSACK_LOG_LEVEL="debug"
+  export KNAPSACK_REPORT_PATH="knapsack/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+
+  if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
+    pattern=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+    export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
+  fi
+
+  echo "KNAPSACK_TEST_FILE_PATTERN: ${KNAPSACK_TEST_FILE_PATTERN}"
+
+  if [[ -d "ee/" ]]; then
+    export KNAPSACK_GENERATE_REPORT="true"
+    export FLAKY_RSPEC_GENERATE_REPORT="true"
+    export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+    export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+
+    cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
+
+    if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
+      echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"
+    fi
+
+    if [[ ! -f $NEW_FLAKY_RSPEC_REPORT_PATH ]]; then
+      echo "{}" > "${NEW_FLAKY_RSPEC_REPORT_PATH}"
+    fi
+  fi
+
+  scripts/gitaly-test-spawn
+
+  mkdir -p tmp/memory_test
+
+  export MEMORY_TEST_PATH="tmp/memory_test/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
+
+  knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
+
+  date
+}

spec/lib/quality/test_level_spec.rb

@@ -4,6 +4,20 @@ require 'fast_spec_helper'
 
 RSpec.describe Quality::TestLevel do
   describe '#pattern' do
+    context 'when level is all' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:all))
+        .to eq("spec/**{,/**/}*_spec.rb")
+      end
+    end
+
+    context 'when level is geo' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:geo))
+        .to eq("spec/**{,/**/}*_spec.rb")
+      end
+    end
+
     context 'when level is unit' do
       it 'returns a pattern' do
         expect(subject.pattern(:unit))
@@ -44,6 +58,20 @@ RSpec.describe Quality::TestLevel do
   end
 
   describe '#regexp' do
+    context 'when level is all' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:all))
+        .to eq(%r{spec/})
+      end
+    end
+
+    context 'when level is geo' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:geo))
+        .to eq(%r{spec/})
+      end
+    end
+
     context 'when level is unit' do
       it 'returns a regexp' do
         expect(subject.regexp(:unit))