[EE] Reorganize test jobs by level

Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

@@ -19,10 +19,8 @@ variables:
   EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master-ee.json
 
 before_script:
-  - bundle --version
   - date
   - source scripts/utils.sh
-  - date
   - source scripts/prepare_build.sh
   - date
 

.gitlab/ci/rails.gitlab-ci.yml

@@ -6,13 +6,13 @@
 
 .use-pg-10: &use-pg-10
   services:
-    - name: postgres:10.0
+    - name: postgres:10.7
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
 
 .use-mysql: &use-mysql
   services:
-    - mysql:5.7.24
+    - mysql:5.7
     - redis:alpine
 
 .only-schedules-master: &only-schedules-master
@@ -53,8 +53,10 @@
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - TEST_TOOL=${JOB_NAME[0]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
-    - export KNAPSACK_GENERATE_REPORT=true
+    - TEST_LEVEL=${JOB_NAME[1]}
+    - DATABASE=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec
     - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH}
     - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
@@ -64,7 +66,10 @@
     - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
     - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
     - scripts/gitaly-test-spawn
-    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag ~geo"
+    - date
+    - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")'
+    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
+    - date
   artifacts:
     expire_in: 31d
     when: always
@@ -141,19 +146,47 @@ setup-test-env:
   except:
     - /(^docs[\/-].*|.*-docs$)/
 
-rspec-pg:
+rspec unit pg:
+  <<: *rspec-metadata-pg
+  parallel: 20
+
+rspec integration pg:
+  <<: *rspec-metadata-pg
+  parallel: 6
+
+rspec system pg:
   <<: *rspec-metadata-pg
-  parallel: 50
+  parallel: 24
+
+rspec unit pg-10:
+  <<: *rspec-metadata-pg-10
+  <<: *only-schedules-master
+  parallel: 20
+
+rspec integration pg-10:
+  <<: *rspec-metadata-pg-10
+  <<: *only-schedules-master
+  parallel: 6
 
-rspec-pg-10:
+rspec system pg-10:
   <<: *rspec-metadata-pg-10
   <<: *only-schedules-master
-  parallel: 50
+  parallel: 24
 
-rspec-mysql:
+rspec unit mysql:
   <<: *rspec-metadata-mysql
   <<: *only-schedules-master
-  parallel: 50
+  parallel: 20
+
+rspec integration mysql:
+  <<: *rspec-metadata-mysql
+  <<: *only-schedules-master
+  parallel: 6
+
+rspec system mysql:
+  <<: *rspec-metadata-mysql
+  <<: *only-schedules-master
+  parallel: 24
 
 rspec-fast-spec-helper:
   <<: *rspec-metadata-pg
@@ -165,7 +198,7 @@ rspec-fast-spec-helper:
   script:
     - export CACHE_CLASSES=true
     - scripts/gitaly-test-spawn
-    - bin/rspec --color --format documentation --tag quarantine spec/
+    - bin/rspec --color --format documentation --tag quarantine -- spec/
 
 rspec-pg-quarantine:
   <<: *rspec-metadata-pg
@@ -294,22 +327,12 @@ coverage:
     - /(^qa[\/-].*|.*-qa$)/
 
 ## EE-specific content
-
-.use-pg-9-6: &use-pg-9-6
-  services:
-    - postgres:9.6
-    - redis:alpine
-
-.use-pg-10-2: &use-pg-10-2
-  services:
-    - postgres:10.2
-    - redis:alpine
-
 .use-pg-with-elasticsearch: &use-pg-with-elasticsearch
   services:
-    - postgres:9.6
-    - redis:alpine
-    - docker.elastic.co/elasticsearch/elasticsearch:5.6.12
+    - name: postgres:9.6
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+    - name: docker.elastic.co/elasticsearch/elasticsearch:5.6.12
 
 .use-mysql-with-elasticsearch: &use-mysql-with-elasticsearch
   services:
@@ -323,21 +346,54 @@ coverage:
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - TEST_TOOL=${JOB_NAME[0]}
-    - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - TEST_LEVEL=${JOB_NAME[1]}
+    - DATABASE=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_ee_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true KNAPSACK_LOG_LEVEL=debug KNAPSACK_TEST_DIR=spec
+    - export CACHE_CLASSES=true
     - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - scripts/gitaly-test-spawn
-    - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag ~geo"
+    - date
+    - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(ee/)).pattern(:${TEST_LEVEL})")'
+    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
+    - date
 
-.rspec-ee-pg: &rspec-ee-pg
+.rspec-metadata-pg-ee: &rspec-metadata-pg-ee
   <<: *rspec-metadata-ee
   <<: *use-pg-with-elasticsearch
 
-.rspec-ee-mysql: &rspec-ee-mysql
+.rspec-metadata-mysql-ee: &rspec-metadata-mysql-ee
   <<: *rspec-metadata-ee
   <<: *use-mysql-with-elasticsearch
 
-.rspec-geo: &rspec-metadata-geo
+rspec unit pg ee:
+  <<: *rspec-metadata-pg-ee
+  parallel: 7
+
+rspec integration pg ee:
+  <<: *rspec-metadata-pg-ee
+  parallel: 3
+
+rspec system pg ee:
+  <<: *rspec-metadata-pg-ee
+  parallel: 5
+
+rspec unit mysql ee:
+  <<: *rspec-metadata-mysql-ee
+  <<: *only-schedules-master
+  parallel: 7
+
+rspec integration mysql ee:
+  <<: *rspec-metadata-mysql-ee
+  <<: *only-schedules-master
+  parallel: 3
+
+rspec system mysql ee:
+  <<: *rspec-metadata-mysql-ee
+  <<: *only-schedules-master
+  parallel: 5
+
+.rspec-metadata-pg-geo: &rspec-metadata-pg-geo
   <<: *rspec-metadata
   stage: test
   script:
@@ -350,45 +406,9 @@ coverage:
     - scripts/gitaly-test-spawn
     - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo"
 
-.rspec-geo-pg-9-6: &rspec-metadata-pg-geo-9-6
-  <<: *rspec-metadata-geo
-  <<: *use-pg-9-6
-
-.rspec-geo-pg-10-2: &rspec-metadata-pg-geo-10-2
-  <<: *rspec-metadata-geo
-  <<: *use-pg-10-2
-
-.migration-paths-upgrade-ce-to-ee: &migration-paths-upgrade-ce-to-ee
-  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
-  variables:
-    SETUP_DB: "false"
-  script:
-    - ruby -r./scripts/ee_specific_check/ee_specific_check -e'EESpecificCheck.fetch_remote_ce_branch'
-    - git checkout -f FETCH_HEAD
-    - . scripts/utils.sh
-    - . scripts/prepare_build.sh
-    - date
-    - setup_db
-    - date
-    - git checkout -f $CI_COMMIT_SHA
-    - date
-    - . scripts/prepare_build.sh
-    - date
-    - bundle exec rake db:migrate
-  dependencies:
-    - setup-test-env
-
-rspec-pg-ee:
-  <<: *rspec-ee-pg
-  parallel: 10
-
-rspec-mysql-ee:
-  <<: *rspec-ee-mysql
-  <<: *only-schedules-master
-  parallel: 10
-
 rspec-pg-ee geo:
-  <<: *rspec-metadata-pg-geo-9-6
+  <<: *rspec-metadata-pg-geo
+  <<: *use-pg
   parallel: 3
   except:
     - /(^geo[\/-].*|.*-geo$)/
@@ -396,7 +416,8 @@ rspec-pg-ee geo:
     - /(^qa[\/-].*|.*-qa$)/
 
 rspec-pg-10-ee geo:
-  <<: *rspec-metadata-pg-geo-10-2
+  <<: *rspec-metadata-pg-geo
+  <<: *use-pg-10
   parallel: 3
   except:
     - /(^geo[\/-].*|.*-geo$)/
@@ -404,33 +425,54 @@ rspec-pg-10-ee geo:
     - /(^qa[\/-].*|.*-qa$)/
 
 quick-rspec-pg-ee geo:
-  <<: *rspec-metadata-pg-geo-9-6
+  <<: *rspec-metadata-pg-geo
+  <<: *use-pg
   stage: quick-test
   only:
     - /(^geo[\/-].*|.*-geo$)/
 
 quick-rspec-pg-10-ee geo:
-  <<: *rspec-metadata-pg-geo-10-2
+  <<: *rspec-metadata-pg-geo
+  <<: *use-pg-10
   stage: quick-test
   only:
     - /(^geo[\/-].*|.*-geo$)/
 
 .rspec-quarantine-ee: &rspec-quarantine-ee
   <<: *only-schedules-master
+  allow_failure: true
   script:
-    - export CACHE_CLASSES=true
+    - export NO_KNAPSACK=1 CACHE_CLASSES=true
     - scripts/gitaly-test-spawn
-    - bin/rspec --color --format documentation --tag quarantine ee/spec/
+    - bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag quarantine -- ee/spec/
 
-rspec-pg-quarantine-ee:
-  <<: *rspec-metadata-pg
+rspec quarantine pg ee:
   <<: *rspec-quarantine-ee
-  allow_failure: true
+  <<: *rspec-metadata-pg
 
-rspec-mysql-quarantine-ee:
-  <<: *rspec-metadata-mysql
+rspec quarantine mysql ee:
   <<: *rspec-quarantine-ee
-  allow_failure: true
+  <<: *rspec-metadata-mysql
+
+.migration-paths-upgrade-ce-to-ee: &migration-paths-upgrade-ce-to-ee
+  extends: .dedicated-no-docs-and-no-qa-pull-cache-job
+  variables:
+    SETUP_DB: "false"
+  script:
+    - ruby -r./scripts/ee_specific_check/ee_specific_check -e'EESpecificCheck.fetch_remote_ce_branch'
+    - git checkout -f FETCH_HEAD
+    - . scripts/utils.sh
+    - . scripts/prepare_build.sh
+    - date
+    - setup_db
+    - date
+    - git checkout -f $CI_COMMIT_SHA
+    - date
+    - . scripts/prepare_build.sh
+    - date
+    - bundle exec rake db:migrate
+  dependencies:
+    - setup-test-env
 
 migration:upgrade-pg-ce-to-ee:
   <<: *migration-paths-upgrade-ce-to-ee

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -42,14 +42,14 @@ update-tests-metadata:
     policy: push
   script:
     - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
-    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
-    - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg-ee_*node_*.json
-    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
-    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
+    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_node_*.json
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
+    - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
-    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
+    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
+    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
+    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
     - scripts/insert-rspec-profiling-data
   only:

doc/development/rake_tasks.md

@@ -108,11 +108,13 @@ To make sure that indices still fit. You could find great details in:
 
 In order to run the test you can use the following commands:
 
-- `rake spec` to run the rspec suite
-- `rake karma` to run the karma test suite
-- `rake gitlab:test` to run all the tests
+- `bin/rake spec` to run the rspec suite
+- `bin/rake spec:unit` to run the only the unit tests
+- `bin/rake spec:integration` to run the only the integration tests
+- `bin/rake spec:system` to run the only the system tests
+- `bin/rake karma` to run the karma test suite
 
-Note: `rake spec` takes significant time to pass.
+Note: `bin/rake spec` takes significant time to pass.
 Instead of running full test suite locally you can save a lot of time by running
 a single test or directory related to your changes. After you submit merge request
 CI will run full test suite for you. Green CI status in the merge request means
@@ -121,6 +123,9 @@ full test suite is passed.
 Note: You can't run `rspec .` since this will try to run all the `_spec.rb`
 files it can find, also the ones in `/tmp`
 
+Note: You can pass RSpec command line options to the `spec:unit`,
+`spec:integration`, and `spec:system` tasks, e.g. `bin/rake "spec:unit[--tag ~geo --dry-run]"`.
+
 To run a single test file you can use:
 
 - `bin/rspec spec/controllers/commit_controller_spec.rb` for a rspec test

doc/development/testing_guide/testing_levels.md

@@ -4,12 +4,14 @@
 
 _This diagram demonstrates the relative priority of each test type we use. `e2e` stands for end-to-end._
 
-As of 2019-04-16, we have the following distribution of tests per level:
+As of 2019-05-01, we have the following distribution of tests per level:
 
-- 67 black-box tests at the system level (aka end-to-end or QA tests) in CE, 98 in EE. This represents 0.3% of all the CE tests (0.3% in EE).
-- 5,457 white-box tests at the system level (aka system or feature tests) in CE, 6,585 in EE. This represents 24.6% of all the CE tests (20.3% in EE).
-- 8,298 integration tests in CE, 10,633 in EE: 0.3% of all the CE tests (0.3% in EE). This represents 37.2% of all the CE tests (32.8% in EE).
-- 8,403 unit tests in CE, 15,090 in EE: 0.3% of all the CE tests (0.3% in EE). This represents 37.8% of all the CE tests (46.6% in EE).
+| Test level | Community Edition | Enterprise Edition  | Community + Enterprise Edition |
+| --------- | ---------- | -------------- | ----- |
+| Black-box tests at the system level (aka end-to-end or QA tests) | 68 (0.14%) | 31 (0.2%) | 99 (0.17%) |
+| White-box tests at the system level (aka system or feature tests) | 5,471 (11.9%) | 969 (7.4%) | 6440 (10.9%) |
+| Integration tests | 8,333 (18.2%) | 2,244 (17.2%) | 10,577 (17.9%) |
+| Unit tests | 32,031 (69.7%) | 9,778 (75.1%) | 41,809 (71%) |
 
 ## Unit tests
 

lib/quality/test_level.rb

+# frozen_string_literal: true
+
+module Quality
+  class TestLevel
+    UnknownTestLevelError = Class.new(StandardError)
+
+    TEST_LEVEL_FOLDERS = {
+      unit: %w[
+        bin
+        config
+        db
+        dependencies
+        factories
+        finders
+        frontend
+        graphql
+        helpers
+        initializers
+        javascripts
+        lib
+        migrations
+        models
+        policies
+        presenters
+        rack_servers
+        routing
+        rubocop
+        serializers
+        services
+        sidekiq
+        tasks
+        uploaders
+        validators
+        views
+        workers
+        elastic_integration
+      ],
+      integration: %w[
+        controllers
+        mailers
+        requests
+      ],
+      system: ['features']
+    }.freeze
+
+    attr_reader :prefix
+
+    def initialize(prefix = nil)
+      @prefix = prefix
+      @patterns = {}
+      @regexps = {}
+    end
+
+    def pattern(level)
+      @patterns[level] ||= "#{prefix}spec/{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}{,/**/}*_spec.rb".freeze
+    end
+
+    def regexp(level)
+      @regexps[level] ||= Regexp.new("#{prefix}spec/(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})").freeze
+    end
+
+    def level_for(file_path)
+      case file_path
+      when regexp(:unit)
+        :unit
+      when regexp(:integration)
+        :integration
+      when regexp(:system)
+        :system
+      else
+        raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}."
+      end
+    end
+  end
+end

lib/tasks/spec.rake

+# frozen_string_literal: true
+
+return if Rails.env.production?
+
 Rake::Task["spec"].clear if Rake::Task.task_defined?('spec')
 
 namespace :spec do
-  desc 'GitLab | Rspec | Run request specs'
+  desc 'GitLab | RSpec | Run unit tests'
+  RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args|
+    require_dependency 'quality/test_level'
+    t.pattern = Quality::TestLevel.new.pattern(:unit)
+    t.rspec_opts = args[:rspec_opts]
+  end
+
+  desc 'GitLab | RSpec | Run integration tests'
+  RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args|
+    require_dependency 'quality/test_level'
+    t.pattern = Quality::TestLevel.new.pattern(:integration)
+    t.rspec_opts = args[:rspec_opts]
+  end
+
+  desc 'GitLab | RSpec | Run system tests'
+  RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args|
+    require_dependency 'quality/test_level'
+    t.pattern = Quality::TestLevel.new.pattern(:system)
+    t.rspec_opts = args[:rspec_opts]
+  end
+
+  desc '[Deprecated] Use the "bin/rspec --tag api" instead'
   task :api do
     cmds = [
       %w(rake gitlab:setup),
@@ -10,7 +35,7 @@ namespace :spec do
     run_commands(cmds)
   end
 
-  desc 'GitLab | Rspec | Run feature specs'
+  desc '[Deprecated] Use the "spec:system" task instead'
   task :feature do
     cmds = [
       %w(rake gitlab:setup),
@@ -19,7 +44,7 @@ namespace :spec do
     run_commands(cmds)
   end
 
-  desc 'GitLab | Rspec | Run model specs'
+  desc '[Deprecated] Use "bin/rspec spec/models" instead'
   task :models do
     cmds = [
       %w(rake gitlab:setup),
@@ -28,7 +53,7 @@ namespace :spec do
     run_commands(cmds)
   end
 
-  desc 'GitLab | Rspec | Run service specs'
+  desc '[Deprecated] Use "bin/rspec spec/services" instead'
   task :services do
     cmds = [
       %w(rake gitlab:setup),
@@ -37,7 +62,7 @@ namespace :spec do
     run_commands(cmds)
   end
 
-  desc 'GitLab | Rspec | Run lib specs'
+  desc '[Deprecated] Use "bin/rspec spec/lib" instead'
   task :lib do
     cmds = [
       %w(rake gitlab:setup),
@@ -45,15 +70,6 @@ namespace :spec do
     ]
     run_commands(cmds)
   end
-
-  desc 'GitLab | Rspec | Run other specs'
-  task :other do
-    cmds = [
-      %w(rake gitlab:setup),
-      %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services)
-    ]
-    run_commands(cmds)
-  end
 end
 
 desc "GitLab | Run specs"

scripts/prepare_build.sh

@@ -5,6 +5,7 @@ export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
 export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet"
 
 if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
+  bundle --version
   bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check
 fi
 

spec/lib/quality/test_level_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Quality::TestLevel do
+  describe '#pattern' do
+    context 'when level is unit' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:unit))
+          .to eq("spec/{bin,config,db,dependencies,factories,finders,frontend,graphql,helpers,initializers,javascripts,lib,migrations,models,policies,presenters,rack_servers,routing,rubocop,serializers,services,sidekiq,tasks,uploaders,validators,views,workers,elastic_integration}{,/**/}*_spec.rb")
+      end
+    end
+
+    context 'when level is integration' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:integration))
+          .to eq("spec/{controllers,mailers,requests}{,/**/}*_spec.rb")
+      end
+    end
+
+    context 'when level is system' do
+      it 'returns a pattern' do
+        expect(subject.pattern(:system))
+          .to eq("spec/{features}{,/**/}*_spec.rb")
+      end
+    end
+
+    context 'with a prefix' do
+      it 'returns a pattern' do
+        expect(described_class.new('ee/').pattern(:system))
+          .to eq("ee/spec/{features}{,/**/}*_spec.rb")
+      end
+    end
+
+    describe 'performance' do
+      it 'memoizes the pattern for a given level' do
+        expect(subject.pattern(:system).object_id).to eq(subject.pattern(:system).object_id)
+      end
+
+      it 'freezes the pattern for a given level' do
+        expect(subject.pattern(:system)).to be_frozen
+      end
+    end
+  end
+
+  describe '#regexp' do
+    context 'when level is unit' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:unit))
+          .to eq(%r{spec/(bin|config|db|dependencies|factories|finders|frontend|graphql|helpers|initializers|javascripts|lib|migrations|models|policies|presenters|rack_servers|routing|rubocop|serializers|services|sidekiq|tasks|uploaders|validators|views|workers|elastic_integration)})
+      end
+    end
+
+    context 'when level is integration' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:integration))
+          .to eq(%r{spec/(controllers|mailers|requests)})
+      end
+    end
+
+    context 'when level is system' do
+      it 'returns a regexp' do
+        expect(subject.regexp(:system))
+          .to eq(%r{spec/(features)})
+      end
+    end
+
+    context 'with a prefix' do
+      it 'returns a regexp' do
+        expect(described_class.new('ee/').regexp(:system))
+          .to eq(%r{ee/spec/(features)})
+      end
+    end
+
+    describe 'performance' do
+      it 'memoizes the regexp for a given level' do
+        expect(subject.regexp(:system).object_id).to eq(subject.regexp(:system).object_id)
+      end
+
+      it 'freezes the regexp for a given level' do
+        expect(subject.regexp(:system)).to be_frozen
+      end
+    end
+  end
+
+  describe '#level_for' do
+    it 'returns the correct level for a unit test' do
+      expect(subject.level_for('spec/models/abuse_report_spec.rb')).to eq(:unit)
+    end
+
+    it 'returns the correct level for an integration test' do
+      expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
+    end
+
+    it 'returns the correct level for a system test' do
+      expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system)
+    end
+
+    it 'raises an error for an unknown level' do
+      expect { subject.level_for('spec/unknown/foo_spec.rb') }
+        .to raise_error(described_class::UnknownTestLevelError,
+          %r{Test level for spec/unknown/foo_spec.rb couldn't be set. Please rename the file properly or change the test level detection regexes in .+/lib/quality/test_level.rb.})
+    end
+  end
+end

spec/spec_helper.rb

@@ -46,6 +46,8 @@ Dir[Rails.root.join("spec/support/shared_contexts/*.rb")].each { |f| require f }
 Dir[Rails.root.join("spec/support/shared_examples/*.rb")].each { |f| require f }
 Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
 
+quality_level = Quality::TestLevel.new
+
 RSpec.configure do |config|
   config.use_transactional_fixtures = false
   config.use_instantiated_fixtures  = false
@@ -57,9 +59,10 @@ RSpec.configure do |config|
   config.infer_spec_type_from_file_location!
   config.full_backtrace = !!ENV['CI']
 
-  config.define_derived_metadata(file_path: %r{/spec/}) do |metadata|
+  config.define_derived_metadata(file_path: %r{(ee)?/spec/.+_spec\.rb\z}) do |metadata|
     location = metadata[:location]
 
+    metadata[:level] = quality_level.level_for(location)
     metadata[:api] = true if location =~ %r{/spec/requests/api/}
 
     # do not overwrite type if it's already set