Merge branch 'retry-failing-tests-2' into 'master'

ci: Retry failed specs in a new process after the initial run

See merge request gitlab-org/gitlab!76839

.gitignore

@@ -79,6 +79,7 @@ eslint-report.html
 /deprecations/
 /knapsack/
 /rspec_flaky/
+/rspec/
 /locale/**/LC_MESSAGES
 /locale/**/*.time_stamp
 /.rspec

.gitlab-ci.yml

@@ -69,10 +69,14 @@ variables:
   GET_SOURCES_ATTEMPTS: "3"
 
   KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json
-  FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
+  FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json
   RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json
   RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json
+  RSPEC_PROFILING_FOLDER_PATH: rspec/profiling
   FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json
+  RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
+  JUNIT_RESULT_FILE: rspec/junit_rspec.xml
+  JUNIT_RETRY_FILE: rspec/junit_rspec-retry.xml
 
   ES_JAVA_OPTS: "-Xms256m -Xmx256m"
   ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200"

.gitlab/ci/rails.gitlab-ci.yml

@@ -29,7 +29,9 @@
     GITLAB_USE_MODEL_LOAD_BALANCING: "true"
 
 .rspec-base:
-  extends: .rails-job-base
+  extends:
+    - .rails-job-base
+    - .base-artifacts
   stage: test
   variables:
     RUBY_GC_MALLOC_LIMIT: 67108864
@@ -40,6 +42,8 @@
   script:
     - !reference [.base-script, script]
     - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
+
+.base-artifacts:
   artifacts:
     expire_in: 31d
     when: always
@@ -48,16 +52,17 @@
       - crystalball/
       - deprecations/
       - knapsack/
-      - rspec_flaky/
-      - rspec_profiling/
+      - rspec/
       - tmp/capybara/
       - tmp/memory_test/
       - log/*.log
     reports:
-      junit: junit_rspec.xml
+      junit: ${JUNIT_RESULT_FILE}
 
 .rspec-base-migration:
-  extends: .rails:rules:ee-and-foss-migration
+  extends:
+    - .base-artifacts
+    - .rails:rules:ee-and-foss-migration
   script:
     - !reference [.base-script, script]
     - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
@@ -617,54 +622,26 @@ rspec:feature-flags:
         run_timed_command "bundle exec scripts/used-feature-flags";
       fi
 
-rspec:skipped-flaky-tests-report:
+rspec:flaky-tests-report:
   extends:
     - .default-retry
-    - .rails:rules:skipped-flaky-tests-report
-  image: ruby:2.7-alpine
+    - .rails:rules:flaky-tests-report
   stage: post-test
   # We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
   # so we use `dependencies` here.
-  dependencies:
-    # FOSS/EE jobs
-    - rspec migration pg12
-    - rspec unit pg12
-    - rspec integration pg12
-    - rspec system pg12
-    # FOSS/EE minimal jobs
-    - rspec migration pg12 minimal
-    - rspec unit pg12 minimal
-    - rspec integration pg12 minimal
-    - rspec system pg12 minimal
-    # EE jobs
-    - rspec-ee migration pg12
-    - rspec-ee unit pg12
-    - rspec-ee integration pg12
-    - rspec-ee system pg12
-    # EE minimal jobs
-    - rspec-ee migration pg12 minimal
-    - rspec-ee unit pg12 minimal
-    - rspec-ee integration pg12 minimal
-    - rspec-ee system pg12 minimal
-    # Geo jobs
-    - rspec-ee unit pg12 geo
-    - rspec-ee integration pg12 geo
-    - rspec-ee system pg12 geo
-    # Geo minimal jobs
-    - rspec-ee unit pg12 geo minimal
-    - rspec-ee integration pg12 geo minimal
-    - rspec-ee system pg12 geo minimal
+  dependencies: !reference ["rspec:coverage", "dependencies"]
   variables:
-    SKIPPED_FLAKY_TESTS_REPORT: skipped_flaky_tests_report.txt
+    SKIPPED_FLAKY_TESTS_REPORT_PATH: rspec/flaky/skipped_flaky_tests_report.txt
+    RETRIED_TESTS_REPORT_PATH: rspec/flaky/retried_tests_report.txt
   before_script:
-    - 'echo "SKIP_FLAKY_TESTS_AUTOMATICALLY: $SKIP_FLAKY_TESTS_AUTOMATICALLY"'
-    - mkdir -p rspec_flaky
+    - source scripts/utils.sh
+    - source scripts/rspec_helpers.sh
   script:
-    - find rspec_flaky/ -type f -name 'skipped_flaky_tests_*_report.txt' -exec cat {} + >> "${SKIPPED_FLAKY_TESTS_REPORT}"
+    - generate_flaky_tests_reports
   artifacts:
     expire_in: 31d
     paths:
-      - ${SKIPPED_FLAKY_TESTS_REPORT}
+      - rspec/
 
 # EE/FOSS: default refs (MRs, default branch, schedules) jobs #
 #######################################################

.gitlab/ci/rules.gitlab-ci.yml

@@ -121,9 +121,6 @@
 .if-security-pipeline-merge-result: &if-security-pipeline-merge-result
   if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == "gitlab-org/security" && $GITLAB_USER_LOGIN == "gitlab-release-tools-bot"'
 
-.if-skip-flaky-tests-automatically: &if-skip-flaky-tests-automatically
-  if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true"'
-
 ####################
 # Changes patterns #
 ####################
@@ -1436,13 +1433,16 @@
       when: never
     - changes: *code-backstage-patterns
 
-.rails:rules:skipped-flaky-tests-report:
+.rails:rules:flaky-tests-report:
   rules:
     - <<: *if-not-ee
       when: never
-    - <<: *if-skip-flaky-tests-automatically
+    - if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true" || $RETRY_FAILED_TESTS_IN_NEW_PROCESS == "true"'
       changes: *code-backstage-patterns
-    - changes: *ci-patterns
+      when: always
+    - if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true" || $RETRY_FAILED_TESTS_IN_NEW_PROCESS == "true"'
+      changes: *ci-patterns
+      when: always
 
 #########################
 # Static analysis rules #

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -6,8 +6,7 @@
     expire_in: 31d
     paths:
       - knapsack/
-      - rspec_flaky/
-      - rspec_profiling/
+      - rspec/
       - crystalball/
 
 retrieve-tests-metadata:
@@ -44,6 +43,6 @@ update-tests-metadata:
   script:
     - run_timed_command "retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document"
     - source ./scripts/rspec_helpers.sh
-    - test -f rspec_flaky/report-suite.json || echo -e "\e[31m" 'Consider add ~"pipeline:run-all-rspec" to run full rspec jobs' "\e[0m"
+    - test -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo -e "\e[31m" 'Consider add ~"pipeline:run-all-rspec" to run full rspec jobs' "\e[0m"
     - update_tests_metadata
     - update_tests_mapping

config/initializers/rspec_profiling.rb

@@ -62,7 +62,7 @@ RspecProfiling.configure do |config|
     config.collector = RspecProfilingExt::Collectors::CSVWithTimestamps
     config.csv_path = -> do
       prefix = "#{ENV['CI_JOB_NAME']}-".gsub(%r{[ /]}, '-') if ENV['CI_JOB_NAME']
-      "rspec_profiling/#{prefix}#{Time.now.to_i}-#{SecureRandom.hex(8)}-rspec-data.csv"
+      "#{ENV['RSPEC_PROFILING_FOLDER_PATH']}/#{prefix}#{Time.now.to_i}-#{SecureRandom.hex(8)}-rspec-data.csv"
     end
   end
 end

doc/development/pipelines.md

@@ -170,10 +170,19 @@ After that, the next pipeline uses the up-to-date `knapsack/report-master.json`
 
 ### Flaky tests
 
+#### Automatic skipping of flaky tests
+
 Tests that are [known to be flaky](testing_guide/flaky_tests.md#automatic-retries-and-flaky-tests-detection) are
 skipped unless the `$SKIP_FLAKY_TESTS_AUTOMATICALLY` variable is set to `false` or if the `~"pipeline:run-flaky-tests"`
 label is set on the MR.
 
+#### Automatic retry of failing tests in a separate process
+
+When the `$RETRY_FAILED_TESTS_IN_NEW_PROCESS` variable is set to `true`, RSpec tests that failed are automatically retried once in a separate
+RSpec process. The goal is to get rid of most side-effects from previous tests that may lead to a subsequent test failure.
+
+We keep track of retried tests in the `$RETRIED_TESTS_REPORT_FILE` file saved as artifact by the `rspec:flaky-tests-report` job.
+
 ### Monitoring
 
 The GitLab test suite is [monitored](performance.md#rspec-profiling) for the `main` branch, and any branch

scripts/insert-rspec-profiling-data

@@ -43,4 +43,4 @@ def insert_data(path)
   end
 end
 
-insert_data('rspec_profiling') if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?
+insert_data(ENV['RSPEC_PROFILING_FOLDER_PATH']) if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?

scripts/rspec_helpers.sh

 #!/usr/bin/env bash
 
 function retrieve_tests_metadata() {
-  mkdir -p $(dirname "$KNAPSACK_RSPEC_SUITE_REPORT_PATH") $(dirname "$FLAKY_RSPEC_SUITE_REPORT_PATH") rspec_profiling/
+  mkdir -p $(dirname "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}") $(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}") "${RSPEC_PROFILING_FOLDER_PATH}"
 
   if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
     if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
-      curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+      curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ||
+        echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
     fi
 
     if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-      curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+      curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" ||
+        curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/rspec_flaky/report-suite.json" || # temporary back-compat
+        echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
     fi
   else
     # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
@@ -31,7 +34,14 @@ function retrieve_tests_metadata() {
       fi
 
       if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
-        scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+        scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ||
+          scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "rspec_flaky/report-suite.json" || # temporary back-compat
+          echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+
+        # temporary back-compat
+        if [[ -f "rspec_flaky/report-suite.json" ]]; then
+          mv "rspec_flaky/report-suite.json" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+        fi
       fi
     else
       echo "test_metadata_job_id couldn't be found!"
@@ -42,21 +52,24 @@ function retrieve_tests_metadata() {
 }
 
 function update_tests_metadata() {
+  local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/"
+  local knapsack_folder_path="$(dirname "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}")/"
+
   echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
 
-  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
-  rm -f knapsack/rspec*.json
+  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ${knapsack_folder_path}rspec*.json
 
   export FLAKY_RSPEC_GENERATE_REPORT="true"
-  scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
+  scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ${rspec_flaky_folder_path}all_*.json
   scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
-  rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
 
   if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
     scripts/insert-rspec-profiling-data
   else
     echo "Not inserting profiling data as the pipeline is not a scheduled one."
   fi
+
+  cleanup_individual_job_reports
 }
 
 function retrieve_tests_mapping() {
@@ -158,12 +171,20 @@ function retrieve_previous_failed_tests() {
   scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --output-directory "${directory_for_output_reports}" --rspec-pg-regex "${rspec_pg_regex}" --rspec-ee-pg-regex "${rspec_ee_pg_regex}"
 }
 
-function rspec_simple_job() {
+function rspec_args() {
   local rspec_opts="${1}"
+  local junit_report_file="${2:-${JUNIT_RESULT_FILE}}"
+
+  echo "-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out ${junit_report_file} ${rspec_opts}"
+}
 
+function rspec_simple_job() {
   export NO_KNAPSACK="1"
 
-  eval "bin/rspec -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
+  local rspec_cmd="bin/rspec $(rspec_args "${1}" "${2}")"
+  echoinfo "Running RSpec command: ${rspec_cmd}"
+
+  eval "${rspec_cmd}"
 }
 
 function rspec_db_library_code() {
@@ -172,6 +193,24 @@ function rspec_db_library_code() {
   rspec_simple_job "-- ${db_files}"
 }
 
+function debug_rspec_variables() {
+  echoinfo "SKIP_FLAKY_TESTS_AUTOMATICALLY: ${SKIP_FLAKY_TESTS_AUTOMATICALLY}"
+  echoinfo "RETRY_FAILED_TESTS_IN_NEW_PROCESS: ${RETRY_FAILED_TESTS_IN_NEW_PROCESS}"
+
+  echoinfo "KNAPSACK_GENERATE_REPORT: ${KNAPSACK_GENERATE_REPORT}"
+  echoinfo "FLAKY_RSPEC_GENERATE_REPORT: ${FLAKY_RSPEC_GENERATE_REPORT}"
+
+  echoinfo "KNAPSACK_TEST_FILE_PATTERN: ${KNAPSACK_TEST_FILE_PATTERN}"
+  echoinfo "KNAPSACK_LOG_LEVEL: ${KNAPSACK_LOG_LEVEL}"
+  echoinfo "KNAPSACK_REPORT_PATH: ${KNAPSACK_REPORT_PATH}"
+
+  echoinfo "FLAKY_RSPEC_SUITE_REPORT_PATH: ${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+  echoinfo "FLAKY_RSPEC_REPORT_PATH: ${FLAKY_RSPEC_REPORT_PATH}"
+  echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH}"
+  echoinfo "SKIPPED_FLAKY_TESTS_REPORT_PATH: ${SKIPPED_FLAKY_TESTS_REPORT_PATH}"
+  echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}"
+}
+
 function rspec_paralellized_job() {
   read -ra job_name <<< "${CI_JOB_NAME}"
   local test_tool="${job_name[0]}"
@@ -179,6 +218,9 @@ function rspec_paralellized_job() {
   local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24'
   local rspec_opts="${1}"
   local spec_folder_prefixes=""
+  local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/"
+  local knapsack_folder_path="$(dirname "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}")/"
+  local rspec_run_status=0
 
   if [[ "${test_tool}" =~ "-ee" ]]; then
     spec_folder_prefixes="'ee/'"
@@ -193,7 +235,7 @@ function rspec_paralellized_job() {
   fi
 
   export KNAPSACK_LOG_LEVEL="debug"
-  export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json"
+  export KNAPSACK_REPORT_PATH="${knapsack_folder_path}${report_name}_report.json"
 
   # There's a bug where artifacts are sometimes not downloaded. Since specs can run without the Knapsack report, we can
   # handle the missing artifact gracefully here. See https://gitlab.com/gitlab-org/gitlab/-/issues/212349.
@@ -208,16 +250,14 @@ function rspec_paralellized_job() {
     export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
   fi
 
-  echo "KNAPSACK_TEST_FILE_PATTERN: ${KNAPSACK_TEST_FILE_PATTERN}"
-  echo "SKIP_FLAKY_TESTS_AUTOMATICALLY: ${SKIP_FLAKY_TESTS_AUTOMATICALLY}"
+  export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json"
+  export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json"
+  export SKIPPED_FLAKY_TESTS_REPORT_PATH="${rspec_flaky_folder_path}skipped_flaky_tests_${report_name}_report.txt"
+  export RETRIED_TESTS_REPORT_PATH="${rspec_flaky_folder_path}retried_tests_${report_name}_report.txt"
 
   if [[ -d "ee/" ]]; then
     export KNAPSACK_GENERATE_REPORT="true"
     export FLAKY_RSPEC_GENERATE_REPORT="true"
-    export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
-    export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${report_name}_report.json"
-    export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${report_name}_report.json"
-    export SKIPPED_FLAKY_TESTS_REPORT_PATH="rspec_flaky/skipped_flaky_tests_${report_name}_report.txt"
 
     if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
       echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"
@@ -228,19 +268,44 @@ function rspec_paralellized_job() {
     fi
   fi
 
+  debug_rspec_variables
+
   mkdir -p tmp/memory_test
 
   export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"
 
-  local rspec_args="-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
-
   if [[ -n $RSPEC_TESTS_MAPPING_ENABLED ]]; then
-    tooling/bin/parallel_rspec --rspec_args "${rspec_args}" --filter "tmp/matching_tests.txt"
+    tooling/bin/parallel_rspec --rspec_args "$(rspec_args)" --filter "tmp/matching_tests.txt" || rspec_run_status=$?
+  else
+    tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" || rspec_run_status=$?
+  fi
+
+  echoinfo "RSpec exited with ${rspec_run_status}."
+
+  # Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148
+  if [[ $rspec_run_status -ne 0 ]]; then
+    if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then
+      # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
+      local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
+      echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
+      echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
+
+      echoinfo "Retrying the failing examples in a new RSpec proces..."
+
+      install_junit_merge_gem
+
+      # Retry only the tests that failed on first try
+      rspec_simple_job "--only-failures" "${JUNIT_RETRY_FILE}"
+      rspec_run_status=$?
+
+      # Merge the JUnit report from retry into the first-try report
+      junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
+    fi
   else
-    tooling/bin/parallel_rspec --rspec_args "${rspec_args}"
+    echosuccess "No examples to retry, congrats!"
   fi
 
-  date
+  exit $rspec_run_status
 }
 
 function rspec_rerun_previous_failed_tests() {
@@ -330,3 +395,30 @@ function generate_frontend_fixtures_mapping() {
 
   rspec_simple_job "--pattern \"${pattern}\""
 }
+
+function cleanup_individual_job_reports() {
+  local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/"
+  local knapsack_folder_path="$(dirname "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}")/"
+
+  rm -rf ${knapsack_folder_path}rspec*.json \
+    ${rspec_flaky_folder_path}all_*.json \
+    ${rspec_flaky_folder_path}new_*.json \
+    ${rspec_flaky_folder_path}skipped_flaky_tests_*_report.txt \
+    ${rspec_flaky_folder_path}retried_tests_*_report.txt \
+    ${RSPEC_LAST_RUN_RESULTS_FILE} \
+    ${RSPEC_PROFILING_FOLDER_PATH}/**/*
+  rmdir ${RSPEC_PROFILING_FOLDER_PATH} || true
+}
+
+function generate_flaky_tests_reports() {
+  local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/"
+
+  debug_rspec_variables
+
+  mkdir -p ${rspec_flaky_folder_path}
+
+  find ${rspec_flaky_folder_path} -type f -name 'skipped_flaky_tests_*_report.txt' -exec cat {} + >> "${SKIPPED_FLAKY_TESTS_REPORT_PATH}"
+  find ${rspec_flaky_folder_path} -type f -name 'retried_tests_*_report.txt' -exec cat {} + >> "${RETRIED_TESTS_REPORT_PATH}"
+
+  cleanup_individual_job_reports
+}

scripts/utils.sh

@@ -64,16 +64,20 @@ function setup_db() {
 }
 
 function install_api_client_dependencies_with_apk() {
-  apk add --update openssl curl jq
+  run_timed_command "apk add --update openssl curl jq"
 }
 
 function install_gitlab_gem() {
-  gem install httparty --no-document --version 0.18.1
-  gem install gitlab --no-document --version 4.17.0
+  run_timed_command "gem install httparty --no-document --version 0.18.1"
+  run_timed_command "gem install gitlab --no-document --version 4.17.0"
 }
 
 function install_tff_gem() {
-  gem install test_file_finder --version 0.1.1
+  run_timed_command "gem install test_file_finder --no-document --version 0.1.1"
+}
+
+function install_junit_merge_gem() {
+  run_timed_command "gem install junit_merge --no-document --version 0.1.2"
 }
 
 function run_timed_command() {

spec/spec_helper.rb

@@ -112,10 +112,10 @@ RSpec.configure do |config|
     # falling back to all tests when there is no `:focus` example.
     config.filter_run focus: true
     config.run_all_when_everything_filtered = true
+  end
 
   # Re-run failures locally with `--only-failures`
-    config.example_status_persistence_file_path = './spec/examples.txt'
-  end
+  config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', './spec/examples.txt')
 
   config.define_derived_metadata(file_path: %r{(ee)?/spec/.+_spec\.rb\z}) do |metadata|
     location = metadata[:location]
@@ -208,7 +208,9 @@ RSpec.configure do |config|
     config.exceptions_to_hard_fail = [DeprecationToolkitEnv::DeprecationBehaviors::SelectiveRaise::RaiseDisallowedDeprecation]
   end
 
-  if ENV['FLAKY_RSPEC_GENERATE_REPORT']
+  require_relative '../tooling/rspec_flaky/config'
+
+  if RspecFlaky::Config.generate_report?
     require_relative '../tooling/rspec_flaky/listener'
 
     config.reporter.register_listener(

spec/support/flaky_tests.rb

@@ -4,14 +4,14 @@ return unless ENV['CI']
 return if ENV['SKIP_FLAKY_TESTS_AUTOMATICALLY'] == "false"
 return if ENV['CI_MERGE_REQUEST_LABELS'].to_s.include?('pipeline:run-flaky-tests')
 
+require_relative '../../tooling/rspec_flaky/config'
 require_relative '../../tooling/rspec_flaky/report'
 
 RSpec.configure do |config|
   $flaky_test_example_ids = begin # rubocop:disable Style/GlobalVars
-    raise "$SUITE_FLAKY_RSPEC_REPORT_PATH is empty." if ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'].to_s.empty?
-    raise "#{ENV['SUITE_FLAKY_RSPEC_REPORT_PATH']} doesn't exist" unless File.exist?(ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'])
+    raise "#{RspecFlaky::Config.suite_flaky_examples_report_path} doesn't exist" unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path)
 
-    RspecFlaky::Report.load(ENV['SUITE_FLAKY_RSPEC_REPORT_PATH']).map { |_, flaky_test_data| flaky_test_data.to_h[:example_id] }
+    RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).map { |_, flaky_test_data| flaky_test_data.to_h[:example_id] }
   rescue => e # rubocop:disable Style/RescueStandardError
     puts e
     []
@@ -29,8 +29,9 @@ RSpec.configure do |config|
   end
 
   config.after(:suite) do
-    next unless ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH']
+    next unless RspecFlaky::Config.skipped_flaky_tests_report_path
+    next if $skipped_flaky_tests_report.empty? # rubocop:disable Style/GlobalVars
 
-    File.write(ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'], "#{$skipped_flaky_tests_report.join("\n")}\n") # rubocop:disable Style/GlobalVars
+    File.write(RspecFlaky::Config.skipped_flaky_tests_report_path, "#{ENV['CI_JOB_URL']}\n#{$skipped_flaky_tests_report.join("\n")}\n\n") # rubocop:disable Style/GlobalVars
   end
 end

spec/tooling/lib/tooling/parallel_rspec_runner_spec.rb

@@ -18,7 +18,9 @@ RSpec.describe Tooling::ParallelRSpecRunner do # rubocop:disable RSpec/FilePath
       allow(File).to receive(:exist?).with(filter_tests_file).and_return(true)
       allow(File).to receive(:read).and_call_original
       allow(File).to receive(:read).with(filter_tests_file).and_return(filter_tests)
-      allow(subject).to receive(:exec)
+      allow(Process).to receive(:spawn)
+      allow(Process).to receive(:wait)
+      allow(Process).to receive(:last_status).and_return(double(exitstatus: 0))
     end
 
     subject { described_class.new(allocator: allocator, filter_tests_file: filter_tests_file, rspec_args: rspec_args) }
@@ -86,7 +88,7 @@ RSpec.describe Tooling::ParallelRSpecRunner do # rubocop:disable RSpec/FilePath
     end
 
     def expect_command(cmd)
-      expect(subject).to receive(:exec).with(*cmd)
+      expect(Process).to receive(:spawn).with(*cmd)
     end
   end
 end

spec/tooling/rspec_flaky/config_spec.rb

@@ -11,9 +11,10 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   before do
     # Stub these env variables otherwise specs don't behave the same on the CI
     stub_env('FLAKY_RSPEC_GENERATE_REPORT', nil)
-    stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', nil)
+    stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', nil)
     stub_env('FLAKY_RSPEC_REPORT_PATH', nil)
     stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', nil)
+    stub_env('SKIPPED_FLAKY_TESTS_REPORT_PATH', nil)
     # Ensure the behavior is the same locally and on CI (where Rails is defined since we run this test as part of the whole suite), i.e. Rails isn't defined
     allow(described_class).to receive(:rails_path).and_wrap_original do |method, path|
       path
@@ -51,15 +52,15 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   end
 
   describe '.suite_flaky_examples_report_path' do
-    context "when ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] is not set" do
+    context "when ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(described_class.suite_flaky_examples_report_path).to eq('rspec_flaky/suite-report.json')
+        expect(described_class.suite_flaky_examples_report_path).to eq('rspec/flaky/suite-report.json')
       end
     end
 
-    context "when ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] is set" do
+    context "when ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] is set" do
       before do
-        stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', 'foo/suite-report.json')
+        stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', 'foo/suite-report.json')
       end
 
       it 'returns the value of the env variable' do
@@ -71,7 +72,7 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   describe '.flaky_examples_report_path' do
     context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(described_class.flaky_examples_report_path).to eq('rspec_flaky/report.json')
+        expect(described_class.flaky_examples_report_path).to eq('rspec/flaky/report.json')
       end
     end
 
@@ -89,7 +90,7 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
   describe '.new_flaky_examples_report_path' do
     context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is not set" do
       it 'returns the default path' do
-        expect(described_class.new_flaky_examples_report_path).to eq('rspec_flaky/new-report.json')
+        expect(described_class.new_flaky_examples_report_path).to eq('rspec/flaky/new-report.json')
       end
     end
 
@@ -103,4 +104,22 @@ RSpec.describe RspecFlaky::Config, :aggregate_failures do
       end
     end
   end
+
+  describe '.skipped_flaky_tests_report_path' do
+    context "when ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'] is not set" do
+      it 'returns the default path' do
+        expect(described_class.skipped_flaky_tests_report_path).to eq('rspec/flaky/skipped_flaky_tests_report.txt')
+      end
+    end
+
+    context "when ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'] is set" do
+      before do
+        stub_env('SKIPPED_FLAKY_TESTS_REPORT_PATH', 'foo/skipped_flaky_tests_report.txt')
+      end
+
+      it 'returns the value of the env variable' do
+        expect(described_class.skipped_flaky_tests_report_path).to eq('foo/skipped_flaky_tests_report.txt')
+      end
+    end
+  end
 end

spec/tooling/rspec_flaky/listener_spec.rb

@@ -54,7 +54,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
   before do
     # Stub these env variables otherwise specs don't behave the same on the CI
     stub_env('CI_JOB_URL', nil)
-    stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', nil)
+    stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', nil)
   end
 
   describe '#initialize' do
@@ -73,11 +73,11 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
       it_behaves_like 'a valid Listener instance'
     end
 
-    context 'when SUITE_FLAKY_RSPEC_REPORT_PATH is set' do
+    context 'when FLAKY_RSPEC_SUITE_REPORT_PATH is set' do
       let(:report_file_path) { 'foo/report.json' }
 
       before do
-        stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', report_file_path)
+        stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', report_file_path)
       end
 
       context 'and report file exists' do

tooling/bin/parallel_rspec

@@ -16,4 +16,4 @@ OptionParser.new do |opts|
   end
 end.parse!
 
-Tooling::ParallelRSpecRunner.run(**options)
+exit Tooling::ParallelRSpecRunner.run(**options)

tooling/lib/tooling/parallel_rspec_runner.rb

@@ -38,12 +38,8 @@ module Tooling
       Knapsack.logger.info tests_to_run
       Knapsack.logger.info
 
-      if tests_to_run.empty?
-        Knapsack.logger.info 'No tests to run on this node, exiting.'
-        return
-      end
-
-      exec(*rspec_command)
+      Process.wait Process.spawn(*rspec_command)
+      Process.last_status.exitstatus
     end
 
     private

tooling/rspec_flaky/config.rb

@@ -7,15 +7,19 @@ module RspecFlaky
     end
 
     def self.suite_flaky_examples_report_path
-      ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/suite-report.json")
+      ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || rails_path("rspec/flaky/suite-report.json")
     end
 
     def self.flaky_examples_report_path
-      ENV['FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/report.json")
+      ENV['FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/report.json")
     end
 
     def self.new_flaky_examples_report_path
-      ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec_flaky/new-report.json")
+      ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/new-report.json")
+    end
+
+    def self.skipped_flaky_tests_report_path
+      ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'] || rails_path("rspec/flaky/skipped_flaky_tests_report.txt")
     end
 
     def self.rails_path(path)