Merge branch...

Merge branch '351341-rspec-retrying-only-failures-doesn-t-run-any-examples-and-crystalball-report-is-empty-as-a' into 'master'

ci: Fix the retrying of failing specs for `-ee` jobs

See merge request gitlab-org/gitlab!79510

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -8,6 +8,7 @@
       - knapsack/
       - rspec/
       - crystalball/
+    when: always
 
 retrieve-tests-metadata:
   extends:

scripts/rspec_helpers.sh

@@ -209,6 +209,8 @@ function debug_rspec_variables() {
   echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH}"
   echoinfo "SKIPPED_FLAKY_TESTS_REPORT_PATH: ${SKIPPED_FLAKY_TESTS_REPORT_PATH}"
   echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}"
+
+  echoinfo "CRYSTALBALL: ${CRYSTALBALL}"
 }
 
 function rspec_paralellized_job() {
@@ -245,11 +247,7 @@ function rspec_paralellized_job() {
 
   cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
 
-  if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
-    pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
-    export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
-  fi
-
+  export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
   export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json"
   export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json"
   export SKIPPED_FLAKY_TESTS_REPORT_PATH="${rspec_flaky_folder_path}skipped_flaky_tests_${report_name}_report.txt"
@@ -285,21 +283,7 @@ function rspec_paralellized_job() {
   # Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148
   if [[ $rspec_run_status -ne 0 ]]; then
     if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then
-      # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
-      local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
-      echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
-      echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
-
-      echoinfo "Retrying the failing examples in a new RSpec proces..."
-
-      install_junit_merge_gem
-
-      # Retry only the tests that failed on first try
-      rspec_simple_job "--only-failures" "${JUNIT_RETRY_FILE}"
-      rspec_run_status=$?
-
-      # Merge the JUnit report from retry into the first-try report
-      junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
+      $rspec_run_status=$(retry_failed_rspec_examples)
     fi
   else
     echosuccess "No examples to retry, congrats!"
@@ -308,6 +292,31 @@ function rspec_paralellized_job() {
   exit $rspec_run_status
 }
 
+function retry_failed_rspec_examples() {
+  local rspec_run_status=0
+
+  # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
+  local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
+  echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
+  echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
+
+  echoinfo "Retrying the failing examples in a new RSpec proces..."
+
+  install_junit_merge_gem
+
+  # Disable Crystalball on retry to not overwrite the existing report
+  export CRYSTALBALL="false"
+
+  # Retry only the tests that failed on first try
+  rspec_simple_job "--only-failures --pattern \"${KNAPSACK_TEST_FILE_PATTERN}\"" "${JUNIT_RETRY_FILE}"
+  rspec_run_status=$?
+
+  # Merge the JUnit report from retry into the first-try report
+  junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
+
+  return $rspec_run_status
+}
+
 function rspec_rerun_previous_failed_tests() {
   local test_file_count_threshold=${RSPEC_PREVIOUS_FAILED_TEST_FILE_COUNT_THRESHOLD:-10}
   local matching_tests_file=${1}

spec/crystalball_env.rb

@@ -6,7 +6,7 @@ module CrystalballEnv
   extend self
 
   def start!
-    return unless ENV['CRYSTALBALL']
+    return unless ENV['CRYSTALBALL'] == 'true'
 
     require 'crystalball'
     require_relative '../tooling/lib/tooling/crystalball/coverage_lines_execution_detector'

spec/tooling/lib/tooling/test_map_generator_spec.rb

@@ -39,11 +39,23 @@ RSpec.describe Tooling::TestMapGenerator do
       YAML
     end
 
+    let(:yaml3) do
+      <<~YAML
+        ---
+        :type: Crystalball::ExecutionMap
+        :commit: 74056e8d9cf3773f43faa1cf5416f8779c8284c9
+        :timestamp: 1602671965
+        :version:
+        ---
+      YAML
+    end
+
     let(:pathname) { instance_double(Pathname) }
 
     before do
       stub_file_read('yaml1.yml', content: yaml1)
       stub_file_read('yaml2.yml', content: yaml2)
+      stub_file_read('yaml3.yml', content: yaml3)
     end
 
     context 'with single yaml' do
@@ -74,6 +86,10 @@ RSpec.describe Tooling::TestMapGenerator do
           expect(subject.mapping[file]).to match_array(tests)
         end
       end
+
+      it 'displays a warning when report has no examples' do
+        expect { subject.parse('yaml3.yml') }.to output(%|No examples in yaml3.yml! Metadata: {:type=>"Crystalball::ExecutionMap", :commit=>"74056e8d9cf3773f43faa1cf5416f8779c8284c9", :timestamp=>1602671965, :version=>nil}\n|).to_stdout
+      end
     end
 
     context 'with multiple yamls' do

tooling/lib/tooling/test_map_generator.rb

@@ -12,7 +12,12 @@ module Tooling
     def parse(yaml_files)
       Array(yaml_files).each do |yaml_file|
         data = File.read(yaml_file)
-        _metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+        metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+
+        if example_groups.nil?
+          puts "No examples in #{yaml_file}! Metadata: #{metadata}"
+          next
+        end
 
         example_groups.each do |example_id, files|
           files.each do |file|