Merge branch 'acunskis-mark-flaky-tests' into 'master'

E2E: Automatically mark test as flaky if previous failures are present

See merge request gitlab-org/gitlab!79790

qa/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 
 gem 'gitlab-qa', require: 'gitlab/qa'
 gem 'activesupport', '~> 6.1.4.6' # This should stay in sync with the root's Gemfile
-gem 'allure-rspec', '~> 2.15.0'
+gem 'allure-rspec', '~> 2.16.0'
 gem 'capybara', '~> 3.35.0'
 gem 'capybara-screenshot', '~> 1.0.23'
 gem 'rake', '~> 13'

qa/Gemfile.lock

@@ -19,10 +19,10 @@ GEM
       rack-test (>= 1.1.0, < 2.0)
       rest-client (>= 2.0.2, < 3.0)
       rspec (~> 3.8)
-    allure-rspec (2.15.0)
-      allure-ruby-commons (= 2.15.0)
+    allure-rspec (2.16.1)
+      allure-ruby-commons (= 2.16.1)
       rspec-core (>= 3.8, < 4)
-    allure-ruby-commons (2.15.0)
+    allure-ruby-commons (2.16.1)
       mime-types (>= 3.3, < 4)
       oj (>= 3.10, < 4)
       require_all (>= 2, < 4)
@@ -203,7 +203,7 @@ GEM
     octokit (4.21.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
-    oj (3.13.8)
+    oj (3.13.11)
     os (1.1.4)
     parallel (1.19.2)
     parallel_tests (2.29.0)
@@ -324,7 +324,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport (~> 6.1.4.6)
   airborne (~> 0.3.4)
-  allure-rspec (~> 2.15.0)
+  allure-rspec (~> 2.16.0)
   capybara (~> 3.35.0)
   capybara-screenshot (~> 1.0.23)
   chemlab (~> 0.9)

qa/qa/runtime/allure_report.rb

@@ -74,8 +74,8 @@ module QA
         # @return [void]
         def configure_rspec
           RSpec.configure do |config|
-            config.add_formatter(AllureRspecFormatter)
             config.add_formatter(QA::Support::Formatters::AllureMetadataFormatter)
+            config.add_formatter(AllureRspecFormatter)
 
             config.append_after do |example|
               Allure.add_attachment(

qa/qa/support/formatters/allure_metadata_formatter.rb

@@ -4,20 +4,41 @@ module QA
   module Support
     module Formatters
       class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
+        include Support::InfluxdbTools
+
         ::RSpec::Core::Formatters.register(
           self,
-          :example_started
+          :start,
+          :example_finished
         )
 
-        # Starts example
+        # Starts test run
+        # Fetch flakiness data in mr pipelines to help identify unrelated flaky failures
+        #
+        # @param [RSpec::Core::Notifications::StartNotification] _start_notification
+        # @return [void]
+        def start(_start_notification)
+          return unless merge_request_iid # on main runs allure native history has pass rate already
+
+          save_failures
+          log(:debug, "Fetched #{failures.length} flaky testcases!")
+        rescue StandardError => e
+          log(:error, "Failed to fetch flaky spec data for report: #{e}")
+          @failures = []
+        end
+
+        # Finished example
+        # Add additional metadata to report
+        #
         # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
         # @return [void]
-        def example_started(example_notification)
+        def example_finished(example_notification)
           example = example_notification.example
 
           add_quarantine_issue_link(example)
           add_failure_issues_link(example)
           add_ci_job_link(example)
+          set_flaky_status(example)
         end
 
         private
@@ -55,6 +76,66 @@ module QA
 
           example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url)
         end
+
+        # Mark test as flaky
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [void]
+        def set_flaky_status(example)
+          return unless merge_request_iid
+          return unless example.execution_result.status == :failed && failures.key?(example.metadata[:testcase])
+
+          example.set_flaky
+          example.parameter("pass_rate", "#{failures[example.metadata[:testcase]].round(1)}%")
+          log(:debug, "Setting spec as flaky due to present failures in last 14 days!")
+        end
+
+        # Failed spec testcases
+        #
+        # @return [Array]
+        def failures
+          @failures ||= influx_data.lazy.each_with_object({}) do |data, result|
+            # TODO: replace with mr_iid once stats are populated
+            records = data.records.reject { |r| r.values["_value"] == env("CI_PIPELINE_ID") }
+
+            runs = records.count
+            failed = records.count { |r| r.values["status"] == "failed" }
+            pass_rate = 100 - ((failed.to_f / runs.to_f) * 100)
+
+            # Consider spec with a pass rate less than 98% as flaky
+            result[records.last.values["testcase"]] = pass_rate if pass_rate < 98
+          end.compact
+        end
+
+        alias_method :save_failures, :failures
+
+        # Records of previous failures for runs of same type
+        #
+        # @return [Array]
+        def influx_data
+          return [] unless run_type
+
+          query_api.query(query: <<~QUERY).values
+            from(bucket: "#{Support::InfluxdbTools::INFLUX_TEST_METRICS_BUCKET}")
+              |> range(start: -14d)
+              |> filter(fn: (r) => r._measurement == "test-stats")
+              |> filter(fn: (r) => r.run_type == "#{run_type}" and
+                r.status != "pending" and
+                r.quarantined == "false" and
+                r._field == "pipeline_id"
+              )
+              |> group(columns: ["testcase"])
+          QUERY
+        end
+
+        # Print log message
+        #
+        # @param [Symbol] level
+        # @param [String] message
+        # @return [void]
+        def log(level, message)
+          QA::Runtime::Logger.public_send(level, "[Allure]: #{message}")
+        end
       end
     end
   end

qa/qa/support/formatters/test_stats_formatter.rb

@@ -4,6 +4,8 @@ module QA
   module Support
     module Formatters
       class TestStatsFormatter < RSpec::Core::Formatters::BaseFormatter
+        include Support::InfluxdbTools
+
         RSpec::Core::Formatters.register(self, :stop)
 
         # Finish test execution
@@ -11,9 +13,6 @@ module QA
         # @param [RSpec::Core::Notifications::ExamplesNotification] notification
         # @return [void]
         def stop(notification)
-          return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url
-          return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token
-
           push_test_stats(notification.examples)
           push_fabrication_stats
         end
@@ -27,7 +26,7 @@ module QA
         def push_test_stats(examples)
           data = examples.map { |example| test_stats(example) }.compact
 
-          influx_client.write(data: data)
+          write_api.write(data: data)
           log(:debug, "Pushed #{data.length} test execution entries to influxdb")
         rescue StandardError => e
           log(:error, "Failed to push test execution stats to influxdb, error: #{e}")
@@ -42,7 +41,7 @@ module QA
           end
           return if data.empty?
 
-          influx_client.write(data: data)
+          write_api.write(data: data)
           log(:debug, "Pushed #{data.length} resource fabrication entries to influxdb")
         rescue StandardError => e
           log(:error, "Failed to push fabrication stats to influxdb, error: #{e}")
@@ -70,7 +69,7 @@ module QA
               retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
               job_name: job_name,
               merge_request: merge_request,
-              run_type: env('QA_RUN_TYPE') || run_type,
+              run_type: run_type,
               stage: devops_stage(file_path),
               testcase: example.metadata[:testcase]
             },
@@ -83,7 +82,8 @@ module QA
               retry_attempts: example.metadata[:retry_attempts] || 0,
               job_url: QA::Runtime::Env.ci_job_url,
               pipeline_url: env('CI_PIPELINE_URL'),
-              pipeline_id: env('CI_PIPELINE_ID')
+              pipeline_id: env('CI_PIPELINE_ID'),
+              merge_request_iid: merge_request_iid
             }
           }
         rescue StandardError => e
@@ -119,13 +119,6 @@ module QA
           }
         end
 
-        # Project name
-        #
-        # @return [String]
-        def project_name
-          @project_name ||= QA::Runtime::Env.ci_project_name
-        end
-
         # Base ci job name
         #
         # @return [String]
@@ -148,26 +141,7 @@ module QA
         #
         # @return [String]
         def merge_request
-          @merge_request ||= (!!env('CI_MERGE_REQUEST_IID') || !!env('TOP_UPSTREAM_MERGE_REQUEST_IID')).to_s
-        end
-
-        # Test run type from staging (`gstg`, `gstg-cny`, `gstg-ref`), canary, preprod or production env
-        #
-        # @return [String, nil]
-        def run_type
-          return unless %w[staging staging-canary staging-ref canary preprod production].include?(project_name)
-
-          @run_type ||= begin
-            test_subset = if env('NO_ADMIN') == 'true'
-                            'sanity-no-admin'
-                          elsif env('SMOKE_ONLY') == 'true'
-                            'sanity'
-                          else
-                            'full'
-                          end
-
-            "#{project_name}-#{test_subset}"
-          end
+          (!!merge_request_iid).to_s
         end
 
         # Print log message
@@ -179,16 +153,6 @@ module QA
           QA::Runtime::Logger.public_send(level, "[influxdb exporter]: #{message}")
         end
 
-        # Return non empty environment variable value
-        #
-        # @param [String] name
-        # @return [String, nil]
-        def env(name)
-          return unless ENV[name] && !ENV[name].empty?
-
-          ENV[name]
-        end
-
         # Get spec devops stage
         #
         # @param [String] location
@@ -196,33 +160,6 @@ module QA
         def devops_stage(file_path)
           file_path.match(%r{\d{1,2}_(\w+)/})&.captures&.first
         end
-
-        # InfluxDb client
-        #
-        # @return [InfluxDB2::WriteApi]
-        def influx_client
-          @influx_client ||= InfluxDB2::Client.new(
-            influxdb_url,
-            influxdb_token,
-            bucket: 'e2e-test-stats',
-            org: 'gitlab-qa',
-            precision: InfluxDB2::WritePrecision::NANOSECOND
-          ).create_write_api
-        end
-
-        # InfluxDb instance url
-        #
-        # @return [String]
-        def influxdb_url
-          @influxdb_url ||= env('QA_INFLUXDB_URL')
-        end
-
-        # Influxdb token
-        #
-        # @return [String]
-        def influxdb_token
-          @influxdb_token ||= env('QA_INFLUXDB_TOKEN')
-        end
       end
     end
   end

qa/qa/support/influxdb_tools.rb

+# frozen_string_literal: true
+
+module QA
+  module Support
+    # Common tools for use with influxdb metrics setup
+    #
+    module InfluxdbTools
+      INFLUX_TEST_METRICS_BUCKET = "e2e-test-stats"
+      LIVE_ENVS = %w[staging staging-canary staging-ref canary preprod production].freeze
+
+      private
+
+      delegate :ci_project_name, to: "QA::Runtime::Env"
+
+      # Query client
+      #
+      # @return [QueryApi]
+      def query_api
+        @query_api ||= influx_client.create_query_api
+      end
+
+      # Write client
+      #
+      # @return [WriteApi]
+      def write_api
+        @write_api ||= influx_client.create_write_api
+      end
+
+      # InfluxDb client
+      #
+      # @return [InfluxDB2::Client]
+      def influx_client
+        @influx_client ||= InfluxDB2::Client.new(
+          ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable"),
+          ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN env variable"),
+          bucket: INFLUX_TEST_METRICS_BUCKET,
+          org: "gitlab-qa",
+          precision: InfluxDB2::WritePrecision::NANOSECOND
+        )
+      end
+
+      # Test run type
+      # Automatically infer for staging (`gstg`, `gstg-cny`, `gstg-ref`), canary, preprod or production env
+      #
+      # @return [String, nil]
+      def run_type
+        @run_type ||= begin
+          return env('QA_RUN_TYPE') if env('QA_RUN_TYPE')
+          return unless LIVE_ENVS.include?(ci_project_name)
+
+          test_subset = if env('NO_ADMIN') == 'true'
+                          'sanity-no-admin'
+                        elsif env('SMOKE_ONLY') == 'true'
+                          'sanity'
+                        else
+                          'full'
+                        end
+
+          "#{ci_project_name}-#{test_subset}"
+        end
+      end
+
+      # Merge request iid
+      #
+      # @return [String]
+      def merge_request_iid
+        env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+      end
+
+      # Return non empty environment variable value
+      #
+      # @param [String] name
+      # @return [String, nil]
+      def env(name)
+        return unless ENV[name] && !ENV[name].empty?
+
+        ENV[name]
+      end
+    end
+  end
+end

qa/qa/tools/reliable_report.rb

@@ -8,6 +8,7 @@ require "colorize"
 module QA
   module Tools
     class ReliableReport
+      include Support::InfluxdbTools
       include Support::API
 
       # Project for report creation: https://gitlab.com/gitlab-org/gitlab
@@ -15,10 +16,7 @@ module QA
 
       def initialize(range)
         @range = range.to_i
-        @influxdb_bucket = "e2e-test-stats"
         @slack_channel = "#quality-reports"
-        @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable")
-        @influxdb_token = ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN env variable")
       end
 
       # Run reliable reporter
@@ -91,7 +89,7 @@ module QA
 
       private
 
-      attr_reader :range, :influxdb_bucket, :slack_channel, :influxdb_url, :influxdb_token
+      attr_reader :range, :slack_channel
 
       # Markdown formatted report issue body
       #
@@ -304,7 +302,7 @@ module QA
       # @return [String]
       def query(reliable)
         <<~QUERY
-          from(bucket: "#{influxdb_bucket}")
+          from(bucket: "#{Support::InfluxdbTools::INFLUX_TEST_METRICS_BUCKET}")
             |> range(start: -#{range}d)
             |> filter(fn: (r) => r._measurement == "test-stats")
             |> filter(fn: (r) => r.run_type == "staging-full" or
@@ -325,26 +323,6 @@ module QA
         QUERY
       end
 
-      # Query client
-      #
-      # @return [QueryApi]
-      def query_api
-        @query_api ||= influx_client.create_query_api
-      end
-
-      # InfluxDb client
-      #
-      # @return [InfluxDB2::Client]
-      def influx_client
-        @influx_client ||= InfluxDB2::Client.new(
-          influxdb_url,
-          influxdb_token,
-          bucket: influxdb_bucket,
-          org: "gitlab-qa",
-          precision: InfluxDB2::WritePrecision::NANOSECOND
-        )
-      end
-
       # Slack notifier
       #
       # @return [Slack::Notifier]

qa/spec/specs/allure_report_spec.rb

@@ -76,9 +76,10 @@ describe QA::Runtime::AllureReport do
     end
 
     it 'adds rspec and metadata formatter' do
+      expect(rspec_config).to have_received(:add_formatter).with(
+        QA::Support::Formatters::AllureMetadataFormatter
+      ).ordered
       expect(rspec_config).to have_received(:add_formatter).with(AllureRspecFormatter).ordered
-      expect(rspec_config).to have_received(:add_formatter)
-        .with(QA::Support::Formatters::AllureMetadataFormatter).ordered
     end
 
     it 'configures attachments saving' do

qa/spec/support/formatters/allure_metadata_formatter_spec.rb

@@ -14,6 +14,7 @@ describe QA::Support::Formatters::AllureMetadataFormatter do
       add_link: nil,
       attempts: 0,
       file_path: 'file/path/spec.rb',
+      execution_result: instance_double("RSpec::Core::Example::ExecutionResult", status: :passed),
       metadata: {
         testcase: 'testcase',
         quarantine: { issue: 'issue' }
@@ -31,7 +32,7 @@ describe QA::Support::Formatters::AllureMetadataFormatter do
   end
 
   it "adds additional data to report" do
-    formatter.example_started(rspec_example_notification)
+    formatter.example_finished(rspec_example_notification)
 
     aggregate_failures do
       expect(rspec_example).to have_received(:issue).with('Quarantine issue', 'issue')

qa/spec/support/formatters/test_stats_formatter_spec.rb

@@ -60,7 +60,8 @@ describe QA::Support::Formatters::TestStatsFormatter do
         retry_attempts: 0,
         job_url: ci_job_url,
         pipeline_url: ci_pipeline_url,
-        pipeline_id: ci_pipeline_id
+        pipeline_id: ci_pipeline_id,
+        merge_request_iid: nil
       }
     }
   end