Fix flaky allure report spec

Fixup rubocop

.gitlab/ci/qa.gitlab-ci.yml

@@ -18,7 +18,7 @@ qa:internal:
     - .qa-job-base
     - .qa:rules:internal
   script:
-    - bundle exec rspec
+    - bundle exec rspec -O .rspec_internal
 
 qa:internal-as-if-foss:
   extends:

qa/.gitignore

 tmp/
+reports/
+no_of_examples/
+
 .ruby-version
 .tool-versions
 .ruby-gemset
 urls.yml
-reports/

qa/.rspec_internal

+--force-color
+--order random
+--format documentation
+--require specs/spec_helper

qa/qa/page/base.rb

@@ -133,7 +133,9 @@ module QA
       end
 
       def all_elements(name, **kwargs)
-        if kwargs.keys.none? { |key| [:minimum, :maximum, :count, :between].include?(key) }
+        all_args = [:minimum, :maximum, :count, :between]
+
+        if kwargs.keys.none? { |key| all_args.include?(key) }
           raise ArgumentError, "Please use :minimum, :maximum, :count, or :between so that all is more reliable"
         end
 

qa/spec/page/base_spec.rb

 # frozen_string_literal: true
 
+# rubocop:disable QA/ElementWithPattern
 RSpec.describe QA::Page::Base do
   describe 'page helpers' do
     it 'exposes helpful page helpers' do
@@ -11,12 +12,12 @@ RSpec.describe QA::Page::Base do
     subject do
       Class.new(described_class) do
         view 'path/to/some/view.html.haml' do
-          element :something, 'string pattern' # rubocop:disable QA/ElementWithPattern
-          element :something_else, /regexp pattern/ # rubocop:disable QA/ElementWithPattern
+          element :something, 'string pattern'
+          element :something_else, /regexp pattern/
         end
 
         view 'path/to/some/_partial.html.haml' do
-          element :another_element, 'string pattern' # rubocop:disable QA/ElementWithPattern
+          element :another_element, 'string pattern'
         end
       end
     end
@@ -95,6 +96,7 @@ RSpec.describe QA::Page::Base do
   describe '#all_elements' do
     before do
       allow(subject).to receive(:all)
+      allow(subject).to receive(:wait_for_requests)
     end
 
     it 'raises an error if count or minimum are not specified' do
@@ -108,7 +110,7 @@ RSpec.describe QA::Page::Base do
     end
   end
 
-  context 'elements' do
+  describe 'elements' do
     subject do
       Class.new(described_class) do
         view 'path/to/some/view.html.haml' do
@@ -133,35 +135,37 @@ RSpec.describe QA::Page::Base do
     describe '#visible?', 'Page is currently visible' do
       let(:page) { subject.new }
 
+      before do
+        allow(page).to receive(:wait_for_requests)
+      end
+
       context 'with elements' do
-        context 'on the page' do
         before do
-            # required elements not there, meaning not on page
-            allow(page).to receive(:has_no_element?).and_return(false)
+          allow(page).to receive(:has_no_element?).and_return(has_no_element)
         end
 
+        context 'with element on the page' do
+          let(:has_no_element) { false }
+
           it 'is visible' do
             expect(page).to be_visible
           end
-        end
 
-        context 'not on the page' do
-          before do
-            # required elements are not on the page
-            allow(page).to receive(:has_no_element?).and_return(true)
+          it 'does not raise error if page has elements' do
+            expect { page.visible? }.not_to raise_error
+          end
         end
 
+        context 'with element not on the page' do
+          let(:has_no_element) { true }
+
           it 'is not visible' do
             expect(page).not_to be_visible
           end
         end
-
-        it 'does not raise error if page has elements' do
-          expect { page.visible? }.not_to raise_error
-        end
       end
 
-      context 'no elements' do
+      context 'with no elements' do
         subject do
           Class.new(described_class) do
             view 'path/to/some/view.html.haml' do
@@ -180,3 +184,4 @@ RSpec.describe QA::Page::Base do
     end
   end
 end
+# rubocop:enable QA/ElementWithPattern

qa/spec/page/logging_spec.rb

@@ -72,41 +72,47 @@ RSpec.describe QA::Support::Page::Logging do
   end
 
   it 'logs has_element?' do
-    expect { subject.has_element?(:element) }
-      .to output(/has_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process
+    expect { subject.has_element?(:element) }.to output(
+      /has_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs has_element? with text' do
-    expect { subject.has_element?(:element, text: "some text") }
-      .to output(/has_element\? :element with text "some text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process
+    expect { subject.has_element?(:element, text: "some text") }.to output(
+      /has_element\? :element with text "some text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs has_no_element?' do
     allow(page).to receive(:has_no_css?).and_return(true)
 
-    expect { subject.has_no_element?(:element) }
-      .to output(/has_no_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process
+    expect { subject.has_no_element?(:element) }.to output(
+      /has_no_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs has_no_element? with text' do
     allow(page).to receive(:has_no_css?).and_return(true)
 
-    expect { subject.has_no_element?(:element, text: "more text") }
-      .to output(/has_no_element\? :element with text "more text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process
+    expect { subject.has_no_element?(:element, text: "more text") }.to output(
+      /has_no_element\? :element with text "more text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs has_text?' do
     allow(page).to receive(:has_text?).and_return(true)
 
-    expect { subject.has_text? 'foo' }
-      .to output(/has_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process
+    expect { subject.has_text? 'foo' }.to output(
+      /has_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs has_no_text?' do
     allow(page).to receive(:has_no_text?).with('foo', any_args).and_return(true)
 
-    expect { subject.has_no_text? 'foo' }
-      .to output(/has_no_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process
+    expect { subject.has_no_text? 'foo' }.to output(
+      /has_no_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o
+    ).to_stdout_from_any_process
   end
 
   it 'logs finished_loading?' do
@@ -123,7 +129,7 @@ RSpec.describe QA::Support::Page::Logging do
       .to output(/end within element :element/).to_stdout_from_any_process
   end
 
-  context 'all_elements' do
+  context 'with all_elements' do
     it 'logs the number of elements found' do
       allow(page).to receive(:all).and_return([1, 2])
 

qa/spec/resource/base_spec.rb

@@ -4,6 +4,7 @@ RSpec.describe QA::Resource::Base do
   include QA::Support::Helpers::StubEnv
 
   let(:resource) { spy('resource') }
+  let(:api_client) { instance_double('Runtime::API::Client') }
   let(:location) { 'http://location' }
   let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} }
 
@@ -114,6 +115,7 @@ RSpec.describe QA::Resource::Base do
         allow(QA::Runtime::Logger).to receive(:debug)
         allow(resource).to receive(:api_support?).and_return(true)
         allow(resource).to receive(:fabricate_via_api!)
+        allow(resource).to receive(:api_client) { api_client }
       end
 
       it 'logs the resource and build method' do
@@ -154,7 +156,6 @@ RSpec.describe QA::Resource::Base do
 
       before do
         allow(QA::Runtime::Logger).to receive(:debug)
-        # allow(resource).to receive(:fabricate!)
       end
 
       it 'logs the resource and build method' do

qa/spec/runtime/script_extensions/interceptor_spec.rb

@@ -8,6 +8,7 @@ RSpec.describe 'Interceptor' do
   before(:context) do
     skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept?
 
+    QA::Runtime::Browser.configure!
     QA::Runtime::Browser::Session.enable_interception
   end
 
@@ -26,7 +27,7 @@ RSpec.describe 'Interceptor' do
   end
 
   context 'with Interceptor' do
-    context 'caching' do
+    context 'with caching' do
       it 'checks the cache' do
         expect(check_cache).to be(true)
       end

qa/spec/specs/allure_report_spec.rb

@@ -3,7 +3,7 @@
 describe QA::Runtime::AllureReport do
   include QA::Support::Helpers::StubEnv
 
-  let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) }
+  let(:rspec_config) { instance_double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) }
 
   let(:png_path) { 'png_path' }
   let(:html_path) { 'html_path' }
@@ -42,11 +42,14 @@ describe QA::Runtime::AllureReport do
   context 'with report generation enabled' do
     let(:generate_report) { 'true' }
 
+    let(:session) { instance_double('Capybara::Session') }
+    let(:attributes) { class_spy('Runtime::Scenario') }
+    let(:version_response) { instance_double('HTTPResponse', code: 200, body: versions.to_json) }
+
     let(:png_file) { 'png-file' }
     let(:html_file) { 'html-file' }
     let(:ci_job) { 'ee:relative 5' }
     let(:versions) { { version: '14', revision: '6ced31db947' } }
-    let(:session) { double('session') }
     let(:browser_log) { ['log message 1', 'log message 2'] }
 
     before do
@@ -54,11 +57,13 @@ describe QA::Runtime::AllureReport do
       stub_env('CI_JOB_NAME', ci_job)
       stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token')
 
+      stub_const('QA::Runtime::Scenario', attributes)
+
       allow(Allure).to receive(:add_attachment)
       allow(File).to receive(:open).with(png_path) { png_file }
       allow(File).to receive(:open).with(html_path) { html_file }
-      allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) }
-      allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com')
+      allow(RestClient::Request).to receive(:execute) { version_response }
+      allow(attributes).to receive(:gitlab_address).and_return("https://gitlab.com")
 
       allow(Capybara).to receive(:current_session).and_return(session)
       allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log)
@@ -66,7 +71,7 @@ describe QA::Runtime::AllureReport do
       described_class.configure!
     end
 
-    it 'configures Allure options', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/357816' do
+    it 'configures Allure options' do
       aggregate_failures do
         expect(allure_config.results_directory).to eq('tmp/allure-results')
         expect(allure_config.clean_results_directory).to eq(true)

qa/spec/support/shared_examples/scenario_shared_examples.rb → qa/spec/specs/scenario_shared_examples.rb

@@ -2,10 +2,10 @@
 
 module QA
   RSpec.shared_examples 'a QA scenario class' do
-    let(:attributes) { spy('Runtime::Scenario') }
-    let(:runner) { spy('Specs::Runner') }
-    let(:release) { spy('Runtime::Release') }
-    let(:feature) { spy('Runtime::Feature') }
+    let(:attributes) { class_spy('Runtime::Scenario') }
+    let(:runner) { class_spy('Specs::Runner') }
+    let(:release) { class_spy('Runtime::Release') }
+    let(:feature) { class_spy('Runtime::Feature') }
 
     let(:args) { { gitlab_address: 'http://gitlab_address' } }
     let(:named_options) { %w[--address http://gitlab_address] }
@@ -45,7 +45,7 @@ module QA
       expect(runner).to have_received(:tags=).with(tags)
     end
 
-    context 'specifying RSpec options' do
+    context 'with RSpec options' do
       it 'sets options on runner' do
         subject.perform(args, *options)
 

qa/spec/specs/spec_helper.rb

+# frozen_string_literal: true
+
+require_relative '../../qa'
+
+require_relative 'scenario_shared_examples'

qa/spec/support/formatters/test_stats_formatter_spec.rb

@@ -25,6 +25,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
   let(:ui_fabrication) { 0 }
   let(:api_fabrication) { 0 }
   let(:fabrication_resources) { {} }
+  let(:testcase) { 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' }
 
   let(:influx_client_args) do
     {
@@ -49,7 +50,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
         merge_request: "false",
         run_type: run_type,
         stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first,
-        testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234'
+        testcase: testcase
       },
       fields: {
         id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
@@ -78,12 +79,6 @@ describe QA::Support::Formatters::TestStatsFormatter do
   around do |example|
     RSpec::Core::Sandbox.sandboxed do |config|
       config.formatter = QA::Support::Formatters::TestStatsFormatter
-
-      config.append_after do |example|
-        example.metadata[:api_fabrication] = Thread.current[:api_fabrication]
-        example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication]
-      end
-
       config.before(:context) { RSpec.current_example = nil }
 
       example.run
@@ -211,16 +206,18 @@ describe QA::Support::Formatters::TestStatsFormatter do
     end
 
     context 'with fabrication runtimes' do
-      let(:ui_fabrication) { 10 }
       let(:api_fabrication) { 4 }
-
-      before do
-        Thread.current[:api_fabrication] = api_fabrication
-        Thread.current[:browser_ui_fabrication] = ui_fabrication
-      end
+      let(:ui_fabrication) { 10 }
+      let(:testcase) { nil }
 
       it 'exports data to influxdb with fabrication times' do
-        run_spec
+        run_spec do
+          # Main logic tracks fabrication time in thread local variable and injects it as metadata from
+          # global after hook defined in main spec_helper.
+          #
+          # Inject the values directly since we do not load e2e test spec_helper in unit tests
+          it('spec', api_fabrication: 4, browser_ui_fabrication: 10) {}
+        end
 
         expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])

qa/spec/support/wait_for_requests_spec.rb

@@ -5,37 +5,38 @@ RSpec.describe QA::Support::WaitForRequests do
     before do
       allow(subject).to receive(:finished_all_ajax_requests?).and_return(true)
       allow(subject).to receive(:finished_loading?).and_return(true)
+      allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code)
     end
 
     context 'when skip_finished_loading_check is defaulted to false' do
       it 'calls finished_loading?' do
-        expect(subject).to receive(:finished_loading?).with(hash_including(wait: 1))
-
         subject.wait_for_requests
+
+        expect(subject).to have_received(:finished_loading?).with(hash_including(wait: 1))
       end
     end
 
     context 'when skip_finished_loading_check is true' do
       it 'does not call finished_loading?' do
-        expect(subject).not_to receive(:finished_loading?)
-
         subject.wait_for_requests(skip_finished_loading_check: true)
+
+        expect(subject).not_to have_received(:finished_loading?)
       end
     end
 
     context 'when skip_resp_code_check is defaulted to false' do
       it 'call report' do
-        allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page)
-
         subject.wait_for_requests
+
+        expect(QA::Support::PageErrorChecker).to have_received(:check_page_for_error_code).with(Capybara.page)
       end
     end
 
     context 'when skip_resp_code_check is true' do
       it 'does not parse for an error code' do
-        expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code)
-
         subject.wait_for_requests(skip_resp_code_check: true)
+
+        expect(QA::Support::PageErrorChecker).not_to have_received(:check_page_for_error_code)
       end
     end
   end