Commit 01a1a602 authored by Andrejs Cunskis's avatar Andrejs Cunskis Committed by Mark Lapierre

Fix flaky allure report spec

Fixup rubocop
parent bd70ef16
...@@ -18,7 +18,7 @@ qa:internal: ...@@ -18,7 +18,7 @@ qa:internal:
- .qa-job-base - .qa-job-base
- .qa:rules:internal - .qa:rules:internal
script: script:
- bundle exec rspec - bundle exec rspec -O .rspec_internal
qa:internal-as-if-foss: qa:internal-as-if-foss:
extends: extends:
......
tmp/ tmp/
reports/
no_of_examples/
.ruby-version .ruby-version
.tool-versions .tool-versions
.ruby-gemset .ruby-gemset
urls.yml urls.yml
reports/
--force-color
--order random
--format documentation
--require specs/spec_helper
...@@ -133,7 +133,9 @@ module QA ...@@ -133,7 +133,9 @@ module QA
end end
def all_elements(name, **kwargs) def all_elements(name, **kwargs)
if kwargs.keys.none? { |key| [:minimum, :maximum, :count, :between].include?(key) } all_args = [:minimum, :maximum, :count, :between]
if kwargs.keys.none? { |key| all_args.include?(key) }
raise ArgumentError, "Please use :minimum, :maximum, :count, or :between so that all is more reliable" raise ArgumentError, "Please use :minimum, :maximum, :count, or :between so that all is more reliable"
end end
...@@ -469,8 +471,8 @@ module QA ...@@ -469,8 +471,8 @@ module QA
return element_when_flag_disabled if has_element?(element_when_flag_disabled) return element_when_flag_disabled if has_element?(element_when_flag_disabled)
raise ElementNotFound, raise ElementNotFound,
"Could not find the expected element as #{element_when_flag_enabled} or #{element_when_flag_disabled}." \ "Could not find the expected element as #{element_when_flag_enabled} or #{element_when_flag_disabled}." \
"The relevant feature flag is #{feature_flag}" "The relevant feature flag is #{feature_flag}"
end end
end end
end end
......
# frozen_string_literal: true # frozen_string_literal: true
# rubocop:disable QA/ElementWithPattern
RSpec.describe QA::Page::Base do RSpec.describe QA::Page::Base do
describe 'page helpers' do describe 'page helpers' do
it 'exposes helpful page helpers' do it 'exposes helpful page helpers' do
...@@ -11,12 +12,12 @@ RSpec.describe QA::Page::Base do ...@@ -11,12 +12,12 @@ RSpec.describe QA::Page::Base do
subject do subject do
Class.new(described_class) do Class.new(described_class) do
view 'path/to/some/view.html.haml' do view 'path/to/some/view.html.haml' do
element :something, 'string pattern' # rubocop:disable QA/ElementWithPattern element :something, 'string pattern'
element :something_else, /regexp pattern/ # rubocop:disable QA/ElementWithPattern element :something_else, /regexp pattern/
end end
view 'path/to/some/_partial.html.haml' do view 'path/to/some/_partial.html.haml' do
element :another_element, 'string pattern' # rubocop:disable QA/ElementWithPattern element :another_element, 'string pattern'
end end
end end
end end
...@@ -95,6 +96,7 @@ RSpec.describe QA::Page::Base do ...@@ -95,6 +96,7 @@ RSpec.describe QA::Page::Base do
describe '#all_elements' do describe '#all_elements' do
before do before do
allow(subject).to receive(:all) allow(subject).to receive(:all)
allow(subject).to receive(:wait_for_requests)
end end
it 'raises an error if count or minimum are not specified' do it 'raises an error if count or minimum are not specified' do
...@@ -108,7 +110,7 @@ RSpec.describe QA::Page::Base do ...@@ -108,7 +110,7 @@ RSpec.describe QA::Page::Base do
end end
end end
context 'elements' do describe 'elements' do
subject do subject do
Class.new(described_class) do Class.new(described_class) do
view 'path/to/some/view.html.haml' do view 'path/to/some/view.html.haml' do
...@@ -133,35 +135,37 @@ RSpec.describe QA::Page::Base do ...@@ -133,35 +135,37 @@ RSpec.describe QA::Page::Base do
describe '#visible?', 'Page is currently visible' do describe '#visible?', 'Page is currently visible' do
let(:page) { subject.new } let(:page) { subject.new }
before do
allow(page).to receive(:wait_for_requests)
end
context 'with elements' do context 'with elements' do
context 'on the page' do before do
before do allow(page).to receive(:has_no_element?).and_return(has_no_element)
# required elements not there, meaning not on page end
allow(page).to receive(:has_no_element?).and_return(false)
end context 'with element on the page' do
let(:has_no_element) { false }
it 'is visible' do it 'is visible' do
expect(page).to be_visible expect(page).to be_visible
end end
end
context 'not on the page' do it 'does not raise error if page has elements' do
before do expect { page.visible? }.not_to raise_error
# required elements are not on the page
allow(page).to receive(:has_no_element?).and_return(true)
end end
end
context 'with element not on the page' do
let(:has_no_element) { true }
it 'is not visible' do it 'is not visible' do
expect(page).not_to be_visible expect(page).not_to be_visible
end end
end end
it 'does not raise error if page has elements' do
expect { page.visible? }.not_to raise_error
end
end end
context 'no elements' do context 'with no elements' do
subject do subject do
Class.new(described_class) do Class.new(described_class) do
view 'path/to/some/view.html.haml' do view 'path/to/some/view.html.haml' do
...@@ -180,3 +184,4 @@ RSpec.describe QA::Page::Base do ...@@ -180,3 +184,4 @@ RSpec.describe QA::Page::Base do
end end
end end
end end
# rubocop:enable QA/ElementWithPattern
...@@ -72,41 +72,47 @@ RSpec.describe QA::Support::Page::Logging do ...@@ -72,41 +72,47 @@ RSpec.describe QA::Support::Page::Logging do
end end
it 'logs has_element?' do it 'logs has_element?' do
expect { subject.has_element?(:element) } expect { subject.has_element?(:element) }.to output(
.to output(/has_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process /has_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
).to_stdout_from_any_process
end end
it 'logs has_element? with text' do it 'logs has_element? with text' do
expect { subject.has_element?(:element, text: "some text") } expect { subject.has_element?(:element, text: "some text") }.to output(
.to output(/has_element\? :element with text "some text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process /has_element\? :element with text "some text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
).to_stdout_from_any_process
end end
it 'logs has_no_element?' do it 'logs has_no_element?' do
allow(page).to receive(:has_no_css?).and_return(true) allow(page).to receive(:has_no_css?).and_return(true)
expect { subject.has_no_element?(:element) } expect { subject.has_no_element?(:element) }.to output(
.to output(/has_no_element\? :element \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process /has_no_element\? :element \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
).to_stdout_from_any_process
end end
it 'logs has_no_element? with text' do it 'logs has_no_element? with text' do
allow(page).to receive(:has_no_css?).and_return(true) allow(page).to receive(:has_no_css?).and_return(true)
expect { subject.has_no_element?(:element, text: "more text") } expect { subject.has_no_element?(:element, text: "more text") }.to output(
.to output(/has_no_element\? :element with text "more text" \(wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned: true/o).to_stdout_from_any_process /has_no_element\? :element with text "more text" \(wait: #{Capybara.default_max_wait_time}\) returned: true/o
).to_stdout_from_any_process
end end
it 'logs has_text?' do it 'logs has_text?' do
allow(page).to receive(:has_text?).and_return(true) allow(page).to receive(:has_text?).and_return(true)
expect { subject.has_text? 'foo' } expect { subject.has_text? 'foo' }.to output(
.to output(/has_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process /has_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o
).to_stdout_from_any_process
end end
it 'logs has_no_text?' do it 'logs has_no_text?' do
allow(page).to receive(:has_no_text?).with('foo', any_args).and_return(true) allow(page).to receive(:has_no_text?).with('foo', any_args).and_return(true)
expect { subject.has_no_text? 'foo' } expect { subject.has_no_text? 'foo' }.to output(
.to output(/has_no_text\?\('foo', wait: #{QA::Runtime::Browser::CAPYBARA_MAX_WAIT_TIME}\) returned true/o).to_stdout_from_any_process /has_no_text\?\('foo', wait: #{Capybara.default_max_wait_time}\) returned true/o
).to_stdout_from_any_process
end end
it 'logs finished_loading?' do it 'logs finished_loading?' do
...@@ -123,7 +129,7 @@ RSpec.describe QA::Support::Page::Logging do ...@@ -123,7 +129,7 @@ RSpec.describe QA::Support::Page::Logging do
.to output(/end within element :element/).to_stdout_from_any_process .to output(/end within element :element/).to_stdout_from_any_process
end end
context 'all_elements' do context 'with all_elements' do
it 'logs the number of elements found' do it 'logs the number of elements found' do
allow(page).to receive(:all).and_return([1, 2]) allow(page).to receive(:all).and_return([1, 2])
......
...@@ -4,6 +4,7 @@ RSpec.describe QA::Resource::Base do ...@@ -4,6 +4,7 @@ RSpec.describe QA::Resource::Base do
include QA::Support::Helpers::StubEnv include QA::Support::Helpers::StubEnv
let(:resource) { spy('resource') } let(:resource) { spy('resource') }
let(:api_client) { instance_double('Runtime::API::Client') }
let(:location) { 'http://location' } let(:location) { 'http://location' }
let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} } let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} }
...@@ -114,6 +115,7 @@ RSpec.describe QA::Resource::Base do ...@@ -114,6 +115,7 @@ RSpec.describe QA::Resource::Base do
allow(QA::Runtime::Logger).to receive(:debug) allow(QA::Runtime::Logger).to receive(:debug)
allow(resource).to receive(:api_support?).and_return(true) allow(resource).to receive(:api_support?).and_return(true)
allow(resource).to receive(:fabricate_via_api!) allow(resource).to receive(:fabricate_via_api!)
allow(resource).to receive(:api_client) { api_client }
end end
it 'logs the resource and build method' do it 'logs the resource and build method' do
...@@ -154,7 +156,6 @@ RSpec.describe QA::Resource::Base do ...@@ -154,7 +156,6 @@ RSpec.describe QA::Resource::Base do
before do before do
allow(QA::Runtime::Logger).to receive(:debug) allow(QA::Runtime::Logger).to receive(:debug)
# allow(resource).to receive(:fabricate!)
end end
it 'logs the resource and build method' do it 'logs the resource and build method' do
......
...@@ -8,6 +8,7 @@ RSpec.describe 'Interceptor' do ...@@ -8,6 +8,7 @@ RSpec.describe 'Interceptor' do
before(:context) do before(:context) do
skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept? skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept?
QA::Runtime::Browser.configure!
QA::Runtime::Browser::Session.enable_interception QA::Runtime::Browser::Session.enable_interception
end end
...@@ -26,7 +27,7 @@ RSpec.describe 'Interceptor' do ...@@ -26,7 +27,7 @@ RSpec.describe 'Interceptor' do
end end
context 'with Interceptor' do context 'with Interceptor' do
context 'caching' do context 'with caching' do
it 'checks the cache' do it 'checks the cache' do
expect(check_cache).to be(true) expect(check_cache).to be(true)
end end
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
describe QA::Runtime::AllureReport do describe QA::Runtime::AllureReport do
include QA::Support::Helpers::StubEnv include QA::Support::Helpers::StubEnv
let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } let(:rspec_config) { instance_double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) }
let(:png_path) { 'png_path' } let(:png_path) { 'png_path' }
let(:html_path) { 'html_path' } let(:html_path) { 'html_path' }
...@@ -42,11 +42,14 @@ describe QA::Runtime::AllureReport do ...@@ -42,11 +42,14 @@ describe QA::Runtime::AllureReport do
context 'with report generation enabled' do context 'with report generation enabled' do
let(:generate_report) { 'true' } let(:generate_report) { 'true' }
let(:session) { instance_double('Capybara::Session') }
let(:attributes) { class_spy('Runtime::Scenario') }
let(:version_response) { instance_double('HTTPResponse', code: 200, body: versions.to_json) }
let(:png_file) { 'png-file' } let(:png_file) { 'png-file' }
let(:html_file) { 'html-file' } let(:html_file) { 'html-file' }
let(:ci_job) { 'ee:relative 5' } let(:ci_job) { 'ee:relative 5' }
let(:versions) { { version: '14', revision: '6ced31db947' } } let(:versions) { { version: '14', revision: '6ced31db947' } }
let(:session) { double('session') }
let(:browser_log) { ['log message 1', 'log message 2'] } let(:browser_log) { ['log message 1', 'log message 2'] }
before do before do
...@@ -54,11 +57,13 @@ describe QA::Runtime::AllureReport do ...@@ -54,11 +57,13 @@ describe QA::Runtime::AllureReport do
stub_env('CI_JOB_NAME', ci_job) stub_env('CI_JOB_NAME', ci_job)
stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token') stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token')
stub_const('QA::Runtime::Scenario', attributes)
allow(Allure).to receive(:add_attachment) allow(Allure).to receive(:add_attachment)
allow(File).to receive(:open).with(png_path) { png_file } allow(File).to receive(:open).with(png_path) { png_file }
allow(File).to receive(:open).with(html_path) { html_file } allow(File).to receive(:open).with(html_path) { html_file }
allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) } allow(RestClient::Request).to receive(:execute) { version_response }
allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com') allow(attributes).to receive(:gitlab_address).and_return("https://gitlab.com")
allow(Capybara).to receive(:current_session).and_return(session) allow(Capybara).to receive(:current_session).and_return(session)
allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log) allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log)
...@@ -66,7 +71,7 @@ describe QA::Runtime::AllureReport do ...@@ -66,7 +71,7 @@ describe QA::Runtime::AllureReport do
described_class.configure! described_class.configure!
end end
it 'configures Allure options', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/357816' do it 'configures Allure options' do
aggregate_failures do aggregate_failures do
expect(allure_config.results_directory).to eq('tmp/allure-results') expect(allure_config.results_directory).to eq('tmp/allure-results')
expect(allure_config.clean_results_directory).to eq(true) expect(allure_config.clean_results_directory).to eq(true)
......
...@@ -2,10 +2,10 @@ ...@@ -2,10 +2,10 @@
module QA module QA
RSpec.shared_examples 'a QA scenario class' do RSpec.shared_examples 'a QA scenario class' do
let(:attributes) { spy('Runtime::Scenario') } let(:attributes) { class_spy('Runtime::Scenario') }
let(:runner) { spy('Specs::Runner') } let(:runner) { class_spy('Specs::Runner') }
let(:release) { spy('Runtime::Release') } let(:release) { class_spy('Runtime::Release') }
let(:feature) { spy('Runtime::Feature') } let(:feature) { class_spy('Runtime::Feature') }
let(:args) { { gitlab_address: 'http://gitlab_address' } } let(:args) { { gitlab_address: 'http://gitlab_address' } }
let(:named_options) { %w[--address http://gitlab_address] } let(:named_options) { %w[--address http://gitlab_address] }
...@@ -45,7 +45,7 @@ module QA ...@@ -45,7 +45,7 @@ module QA
expect(runner).to have_received(:tags=).with(tags) expect(runner).to have_received(:tags=).with(tags)
end end
context 'specifying RSpec options' do context 'with RSpec options' do
it 'sets options on runner' do it 'sets options on runner' do
subject.perform(args, *options) subject.perform(args, *options)
......
# frozen_string_literal: true
require_relative '../../qa'
require_relative 'scenario_shared_examples'
...@@ -25,6 +25,7 @@ describe QA::Support::Formatters::TestStatsFormatter do ...@@ -25,6 +25,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
let(:ui_fabrication) { 0 } let(:ui_fabrication) { 0 }
let(:api_fabrication) { 0 } let(:api_fabrication) { 0 }
let(:fabrication_resources) { {} } let(:fabrication_resources) { {} }
let(:testcase) { 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' }
let(:influx_client_args) do let(:influx_client_args) do
{ {
...@@ -49,7 +50,7 @@ describe QA::Support::Formatters::TestStatsFormatter do ...@@ -49,7 +50,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
merge_request: "false", merge_request: "false",
run_type: run_type, run_type: run_type,
stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first, stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first,
testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' testcase: testcase
}, },
fields: { fields: {
id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
...@@ -78,12 +79,6 @@ describe QA::Support::Formatters::TestStatsFormatter do ...@@ -78,12 +79,6 @@ describe QA::Support::Formatters::TestStatsFormatter do
around do |example| around do |example|
RSpec::Core::Sandbox.sandboxed do |config| RSpec::Core::Sandbox.sandboxed do |config|
config.formatter = QA::Support::Formatters::TestStatsFormatter config.formatter = QA::Support::Formatters::TestStatsFormatter
config.append_after do |example|
example.metadata[:api_fabrication] = Thread.current[:api_fabrication]
example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication]
end
config.before(:context) { RSpec.current_example = nil } config.before(:context) { RSpec.current_example = nil }
example.run example.run
...@@ -211,16 +206,18 @@ describe QA::Support::Formatters::TestStatsFormatter do ...@@ -211,16 +206,18 @@ describe QA::Support::Formatters::TestStatsFormatter do
end end
context 'with fabrication runtimes' do context 'with fabrication runtimes' do
let(:ui_fabrication) { 10 }
let(:api_fabrication) { 4 } let(:api_fabrication) { 4 }
let(:ui_fabrication) { 10 }
before do let(:testcase) { nil }
Thread.current[:api_fabrication] = api_fabrication
Thread.current[:browser_ui_fabrication] = ui_fabrication
end
it 'exports data to influxdb with fabrication times' do it 'exports data to influxdb with fabrication times' do
run_spec run_spec do
# Main logic tracks fabrication time in thread local variable and injects it as metadata from
# global after hook defined in main spec_helper.
#
# Inject the values directly since we do not load e2e test spec_helper in unit tests
it('spec', api_fabrication: 4, browser_ui_fabrication: 10) {}
end
expect(influx_write_api).to have_received(:write).once expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data]) expect(influx_write_api).to have_received(:write).with(data: [data])
......
...@@ -5,37 +5,38 @@ RSpec.describe QA::Support::WaitForRequests do ...@@ -5,37 +5,38 @@ RSpec.describe QA::Support::WaitForRequests do
before do before do
allow(subject).to receive(:finished_all_ajax_requests?).and_return(true) allow(subject).to receive(:finished_all_ajax_requests?).and_return(true)
allow(subject).to receive(:finished_loading?).and_return(true) allow(subject).to receive(:finished_loading?).and_return(true)
allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code)
end end
context 'when skip_finished_loading_check is defaulted to false' do context 'when skip_finished_loading_check is defaulted to false' do
it 'calls finished_loading?' do it 'calls finished_loading?' do
expect(subject).to receive(:finished_loading?).with(hash_including(wait: 1))
subject.wait_for_requests subject.wait_for_requests
expect(subject).to have_received(:finished_loading?).with(hash_including(wait: 1))
end end
end end
context 'when skip_finished_loading_check is true' do context 'when skip_finished_loading_check is true' do
it 'does not call finished_loading?' do it 'does not call finished_loading?' do
expect(subject).not_to receive(:finished_loading?)
subject.wait_for_requests(skip_finished_loading_check: true) subject.wait_for_requests(skip_finished_loading_check: true)
expect(subject).not_to have_received(:finished_loading?)
end end
end end
context 'when skip_resp_code_check is defaulted to false' do context 'when skip_resp_code_check is defaulted to false' do
it 'call report' do it 'call report' do
allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page)
subject.wait_for_requests subject.wait_for_requests
expect(QA::Support::PageErrorChecker).to have_received(:check_page_for_error_code).with(Capybara.page)
end end
end end
context 'when skip_resp_code_check is true' do context 'when skip_resp_code_check is true' do
it 'does not parse for an error code' do it 'does not parse for an error code' do
expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code)
subject.wait_for_requests(skip_resp_code_check: true) subject.wait_for_requests(skip_resp_code_check: true)
expect(QA::Support::PageErrorChecker).not_to have_received(:check_page_for_error_code)
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment