Commit d54cb37d authored by Sanad Liaquat's avatar Sanad Liaquat

Retry failed tests with rspec-retry

Does not retry if tagged :quarantine. Also adds unit tests.
parent 50306121
No related merge requests found
...@@ -8,3 +8,4 @@ gem 'rspec', '~> 3.7' ...@@ -8,3 +8,4 @@ gem 'rspec', '~> 3.7'
gem 'selenium-webdriver', '~> 3.12' gem 'selenium-webdriver', '~> 3.12'
gem 'airborne', '~> 0.2.13' gem 'airborne', '~> 0.2.13'
gem 'nokogiri', '~> 1.10.1' gem 'nokogiri', '~> 1.10.1'
gem 'rspec-retry', '~> 0.6.1'
...@@ -76,6 +76,8 @@ GEM ...@@ -76,6 +76,8 @@ GEM
rspec-mocks (3.7.0) rspec-mocks (3.7.0)
diff-lcs (>= 1.2.0, < 2.0) diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.7.0) rspec-support (~> 3.7.0)
rspec-retry (0.6.1)
rspec-core (> 3.3)
rspec-support (3.7.0) rspec-support (3.7.0)
rubyzip (1.2.2) rubyzip (1.2.2)
selenium-webdriver (3.141.0) selenium-webdriver (3.141.0)
...@@ -101,6 +103,7 @@ DEPENDENCIES ...@@ -101,6 +103,7 @@ DEPENDENCIES
pry-byebug (~> 3.5.1) pry-byebug (~> 3.5.1)
rake (~> 12.3.0) rake (~> 12.3.0)
rspec (~> 3.7) rspec (~> 3.7)
rspec-retry (~> 0.6.1)
selenium-webdriver (~> 3.12) selenium-webdriver (~> 3.12)
BUNDLED WITH BUNDLED WITH
......
require_relative '../qa' require_relative '../qa'
require 'rspec/retry'
%w[helpers shared_examples].each do |d| %w[helpers shared_examples].each do |d|
Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f } Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
...@@ -31,6 +32,17 @@ RSpec.configure do |config| ...@@ -31,6 +32,17 @@ RSpec.configure do |config|
config.profile_examples = 10 config.profile_examples = 10
config.order = :random config.order = :random
Kernel.srand config.seed Kernel.srand config.seed
# show retry status in spec process
config.verbose_retry = true
# show exception that triggers a retry if verbose_retry is set to true
config.display_try_failure_messages = true
config.around do |example|
retry_times = example.metadata.keys.include?(:quarantine) ? 1 : 3
example.run_with_retry retry: retry_times
end
end end
# Skip tests in quarantine unless we explicitly focus on them. # Skip tests in quarantine unless we explicitly focus on them.
......
...@@ -28,6 +28,22 @@ describe 'rspec config tests' do ...@@ -28,6 +28,22 @@ describe 'rspec config tests' do
end end
end end
let(:group_2) do
RSpec.describe do
before(:all) do
@expectations = [1, 2, 3]
end
example 'not in quarantine' do
expect(@expectations.shift).to be(3)
end
example 'in quarantine', :quarantine do
expect(@expectations.shift).to be(3)
end
end
end
context 'with no tags focussed' do context 'with no tags focussed' do
before do before do
group.run group.run
...@@ -301,4 +317,39 @@ describe 'rspec config tests' do ...@@ -301,4 +317,39 @@ describe 'rspec config tests' do
end end
end end
end end
context 'rspec retry' do
context 'in an untagged context' do
before do
group_2.run
end
it 'should run example :retry times' do
examples = group_2.descendant_filtered_examples
ex = examples.find { |e| e.description == 'not in quarantine' }
expect(ex.execution_result.status).to eq(:passed)
end
end
context 'with :quarantine focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = :quarantine
end
group_2.run
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it 'should run example once only' do
examples = group_2.descendant_filtered_examples
ex = examples.find { |e| e.description == 'in quarantine' }
expect(ex.execution_result.status).to eq(:failed)
end
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment