Commit 7d926242 authored by Mark Lapierre's avatar Mark Lapierre

Skip quarantined tests via filters

Instead of modifying the runner, use config hooks to skip quarantined
tests, and to allow only quarantined tests to be run, if desired.

This way quarantined tests are skipped, not excluded completely, so
they are still included in test results.
parent e5a81aec
...@@ -5,6 +5,24 @@ Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f } ...@@ -5,6 +5,24 @@ Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f }
RSpec.configure do |config| RSpec.configure do |config|
config.before do |example| config.before do |example|
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
# If quarantine is tagged, skip tests that have other metadata unless
# they're also tagged. This lets us run quarantined tests in a particular
# category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
if config.inclusion_filter[:quarantine]
skip('Running only tagged tests in quarantine') unless quarantine_and_optional_other_tag?(example, config)
end
end
config.before(:each, :quarantine) do |example|
# Skip tests in quarantine unless we explicitly focus on them
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
skip('In quarantine') unless config.inclusion_filter[:quarantine]
end end
config.expect_with :rspec do |expectations| config.expect_with :rspec do |expectations|
...@@ -22,3 +40,20 @@ RSpec.configure do |config| ...@@ -22,3 +40,20 @@ RSpec.configure do |config|
config.order = :random config.order = :random
Kernel.srand config.seed Kernel.srand config.seed
end end
# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
#
# Returns true if
# - the example metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(example, config)
return false unless example.metadata.keys.include? :quarantine
filters_other_than_quarantine = config.inclusion_filter.rules.keys.dup
filters_other_than_quarantine.delete :quarantine
return true if filters_other_than_quarantine.empty?
filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
end
# frozen_string_literal: true
describe 'rspec config tests' do
let(:group) do
RSpec.describe do
shared_examples 'passing tests' do
example 'pass: not in quarantine' do
end
example 'pass: in quarantine', :quarantine do
end
end
context 'foo', :foo do
it_behaves_like 'passing tests'
end
context 'default' do
it_behaves_like 'passing tests'
end
end
end
context 'default config' do
it 'tests are skipped if in quarantine' do
group.run
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(2)
ex = default_examples.find { |e| e.description == "pass: not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = default_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
context "with 'quarantine' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = :quarantine
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it "only quarantined tests are run" do
group.run
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to be(1)
ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to be(1)
ex = default_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
context "with 'foo' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = :foo
end
group.run
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it "tests are not run if not tagged 'foo'" do
default_context = group.children.find { |c| c.description == "default" }
expect(default_context.descendant_filtered_examples.count).to eq(0)
end
it "tests are skipped if in quarantine" do
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
context "with 'quarantine' and 'foo' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = { quarantine: true, foo: true }
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it 'of tests tagged foo, only tests in quarantine run' do
group.run
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "pass: not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
ex = foo_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
it 'if tests are not tagged they are skipped, even if they are in quarantine' do
group.run
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(1)
ex = default_examples.find { |e| e.description == "pass: in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
end
end
context "with 'foo' and 'bar' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true }
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it "runs tests tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar', :bar do
end
example 'foo and bar', :foo, :bar do
end
end
group.run
expect(group.examples.count).to eq(3)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "bar" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:passed)
end
it "skips quarantined tests tagged 'foo' and/or 'bar'" do
group = RSpec.describe do
example 'foo in quarantine', :foo, :quarantine do
end
example 'foo and bar in quarantine', :foo, :bar, :quarantine do
end
end
group.run
expect(group.examples.count).to eq(2)
ex = group.examples.find { |e| e.description == "foo in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'in quarantine', :quarantine do
end
end
group.run
ex = group.examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to be_nil
end
end
context "with 'foo' and 'bar' and 'quarantined' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true, quarantine: true }
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar and quarantine', :bar, :quarantine do
end
example 'foo and bar', :foo, :bar do
end
example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
end
end
group.run
expect(group.examples.count).to eq(4)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
ex = group.examples.find { |e| e.description == "bar and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running only tagged tests in quarantine')
ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment