Commit 89a79282 authored by Mark Lapierre's avatar Mark Lapierre

Allow tests to be quarantined

Exclude quarantined tests by default.
parent 5106f88a
...@@ -23,6 +23,9 @@ module QA ...@@ -23,6 +23,9 @@ module QA
args.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any? args.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any?
end end
# Skip quarantined tests by default
args.push(%w[--tag ~quarantine]) unless include_quarantine?
args.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled? args.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled?
QA::Runtime::Env.supported_features.each_key do |key| QA::Runtime::Env.supported_features.each_key do |key|
...@@ -38,6 +41,10 @@ module QA ...@@ -38,6 +41,10 @@ module QA
abort if status.nonzero? abort if status.nonzero?
end end
end end
def include_quarantine?
tags.include?(:quarantine) || options.include?('quarantine')
end
end end
end end
end end
...@@ -6,8 +6,12 @@ describe QA::Specs::Runner do ...@@ -6,8 +6,12 @@ describe QA::Specs::Runner do
allow(QA::Runtime::Browser).to receive(:configure!) allow(QA::Runtime::Browser).to receive(:configure!)
end end
it 'excludes the orchestrated tag by default' do it 'excludes the orchestrated and quarantine tags by default' do
expect_rspec_runner_arguments(['--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tag', '~orchestrated',
'--tag', '~quarantine',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
...@@ -16,7 +20,12 @@ describe QA::Specs::Runner do ...@@ -16,7 +20,12 @@ describe QA::Specs::Runner do
subject { described_class.new.tap { |runner| runner.tty = true } } subject { described_class.new.tap { |runner| runner.tty = true } }
it 'sets the `--tty` flag' do it 'sets the `--tty` flag' do
expect_rspec_runner_arguments(['--tty', '--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tty',
'--tag', '~orchestrated',
'--tag', '~quarantine',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
...@@ -25,18 +34,54 @@ describe QA::Specs::Runner do ...@@ -25,18 +34,54 @@ describe QA::Specs::Runner do
context 'when tags are set' do context 'when tags are set' do
subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } }
it 'focuses on the given tags' do it 'focuses on the given tags, plus the ~quarantine tag' do
expect_rspec_runner_arguments(['--tag', 'orchestrated', '--tag', 'github', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tag', 'orchestrated',
'--tag', 'github',
'--tag', '~quarantine',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
end end
context 'when "--tag smoke" is set as options' do context 'when only "--tag quarantine" is set as options' do
subject { described_class.new.tap { |runner| runner.options = %w[--tag quarantine] } }
it 'focuses on the given tags without the default args' do
expect_rspec_runner_arguments([
'--tag', 'quarantine',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform
end
end
context 'when only "--tag smoke" is set as options' do
subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke] } } subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke] } }
it 'focuses on the given tag without excluded the orchestrated tag' do it 'focuses on the given tag without the default args, but with the ~quarantine tag' do
expect_rspec_runner_arguments(['--tag', 'smoke', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tag', '~quarantine',
'--tag', 'smoke',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform
end
end
context 'when "--tag smoke" and "--tag quarantine" are set as options' do
subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke --tag quarantine] } }
it 'focuses on the given tags without the default args' do
expect_rspec_runner_arguments([
'--tag', 'smoke',
'--tag', 'quarantine',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
...@@ -45,8 +90,12 @@ describe QA::Specs::Runner do ...@@ -45,8 +90,12 @@ describe QA::Specs::Runner do
context 'when "qa/specs/features/foo" is set as options' do context 'when "qa/specs/features/foo" is set as options' do
subject { described_class.new.tap { |runner| runner.options = %w[qa/specs/features/foo] } } subject { described_class.new.tap { |runner| runner.options = %w[qa/specs/features/foo] } }
it 'passes the given tests path and excludes the orchestrated tag' do it 'passes the given tests path and excludes the default args' do
expect_rspec_runner_arguments(['--tag', '~orchestrated', 'qa/specs/features/foo']) expect_rspec_runner_arguments([
'--tag', '~orchestrated',
'--tag', '~quarantine',
'qa/specs/features/foo'
])
subject.perform subject.perform
end end
...@@ -55,8 +104,12 @@ describe QA::Specs::Runner do ...@@ -55,8 +104,12 @@ describe QA::Specs::Runner do
context 'when "-- qa/specs/features/foo" is set as options' do context 'when "-- qa/specs/features/foo" is set as options' do
subject { described_class.new.tap { |runner| runner.options = %w[-- qa/specs/features/foo] } } subject { described_class.new.tap { |runner| runner.options = %w[-- qa/specs/features/foo] } }
it 'passes the given tests path and excludes the orchestrated tag' do it 'passes the given tests path and excludes the default args' do
expect_rspec_runner_arguments(['--tag', '~orchestrated', '--', 'qa/specs/features/foo']) expect_rspec_runner_arguments([
'--tag', '~orchestrated',
'--tag', '~quarantine',
'--', 'qa/specs/features/foo'
])
subject.perform subject.perform
end end
...@@ -70,7 +123,12 @@ describe QA::Specs::Runner do ...@@ -70,7 +123,12 @@ describe QA::Specs::Runner do
subject { described_class.new } subject { described_class.new }
it 'it includes default args and excludes the skip_signup_disabled tag' do it 'it includes default args and excludes the skip_signup_disabled tag' do
expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~skip_signup_disabled', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tag', '~orchestrated',
'--tag', '~quarantine',
'--tag', '~skip_signup_disabled',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
...@@ -84,7 +142,12 @@ describe QA::Specs::Runner do ...@@ -84,7 +142,12 @@ describe QA::Specs::Runner do
subject { described_class.new } subject { described_class.new }
it 'it includes default args and excludes the requires_git_protocol_v2 tag' do it 'it includes default args and excludes the requires_git_protocol_v2 tag' do
expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~requires_git_protocol_v2', *described_class::DEFAULT_TEST_PATH_ARGS]) expect_rspec_runner_arguments([
'--tag', '~orchestrated',
'--tag', '~quarantine',
'--tag', '~requires_git_protocol_v2',
*described_class::DEFAULT_TEST_PATH_ARGS
])
subject.perform subject.perform
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment