Merge branch 'qa-quarantine-tests' into 'master'

Allow tests to be quarantined

Closes gitlab-org/quality/team-tasks#80

See merge request gitlab-org/gitlab-ce!24022

qa/README.md

@@ -86,7 +86,7 @@ The environment variable `QA_COOKIES` can be set to send additional cookies
 on every request. This is necessary on gitlab.com to direct traffic to the
 canary fleet. To do this set `QA_COOKIES="gitlab_canary=true"`.
 
-To set multiple cookies, separate them with the `;` character, for example: `QA_COOKIES="cookie1=value;cookie2=value2"` 
+To set multiple cookies, separate them with the `;` character, for example: `QA_COOKIES="cookie1=value;cookie2=value2"`
 
 
 ### Building a Docker image to test
@@ -100,3 +100,24 @@ docker build -t gitlab/gitlab-ce-qa:nightly .
 ```
 
 [GDK]: https://gitlab.com/gitlab-org/gitlab-development-kit/
+
+### Quarantined tests
+
+Tests can be put in quarantine by assigning `:quarantine` metadata. This means
+they will be skipped unless run with `--tag quarantine`. This can be used for
+tests that are expected to fail while a fix is in progress (similar to how
+[`skip` or `pending`](https://relishapp.com/rspec/rspec-core/v/3-8/docs/pending-and-skipped-examples)
+ can be used).
+
+```
+bin/qa Test::Instance::All http://localhost --tag quarantine
+```
+
+If `quarantine` is used with other tags, tests will only be run if they have at
+least one of the tags other than `quarantine`. This is different from how RSpec
+tags usually work, where all tags are inclusive.
+
+For example, suppose one test has `:smoke` and `:quarantine` metadata, and
+another test has `:ldap` and `:quarantine` metadata. If the tests are run with
+`--tag smoke --tag quarantine`, only the first test will run. The test with
+`:ldap` will not run even though it also has `:quarantine`.

qa/spec/spec_helper.rb

@@ -5,6 +5,24 @@ Dir[::File.join(__dir__, 'support', '**', '*.rb')].each { |f| require f }
 RSpec.configure do |config|
   config.before do |example|
     QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
+
+    # If quarantine is tagged, skip tests that have other metadata unless
+    # they're also tagged. This lets us run quarantined tests in a particular
+    # category without running tests in other categories.
+    # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
+    # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
+    # using `--tag quarantine --tag smoke`, without this check we'd end up
+    # running that ldap test as well.
+    if config.inclusion_filter[:quarantine]
+      skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config)
+    end
+  end
+
+  config.before(:each, :quarantine) do |example|
+    # Skip tests in quarantine unless we explicitly focus on them
+    # We could use an exclusion filter, but this way the test report will list
+    # the quarantined tests when they're not run so that we're aware of them
+    skip('In quarantine') unless config.inclusion_filter[:quarantine]
   end
 
   config.expect_with :rspec do |expectations|
@@ -22,3 +40,19 @@ RSpec.configure do |config|
   config.order = :random
   Kernel.srand config.seed
 end
+
+# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
+#
+# Returns true if
+# - the example metadata includes the quarantine tag
+# - and the metadata and inclusion filter both have any other tag
+# - or no other tags are in the inclusion filter
+def quarantine_and_optional_other_tag?(example, config)
+  return false unless example.metadata.keys.include? :quarantine
+
+  filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
+
+  return true if filters_other_than_quarantine.empty?
+
+  filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
+end

qa/spec/spec_helper_spec.rb

+# frozen_string_literal: true
+
+describe 'rspec config tests' do
+  let(:group) do
+    RSpec.describe do
+      shared_examples 'passing tests' do
+        example 'not in quarantine' do
+        end
+        example 'in quarantine', :quarantine do
+        end
+      end
+
+      context 'foo', :foo do
+        it_behaves_like 'passing tests'
+      end
+
+      context 'default' do
+        it_behaves_like 'passing tests'
+      end
+    end
+  end
+
+  context 'default config' do
+    it 'tests are skipped if in quarantine' do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = foo_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to eq(2)
+
+      ex = default_examples.find { |e| e.description == "not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = default_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+  end
+
+  context "with 'quarantine' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = :quarantine
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "only quarantined tests are run" do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to be(1)
+
+      ex = foo_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to be(1)
+
+      ex = default_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+  end
+
+  context "with 'foo' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = :foo
+      end
+
+      group.run
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "tests are not run if not tagged 'foo'" do
+      default_context = group.children.find { |c| c.description == "default" }
+      expect(default_context.descendant_filtered_examples.count).to eq(0)
+    end
+
+    it "tests are skipped if in quarantine" do
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "not in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = foo_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+  end
+
+  context "with 'quarantine' and 'foo' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { quarantine: true, foo: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it 'of tests tagged foo, only tests in quarantine run' do
+      group.run
+
+      foo_context = group.children.find { |c| c.description == "foo" }
+      foo_examples = foo_context.descendant_filtered_examples
+      expect(foo_examples.count).to eq(2)
+
+      ex = foo_examples.find { |e| e.description == "not in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
+
+      ex = foo_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+
+    it 'if tests are not tagged they are skipped, even if they are in quarantine' do
+      group.run
+      default_context = group.children.find { |c| c.description == "default" }
+      default_examples = default_context.descendant_filtered_examples
+      expect(default_examples.count).to eq(1)
+
+      ex = default_examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
+    end
+  end
+
+  context "with 'foo' and 'bar' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { bar: true, foo: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "runs tests tagged either 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'foo', :foo do
+        end
+        example 'bar', :bar do
+        end
+        example 'foo and bar', :foo, :bar do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(3)
+
+      ex = group.examples.find { |e| e.description == "foo" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "bar" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "foo and bar" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+
+    it "skips quarantined tests tagged 'foo' and/or 'bar'" do
+      group = RSpec.describe do
+        example 'foo in quarantine', :foo, :quarantine do
+        end
+        example 'foo and bar in quarantine', :foo, :bar, :quarantine do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(2)
+
+      ex = group.examples.find { |e| e.description == "foo in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+
+      ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('In quarantine')
+    end
+
+    it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'in quarantine', :quarantine do
+        end
+      end
+
+      group.run
+
+      ex = group.examples.find { |e| e.description == "in quarantine" }
+      expect(ex.execution_result.status).to be_nil
+    end
+  end
+
+  context "with 'foo' and 'bar' and 'quarantined' tagged" do
+    before do
+      RSpec.configure do |config|
+        config.inclusion_filter = { bar: true, foo: true, quarantine: true }
+      end
+    end
+    after do
+      RSpec.configure do |config|
+        config.inclusion_filter.clear
+      end
+    end
+
+    it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
+      group = RSpec.describe do
+        example 'foo', :foo do
+        end
+        example 'bar and quarantine', :bar, :quarantine do
+        end
+        example 'foo and bar', :foo, :bar do
+        end
+        example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
+        end
+      end
+
+      group.run
+      expect(group.examples.count).to eq(4)
+
+      ex = group.examples.find { |e| e.description == "foo" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
+
+      ex = group.examples.find { |e| e.description == "bar and quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+
+      ex = group.examples.find { |e| e.description == "foo and bar" }
+      expect(ex.execution_result.status).to eq(:pending)
+      expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
+
+      ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
+      expect(ex.execution_result.status).to eq(:passed)
+    end
+  end
+end