Merge branch 'qa-add-environment-selection-mechanism' into 'master'

Add ability to target specific environments

See merge request gitlab-org/gitlab!35668

doc/development/testing_guide/end_to_end/environment_selection.md

+# Environment selection
+
+Some tests are designed to be run against specific environments. We can specify
+what environments to run tests against using the `only` metadata.
+
+## Available switches
+
+| Switch | Function | Type |
+| -------| ------- | ----- |
+| `tld`  | Set the top-level domain matcher | `String` |
+| `subdomain` | Set the subdomain matcher | `Array` or `String` |
+| `domain` | Set the domain matcher | `String` |
+| `production` | Match against production | `Static` |
+
+WARNING: **Be advised:**
+You cannot specify `:production` and `{ <switch>: 'value' }` simultaneously.  
+These options are mutually exclusive. If you want to specify production, you
+can control the `tld` and `domain` independently.
+
+## Examples
+
+| Environment                              | Key | Matches (regex)                                                            |
+| ----------------                         | --- | ---------------                                                            |
+| `any`                                    | ``  | `.+.com`                                                                   |
+| `gitlab.com`                             | `only: :production` | `gitlab.com`                                               |
+| `staging.gitlab.com`                     | `only: { subdomain: :staging }` | `(staging).+.com`                              |
+| `gitlab.com and staging.gitlab.com`      | `only: { subdomain: /(staging.)?/, domain: 'gitlab' }` | `(staging.)?gitlab.com` |
+| `dev.gitlab.org`                         | `only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' }` | `(dev).gitlab.org` |
+| `staging.gitlab.com & domain.gitlab.com` | `only: { subdomain: %i[staging domain] }` | `(staging|domain).+.com`             |
+
+```ruby
+RSpec.describe 'Area' do
+  it 'runs in any environment' do; end
+
+  it 'runs only in production', only: :production do; end
+  
+  it 'runs only in staging', only: { subdomain: :staging } do; end
+
+  it 'runs in dev', only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' } do; end
+
+  it 'runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' } {}
+end
+```
+
+NOTE: **Note:**
+If the test has a `before` or `after`, you must add the `only` metadata
+to the outer `RSpec.describe`.

doc/development/testing_guide/end_to_end/rspec_metadata_tests.md

@@ -17,3 +17,4 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec
 | `:gitaly_ha`      | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. |
 | `:skip_live_env`  | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. |
 | `:jira`           | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run.
+| `:only`           | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. |

qa/qa/runtime/env.rb

 # frozen_string_literal: true
 
 require 'gitlab/qa'
+require 'uri'
 
 module QA
   module Runtime
@@ -23,9 +24,38 @@ module QA
         SUPPORTED_FEATURES
       end
 
-      def dot_com?
-        Runtime::Scenario.gitlab_address.include?(".com")
+      def address_matches?(*options)
+        return false unless Runtime::Scenario.attributes[:gitlab_address]
+
+        opts = {}
+        opts[:domain] = '.+'
+        opts[:tld] = '.com'
+
+        uri = URI(Runtime::Scenario.gitlab_address)
+
+        if options.any?
+          options.each do |option|
+            opts[:domain] = 'gitlab' if option == :production
+
+            if option.is_a?(Hash) && !option[:subdomain].nil?
+              opts.merge!(option)
+
+              opts[:subdomain] = case option[:subdomain]
+                                 when Array
+                                   "(#{option[:subdomain].join("|")})."
+                                 when Regexp
+                                   option[:subdomain]
+                                 else
+                                   "(#{option[:subdomain]})."
+                                 end
             end
+          end
+        end
+
+        uri.host.match?(/^#{opts[:subdomain]}#{opts[:domain]}#{opts[:tld]}$/)
+      end
+
+      alias_method :dot_com?, :address_matches?
 
       def additional_repository_storage
         ENV['QA_ADDITIONAL_REPOSITORY_STORAGE']

qa/qa/specs/helpers/quarantine.rb

@@ -18,6 +18,10 @@ module QA
 
             config.before do |example|
               Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example)
+
+              if example.metadata.key?(:only)
+                skip('Test is not compatible with this environment') unless Runtime::Env.dot_com?(example.metadata[:only])
+              end
             end
           end
         end

qa/spec/runtime/env_spec.rb

@@ -341,7 +341,7 @@ describe QA::Runtime::Env do
     end
   end
 
-  describe '.dot_com?' do
+  describe '.address_matches?' do
     it 'returns true when url has .com' do
       QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
 
@@ -351,7 +351,45 @@ describe QA::Runtime::Env do
     it 'returns false when url does not have .com' do
       QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test")
 
-      expect(described_class.dot_com?).to be_falsy
+      expect(described_class.dot_com?).to be_falsey
+    end
+
+    context 'with arguments' do
+      it 'returns true when :subdomain is set' do
+        QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
+
+        expect(described_class.dot_com?(subdomain: :staging)).to be_truthy
+      end
+
+      it 'matches multiple subdomains' do
+        QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
+
+        expect(described_class.address_matches?(subdomain: [:release, :staging])).to be_truthy
+        expect(described_class.address_matches?(:production, subdomain: [:release, :staging])).to be_truthy
+      end
+
+      it 'matches :production' do
+        QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.com/")
+
+        expect(described_class.address_matches?(:production)).to be_truthy
+      end
+
+      it 'doesnt match with mismatching switches' do
+        QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.test')
+
+        aggregate_failures do
+          expect(described_class.address_matches?(tld: '.net')).to be_falsey
+          expect(described_class.address_matches?(:production)).to be_falsey
+          expect(described_class.address_matches?(subdomain: [:staging])).to be_falsey
+          expect(described_class.address_matches?(domain: 'example')).to be_falsey
+        end
+      end
+    end
+
+    it 'returns false for mismatching' do
+      QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
+
+      expect(described_class.address_matches?(:production)).to be_falsey
     end
   end
 end

qa/spec/specs/helpers/quarantine_spec.rb

@@ -280,4 +280,94 @@ describe QA::Specs::Helpers::Quarantine do
       end
     end
   end
+
+  describe 'running against specific environments' do
+    before do
+      QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com')
+      described_class.configure_rspec
+    end
+
+    describe 'description and context blocks' do
+      context 'with environment set' do
+        it 'can apply to contexts or descriptions' do
+          group = describe_successfully 'Runs in staging', only: { subdomain: :staging } do
+            it('runs in staging') {}
+          end
+
+          expect(group.examples[0].execution_result.status).to eq(:passed)
+        end
+      end
+
+      context 'with different environment set' do
+        before do
+          QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com')
+          described_class.configure_rspec
+        end
+
+        it 'does not run against production' do
+          group = describe_successfully 'Runs in staging', :something, only: { subdomain: :staging } do
+            it('runs in staging') {}
+          end
+
+          expect(group.examples[0].execution_result.status).to eq(:pending)
+        end
+      end
+    end
+
+    it 'runs only in staging' do
+      group = describe_successfully do
+        it('runs in staging', only: { subdomain: :staging }) {}
+        it('doesnt run in staging', only: :production) {}
+        it('runs in staging also', only: { subdomain: %i[release staging] }) {}
+        it('runs in any env') {}
+      end
+
+      expect(group.examples[0].execution_result.status).to eq(:passed)
+      expect(group.examples[1].execution_result.status).to eq(:pending)
+      expect(group.examples[2].execution_result.status).to eq(:passed)
+      expect(group.examples[3].execution_result.status).to eq(:passed)
+    end
+
+    context 'custom env' do
+      before do
+        QA::Runtime::Scenario.define(:gitlab_address, 'https://release.gitlab.net')
+      end
+
+      it 'runs on a custom environment' do
+        group = describe_successfully do
+          it('runs on release gitlab net', only: { tld: '.net', subdomain: :release, domain: 'gitlab' } ) {}
+          it('does not run on release', only: :production ) {}
+        end
+
+        expect(group.examples.first.execution_result.status).to eq(:passed)
+        expect(group.examples.last.execution_result.status).to eq(:pending)
+      end
+    end
+
+    context 'production' do
+      before do
+        QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/')
+      end
+
+      it 'runs on production' do
+        group = describe_successfully do
+          it('runs on prod', only: :production ) {}
+          it('does not run in prod', only: { subdomain: :staging }) {}
+          it('runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' }) {}
+        end
+
+        expect(group.examples[0].execution_result.status).to eq(:passed)
+        expect(group.examples[1].execution_result.status).to eq(:pending)
+        expect(group.examples[2].execution_result.status).to eq(:passed)
+      end
+    end
+
+    it 'outputs a message for invalid environments' do
+      group = describe_successfully do
+        it('will skip', only: :production) {}
+      end
+
+      expect(group.examples.first.execution_result.pending_message).to match(/[Tt]est.*not compatible.*environment/)
+    end
+  end
 end