Commit 572efb63 authored by Dan Davison's avatar Dan Davison

Merge branch 'qa-add-environment-selection-mechanism' into 'master'

Add ability to target specific environments

See merge request gitlab-org/gitlab!35668
parents 7aa31aa2 e504a91e
# Environment selection
Some tests are designed to be run against specific environments. We can specify
what environments to run tests against using the `only` metadata.
## Available switches
| Switch | Function | Type |
| -------| ------- | ----- |
| `tld` | Set the top-level domain matcher | `String` |
| `subdomain` | Set the subdomain matcher | `Array` or `String` |
| `domain` | Set the domain matcher | `String` |
| `production` | Match against production | `Static` |
WARNING: **Be advised:**
You cannot specify `:production` and `{ <switch>: 'value' }` simultaneously.
These options are mutually exclusive. If you want to specify production, you
can control the `tld` and `domain` independently.
## Examples
| Environment | Key | Matches (regex) |
| ---------------- | --- | --------------- |
| `any` | `` | `.+.com` |
| `gitlab.com` | `only: :production` | `gitlab.com` |
| `staging.gitlab.com` | `only: { subdomain: :staging }` | `(staging).+.com` |
| `gitlab.com and staging.gitlab.com` | `only: { subdomain: /(staging.)?/, domain: 'gitlab' }` | `(staging.)?gitlab.com` |
| `dev.gitlab.org` | `only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' }` | `(dev).gitlab.org` |
| `staging.gitlab.com & domain.gitlab.com` | `only: { subdomain: %i[staging domain] }` | `(staging|domain).+.com` |
```ruby
RSpec.describe 'Area' do
it 'runs in any environment' do; end
it 'runs only in production', only: :production do; end
it 'runs only in staging', only: { subdomain: :staging } do; end
it 'runs in dev', only: { tld: '.org', domain: 'gitlab', subdomain: 'dev' } do; end
it 'runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' } {}
end
```
NOTE: **Note:**
If the test has a `before` or `after`, you must add the `only` metadata
to the outer `RSpec.describe`.
...@@ -17,3 +17,4 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec ...@@ -17,3 +17,4 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec
| `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:gitaly_ha` | The test will run against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements-for-configuring-a-gitaly-cluster). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. |
| `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. | | `:skip_live_env` | The test will be excluded when run against live deployed environments such as Staging, Canary, and Production. |
| `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. | `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) will provision the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run.
| `:only` | The test is only to be run against specific environments. See [Environment selection](environment_selection.md) for more information. |
# frozen_string_literal: true # frozen_string_literal: true
require 'gitlab/qa' require 'gitlab/qa'
require 'uri'
module QA module QA
module Runtime module Runtime
...@@ -23,10 +24,39 @@ module QA ...@@ -23,10 +24,39 @@ module QA
SUPPORTED_FEATURES SUPPORTED_FEATURES
end end
def dot_com? def address_matches?(*options)
Runtime::Scenario.gitlab_address.include?(".com") return false unless Runtime::Scenario.attributes[:gitlab_address]
opts = {}
opts[:domain] = '.+'
opts[:tld] = '.com'
uri = URI(Runtime::Scenario.gitlab_address)
if options.any?
options.each do |option|
opts[:domain] = 'gitlab' if option == :production
if option.is_a?(Hash) && !option[:subdomain].nil?
opts.merge!(option)
opts[:subdomain] = case option[:subdomain]
when Array
"(#{option[:subdomain].join("|")})."
when Regexp
option[:subdomain]
else
"(#{option[:subdomain]})."
end
end
end
end
uri.host.match?(/^#{opts[:subdomain]}#{opts[:domain]}#{opts[:tld]}$/)
end end
alias_method :dot_com?, :address_matches?
def additional_repository_storage def additional_repository_storage
ENV['QA_ADDITIONAL_REPOSITORY_STORAGE'] ENV['QA_ADDITIONAL_REPOSITORY_STORAGE']
end end
......
...@@ -18,6 +18,10 @@ module QA ...@@ -18,6 +18,10 @@ module QA
config.before do |example| config.before do |example|
Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example) Quarantine.skip_or_run_quarantined_tests_or_contexts(config.inclusion_filter.rules, example)
if example.metadata.key?(:only)
skip('Test is not compatible with this environment') unless Runtime::Env.dot_com?(example.metadata[:only])
end
end end
end end
end end
......
...@@ -341,7 +341,7 @@ describe QA::Runtime::Env do ...@@ -341,7 +341,7 @@ describe QA::Runtime::Env do
end end
end end
describe '.dot_com?' do describe '.address_matches?' do
it 'returns true when url has .com' do it 'returns true when url has .com' do
QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com") QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
...@@ -351,7 +351,45 @@ describe QA::Runtime::Env do ...@@ -351,7 +351,45 @@ describe QA::Runtime::Env do
it 'returns false when url does not have .com' do it 'returns false when url does not have .com' do
QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test") QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.test")
expect(described_class.dot_com?).to be_falsy expect(described_class.dot_com?).to be_falsey
end
context 'with arguments' do
it 'returns true when :subdomain is set' do
QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
expect(described_class.dot_com?(subdomain: :staging)).to be_truthy
end
it 'matches multiple subdomains' do
QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
expect(described_class.address_matches?(subdomain: [:release, :staging])).to be_truthy
expect(described_class.address_matches?(:production, subdomain: [:release, :staging])).to be_truthy
end
it 'matches :production' do
QA::Runtime::Scenario.define(:gitlab_address, "https://gitlab.com/")
expect(described_class.address_matches?(:production)).to be_truthy
end
it 'doesnt match with mismatching switches' do
QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.test')
aggregate_failures do
expect(described_class.address_matches?(tld: '.net')).to be_falsey
expect(described_class.address_matches?(:production)).to be_falsey
expect(described_class.address_matches?(subdomain: [:staging])).to be_falsey
expect(described_class.address_matches?(domain: 'example')).to be_falsey
end
end
end
it 'returns false for mismatching' do
QA::Runtime::Scenario.define(:gitlab_address, "https://staging.gitlab.com")
expect(described_class.address_matches?(:production)).to be_falsey
end end
end end
end end
...@@ -280,4 +280,94 @@ describe QA::Specs::Helpers::Quarantine do ...@@ -280,4 +280,94 @@ describe QA::Specs::Helpers::Quarantine do
end end
end end
end end
describe 'running against specific environments' do
before do
QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com')
described_class.configure_rspec
end
describe 'description and context blocks' do
context 'with environment set' do
it 'can apply to contexts or descriptions' do
group = describe_successfully 'Runs in staging', only: { subdomain: :staging } do
it('runs in staging') {}
end
expect(group.examples[0].execution_result.status).to eq(:passed)
end
end
context 'with different environment set' do
before do
QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com')
described_class.configure_rspec
end
it 'does not run against production' do
group = describe_successfully 'Runs in staging', :something, only: { subdomain: :staging } do
it('runs in staging') {}
end
expect(group.examples[0].execution_result.status).to eq(:pending)
end
end
end
it 'runs only in staging' do
group = describe_successfully do
it('runs in staging', only: { subdomain: :staging }) {}
it('doesnt run in staging', only: :production) {}
it('runs in staging also', only: { subdomain: %i[release staging] }) {}
it('runs in any env') {}
end
expect(group.examples[0].execution_result.status).to eq(:passed)
expect(group.examples[1].execution_result.status).to eq(:pending)
expect(group.examples[2].execution_result.status).to eq(:passed)
expect(group.examples[3].execution_result.status).to eq(:passed)
end
context 'custom env' do
before do
QA::Runtime::Scenario.define(:gitlab_address, 'https://release.gitlab.net')
end
it 'runs on a custom environment' do
group = describe_successfully do
it('runs on release gitlab net', only: { tld: '.net', subdomain: :release, domain: 'gitlab' } ) {}
it('does not run on release', only: :production ) {}
end
expect(group.examples.first.execution_result.status).to eq(:passed)
expect(group.examples.last.execution_result.status).to eq(:pending)
end
end
context 'production' do
before do
QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/')
end
it 'runs on production' do
group = describe_successfully do
it('runs on prod', only: :production ) {}
it('does not run in prod', only: { subdomain: :staging }) {}
it('runs in prod and staging', only: { subdomain: /(staging.)?/, domain: 'gitlab' }) {}
end
expect(group.examples[0].execution_result.status).to eq(:passed)
expect(group.examples[1].execution_result.status).to eq(:pending)
expect(group.examples[2].execution_result.status).to eq(:passed)
end
end
it 'outputs a message for invalid environments' do
group = describe_successfully do
it('will skip', only: :production) {}
end
expect(group.examples.first.execution_result.pending_message).to match(/[Tt]est.*not compatible.*environment/)
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment