Merge branch 'gy-qa-unregister-runner' into 'master'

Ensure qa runners are unregistered after tests

Closes gitlab-qa#556

See merge request gitlab-org/gitlab!28013

doc/development/testing_guide/end_to_end/rspec_metadata_tests.md

@@ -13,3 +13,4 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec
 | `:quarantine`     | The test has been [quarantined](https://about.gitlab.com/handbook/engineering/quality/guidelines/debugging-qa-test-failures/#quarantining-tests), will run in a separate job that only includes quarantined tests, and is allowed to fail. The test will be skipped in its regular job so that if it fails it will not hold up the pipeline. |
 | `:reliable`       | The test has been [promoted to a reliable test](https://about.gitlab.com/handbook/engineering/quality/guidelines/reliable-tests/#promoting-an-existing-test-to-reliable) meaning it passes consistently in all pipelines, including merge requests. |
 | `:requires_admin` | The test requires an admin account. Tests with the tag are excluded when run against Canary and Production environments. |
+| `:runner`         | The test depends on and will set up a GitLab Runner instance, typically to run a pipeline. |

qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb

 # frozen_string_literal: true
 
 module QA
-  context 'Verify', :docker do
+  context 'Verify', :docker, :runner do
     describe 'Pipeline creation and processing' do
       let(:executor) { "qa-runner-#{Time.now.to_i}" }
       let(:max_wait) { 30 }
@@ -12,7 +12,7 @@ module QA
         end
       end
 
-      before do
+      let!(:runner) do
         Resource::Runner.fabricate! do |runner|
           runner.project = project
           runner.name = executor
@@ -21,7 +21,7 @@ module QA
       end
 
       after do
-        Service::DockerRun::GitlabRunner.new(executor).remove!
+        runner.remove_via_api!
       end
 
       it 'users creates a pipeline which gets processed', :smoke do

qa/qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb

 # frozen_string_literal: true
 
 module QA
-  context 'Verify', :docker do
+  context 'Verify', :docker, :runner do
     describe 'Runner registration' do
       let(:executor) { "qa-runner-#{Time.now.to_i}" }
+      let!(:runner) do
+        Resource::Runner.fabricate! do |runner|
+          runner.name = executor
+        end
+      end
 
       after do
-        Service::DockerRun::GitlabRunner.new(executor).remove!
+        runner.remove_via_api!
       end
 
       it 'user registers a new specific runner' do
         Flow::Login.sign_in
 
-        Resource::Runner.fabricate! do |runner|
-          runner.name = executor
-        end.project.visit!
+        runner.project.visit!
 
         Page::Project::Menu.perform(&:go_to_ci_cd_settings)
         Page::Project::Settings::CICD.perform do |settings|

qa/qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb

@@ -3,7 +3,7 @@
 require 'digest/sha1'
 
 module QA
-  context 'Release', :docker do
+  context 'Release', :docker, :runner do
     describe 'Git clone using a deploy key' do
       before do
         Flow::Login.sign_in
@@ -16,7 +16,7 @@ module QA
 
         @repository_location = @project.repository_ssh_location
 
-        Resource::Runner.fabricate_via_api! do |resource|
+        @runner = Resource::Runner.fabricate_via_api! do |resource|
           resource.project = @project
           resource.name = @runner_name
           resource.tags = %w[qa docker]
@@ -25,7 +25,7 @@ module QA
       end
 
       after do
-        Service::DockerRun::GitlabRunner.new(@runner_name).remove!
+        @runner.remove_via_api!
       end
 
       keys = [

qa/qa/specs/features/ee/browser_ui/3_create/web_ide/web_terminal_spec.rb

@@ -3,7 +3,7 @@
 module QA
   # This test was quarantined because relative URL isn't supported
   # See https://gitlab.com/gitlab-org/gitlab/issues/13833
-  context 'Create', :quarantine do
+  context 'Create', :runner, :quarantine do
     describe 'Web IDE web terminal', :docker do
       before do
         project = Resource::Project.fabricate_via_api! do |project|
@@ -48,8 +48,7 @@ module QA
       end
 
       after do
-        # Remove the runner even if the test fails
-        Service::DockerRun::GitlabRunner.new(@runner.name).remove! if @runner
+        @runner.remove_via_api! if @runner
       end
 
       it 'user starts the web terminal' do

qa/qa/specs/features/ee/browser_ui/6_release/pipelines_for_merged_results_and_merge_trains_spec.rb

@@ -3,12 +3,12 @@
 require 'securerandom'
 
 module QA
-  context 'Release', :docker do
+  context 'Release', :docker, :runner do
     describe 'Pipelines for merged results and merge trains' do
       before(:context) do
         @group = Resource::Group.fabricate_via_api!
 
-        Resource::Runner.fabricate_via_api! do |runner|
+        @runner = Resource::Runner.fabricate_via_api! do |runner|
           runner.token = @group.reload!.runners_token
           runner.name = @group.name
           runner.tags = [@group.name]
@@ -55,7 +55,7 @@ module QA
       end
 
       after(:context) do
-        Service::DockerRun::GitlabRunner.new(@group.name).remove!
+        @runner.remove_via_api! if @runner
       end
 
       it 'creates a pipeline with merged results' do

qa/qa/specs/features/ee/browser_ui/secure/create_merge_request_with_secure_spec.rb

@@ -3,7 +3,7 @@
 require 'pathname'
 
 module QA
-  context 'Secure', :docker do
+  context 'Secure', :docker, :runner do
     describe 'Security Reports in a Merge Request' do
       let(:sast_vuln_count) { 5 }
       let(:dependency_scan_vuln_count) { 4 }
@@ -12,7 +12,7 @@ module QA
       let(:remediable_vuln_name) { "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js" }
 
       after do
-        Service::DockerRun::GitlabRunner.new(@executor).remove!
+        @runner.remove_via_api! if @runner
 
         Runtime::Feature.enable('job_log_json') if @job_log_json_flag_enabled
       end
@@ -29,7 +29,7 @@ module QA
           p.initialize_with_readme = true
         end
 
-        Resource::Runner.fabricate! do |runner|
+        @runner = Resource::Runner.fabricate! do |runner|
           runner.project = @project
           runner.name = @executor
           runner.tags = %w[qa test]

qa/qa/specs/features/ee/browser_ui/secure/license_compliance_spec.rb

@@ -3,7 +3,7 @@
 require 'pathname'
 
 module QA
-  context 'Secure', :docker do
+  context 'Secure', :docker, :runner do
     let(:approved_license_name) { "MIT" }
     let(:denied_license_name) { "WTFPL" }
 
@@ -42,7 +42,7 @@ module QA
       let(:number_of_licenses_in_fixture) { 2 }
 
       after do
-        Service::DockerRun::GitlabRunner.new(@executor).remove!
+        @runner.remove_via_api!
       end
 
       before do
@@ -55,7 +55,7 @@ module QA
           project.description = 'Project with Secure'
         end
 
-        Resource::Runner.fabricate! do |runner|
+        @runner = Resource::Runner.fabricate! do |runner|
           runner.project = @project
           runner.name = @executor
           runner.tags = %w[qa test]

qa/qa/specs/features/ee/browser_ui/secure/merge_request_license_widget_spec.rb

@@ -3,14 +3,14 @@
 require 'pathname'
 
 module QA
-  context 'Secure', :docker do
+  context 'Secure', :docker, :runner do
     describe 'License merge request widget' do
       let(:approved_license_name) { "MIT" }
       let(:blacklisted_license_name) { "Zlib" }
       let(:executor) {"qa-runner-#{Time.now.to_i}"}
 
       after do
-        Service::DockerRun::GitlabRunner.new(executor).remove!
+        @runner.remove_via_api!
       end
 
       before do
@@ -21,7 +21,7 @@ module QA
           project.description = 'License widget test'
         end
 
-        Resource::Runner.fabricate! do |runner|
+        @runner = Resource::Runner.fabricate! do |runner|
           runner.project = @project
           runner.name = executor
           runner.tags = %w[qa test]

qa/qa/specs/features/ee/browser_ui/secure/security_reports_spec.rb

@@ -3,7 +3,7 @@
 require 'pathname'
 
 module QA
-  context 'Secure', :docker do
+  context 'Secure', :docker, :runner do
     let(:number_of_dependencies_in_fixture) { 7 }
     let(:dependency_scan_example_vuln) { 'Prototype pollution attack in mixin-deep' }
     let(:container_scan_example_vuln) { 'CVE-2017-18269 in glibc' }
@@ -12,7 +12,7 @@ module QA
 
     describe 'Security Reports' do
       after(:all) do
-        Service::DockerRun::GitlabRunner.new(@executor).remove!
+        @runner.remove_via_api!
       end
 
       before(:all) do
@@ -25,7 +25,7 @@ module QA
           p.description = 'Project with Secure'
         end
 
-        Resource::Runner.fabricate! do |runner|
+        @runner = Resource::Runner.fabricate! do |runner|
           runner.project = @project
           runner.name = @executor
           runner.tags = %w[qa test]

qa/qa/specs/features/ee/browser_ui/secure/vulnerability_management_spec.rb

@@ -3,7 +3,7 @@
 require 'pathname'
 
 module QA
-  context 'Secure', :docker do
+  context 'Secure', :docker, :runner do
     describe 'Security Reports in a Merge Request' do
       let(:sast_vuln_count) { 5 }
       let(:dependency_scan_vuln_count) { 4 }
@@ -12,7 +12,7 @@ module QA
       let(:remediable_vuln_name) { "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js" }
 
       after(:all) do
-        Service::DockerRun::GitlabRunner.new(@executor).remove!
+        @runner.remove_via_api!
 
         Runtime::Feature.enable('job_log_json') if @job_log_json_flag_enabled
       end
@@ -29,7 +29,7 @@ module QA
           p.initialize_with_readme = true
         end
 
-        Resource::Runner.fabricate! do |runner|
+        @runner = Resource::Runner.fabricate! do |runner|
           runner.project = @project
           runner.name = @executor
           runner.tags = %w[qa test]