Merge branch 'write-failed-suite-resource-separately' into 'master'

Improve test resources delete process

See merge request gitlab-org/gitlab!82155

qa/Gemfile

@@ -29,6 +29,7 @@ gem 'influxdb-client', '~> 1.17'
 gem 'terminal-table', '~> 3.0.0', require: false
 gem 'slack-notifier', '~> 2.4', require: false
 gem 'fog-google', '~> 1.17', require: false
+gem 'google-cloud-storage', '~> 1.36', require: false
 
 gem 'confiner', '~> 0.2'
 

qa/Gemfile.lock

@@ -65,6 +65,8 @@ GEM
     deprecation_toolkit (1.5.1)
       activesupport (>= 4.2)
     diff-lcs (1.3)
+    digest-crc (0.6.4)
+      rake (>= 12.0.0, < 14.0.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     equalizer (0.0.11)
@@ -150,8 +152,20 @@ GEM
       google-apis-core (>= 0.4, < 2.a)
     google-apis-storage_v1 (0.9.0)
       google-apis-core (>= 0.4, < 2.a)
+    google-cloud-core (1.6.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
     google-cloud-env (1.5.0)
       faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.2.0)
+    google-cloud-storage (1.36.1)
+      addressable (~> 2.8)
+      digest-crc (~> 0.4)
+      google-apis-iamcredentials_v1 (~> 0.1)
+      google-apis-storage_v1 (~> 0.1)
+      google-cloud-core (~> 1.6)
+      googleauth (>= 0.16.2, < 2.a)
+      mini_mime (~> 1.0)
     googleauth (1.1.0)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
@@ -368,6 +382,7 @@ DEPENDENCIES
   faker (~> 2.19, >= 2.19.0)
   fog-google (~> 1.17)
   gitlab-qa
+  google-cloud-storage (~> 1.36)
   influxdb-client (~> 1.17)
   knapsack (~> 4.0)
   octokit (~> 4.21)

qa/Rakefile

@@ -60,13 +60,26 @@ task :delete_projects do
   QA::Tools::DeleteProjects.new.run
 end
 
-desc "Deletes resources created during E2E test runs"
-task :delete_test_resources, :file_pattern do |t, args|
-  QA::Tools::DeleteTestResources.new(args[:file_pattern]).run
-end
-
 desc "Deletes test users"
 task :delete_test_users, [:delete_before, :dry_run, :exclude_users] do |t, args|
   QA::Tools::DeleteTestUsers.new(args).run
 end
+
+namespace :test_resources do
+  desc "Deletes resources created during E2E test runs"
+  task :delete, :file_pattern do |t, args|
+    args.with_defaults(file_pattern: QA::Runtime::Env.test_resources_created_filepath)
+    QA::Tools::TestResourcesHandler.new(args[:file_pattern]).run_delete
+  end
+
+  desc "Upload test resources JSON files to GCS"
+  task :upload, [:file_pattern, :environment_name] do |t, args|
+    QA::Tools::TestResourcesHandler.new(args[:file_pattern]).upload(args[:environment_name])
+  end
+
+  desc "Download test resources JSON files from GCS"
+  task :download, [:environment_name] do |t, args|
+    QA::Tools::TestResourcesHandler.new.download(args[:environment_name])
+  end
+end
 # rubocop:enable Rails/RakeEnvironment

qa/qa/runtime/env.rb

@@ -449,11 +449,6 @@ module QA
         running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true)
       end
 
-      def test_resources_created_filepath
-        file_name = running_in_ci? ? "test-resources-#{SecureRandom.hex(3)}.json" : 'test-resources.json'
-        ENV.fetch('QA_TEST_RESOURCES_CREATED_FILEPATH', File.join(Path.qa_root, 'tmp', file_name))
-      end
-
       def ee_activation_code
         ENV['QA_EE_ACTIVATION_CODE']
       end

qa/qa/tools/test_resource_data_processor.rb

@@ -49,10 +49,12 @@ module QA
       # Otherwise create file and write data hash to file for the first time
       #
       # @return [void]
-      def write_to_file
+      def write_to_file(suite_failed)
         return if resources.empty?
 
-        file = Pathname.new(Runtime::Env.test_resources_created_filepath)
+        start_str = suite_failed ? 'failed-test-resources' : 'test-resources'
+        file_name = Runtime::Env.running_in_ci? ? "#{start_str}-#{SecureRandom.hex(3)}.json" : "#{start_str}.json"
+        file = Pathname.new(File.join(Runtime::Path.qa_root, 'tmp', file_name))
         FileUtils.mkdir_p(file.dirname)
 
         data = resources.deep_dup

qa/qa/tools/delete_test_resources.rb → qa/qa/tools/test_resources_handler.rb

 # frozen_string_literal: true
 
-# This script reads from test-resources JSON file to collect data about resources to delete
-# Filter out resources that cannot be deleted
-# Then deletes all deletable resources that E2E tests created
+require "google/cloud/storage"
+
+# This script handles resources created during E2E test runs
+#
+# Delete: find all matching file pattern, read file and delete resources
+# rake test_resources:delete
+# rake test_resources:delete[<file_pattern>]
+#
+# Upload: find all matching file pattern for failed test resources
+# upload these files to GCS bucket `failed-test-resources` under specific environment name
+# rake test_resources:upload[<file_pattern>,<environment_name>]
+#
+# Download: download JSON files under a given environment name (bucket directory)
+# save to local under `tmp/`
+# rake test_resources:download[<environment_name>]
 #
-# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
-# When in CI also requires: QA_TEST_RESOURCES_FILE_PATTERN
-# Run `rake delete_test_resources[<file_pattern>]`
+# Required environment variables:
+#   GITLAB_ADDRESS, required for delete task
+#   GITLAB_QA_ACCESS_TOKEN, required for delete task
+#   QA_TEST_RESOURCES_FILE_PATTERN, optional for delete task, required for upload task
+#   QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS, required for upload task or download task
 
 module QA
   module Tools
-    class DeleteTestResources
+    class TestResourcesHandler
       include Support::API
 
       IGNORED_RESOURCES = [
@@ -21,15 +35,14 @@ module QA
         'QA::EE::Resource::Settings::Elasticsearch'
       ].freeze
 
-      def initialize(file_pattern = Runtime::Env.test_resources_created_filepath)
-        raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS']
-        raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN']
+      PROJECT = 'gitlab-qa-resources'
+      BUCKET  = 'failed-test-resources'
 
-        @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
+      def initialize(file_pattern = nil)
         @file_pattern = file_pattern
       end
 
-      def run
+      def run_delete
         failures = files.flat_map do |file|
           resources = read_file(file)
           next if resources.nil?
@@ -44,6 +57,41 @@ module QA
         puts failures
       end
 
+      # Upload resources from failed test suites to GCS bucket
+      # Files are organized by environment in which tests were executed
+      #
+      # E.g: staging/failed-test-resources-<randomhex>.json
+      def upload(environment_name)
+        return puts "\nNothing to upload!" if files.empty?
+
+        files.each do |file|
+          file_name = "#{environment_name}/#{file.split('/').last}"
+          Runtime::Logger.info("Uploading #{file_name}...")
+          gcs_bucket.create_file(file, file_name)
+        end
+
+        puts "\nDone"
+      end
+
+      # Download files from GCS bucket by environment name
+      # Delete the files afterward
+      def download(environment_name)
+        files_list = gcs_bucket.files(prefix: "#{environment_name}")
+
+        return puts "\nNothing to download!" if files_list.empty?
+
+        files_list.each do |file|
+          local_path = "tmp/#{file.name.split('/').last}"
+          Runtime::Logger.info("Downloading #{file.name} to #{local_path}")
+          file.download(local_path)
+
+          Runtime::Logger.info("Deleting #{file.name} from bucket")
+          file.delete
+        end
+
+        puts "\nDone"
+      end
+
       private
 
       def files
@@ -85,7 +133,7 @@ module QA
             next if resource_not_found?(resource['api_path'])
 
             resource_info = resource['info'] ? "#{key} - #{resource['info']}" : "#{key} at #{resource['api_path']}"
-            delete_response = delete(Runtime::API::Request.new(@api_client, resource['api_path']).url)
+            delete_response = delete(Runtime::API::Request.new(api_client, resource['api_path']).url)
 
             if delete_response.code == 202 || delete_response.code == 204
               Runtime::Logger.info("Deleting #{resource_info}... SUCCESS")
@@ -99,7 +147,35 @@ module QA
 
       def resource_not_found?(api_path)
         # if api path contains param "?hard_delete=<boolean>", remove it
-        get(Runtime::API::Request.new(@api_client, api_path.split('?').first).url).code.eql? 404
+        get(Runtime::API::Request.new(api_client, api_path.split('?').first).url).code.eql? 404
+      end
+
+      def api_client
+        abort("\nPlease provide GITLAB_ADDRESS") unless ENV['GITLAB_ADDRESS']
+        abort("\nPlease provide GITLAB_QA_ACCESS_TOKEN") unless ENV['GITLAB_QA_ACCESS_TOKEN']
+
+        @api_client ||= Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
+      end
+
+      def gcs_storage
+        @gcs_storage ||= Google::Cloud::Storage.new(
+          project_id: PROJECT,
+          credentials: json_key
+        )
+      rescue StandardError => e
+        abort("\nThere might be something wrong with the JSON key file - [ERROR] #{e}")
+      end
+
+      def gcs_bucket
+        @gcs_bucket ||= gcs_storage.bucket(BUCKET, skip_lookup: true)
+      end
+
+      # Path to GCS service account json key file
+      # Or the content of the key file as a hash
+      def json_key
+        abort("\nPlease provide QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS") unless ENV['QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS']
+
+        @json_key ||= ENV["QA_FAILED_TEST_RESOURCES_GCS_CREDENTIALS"]
       end
     end
   end

qa/spec/runtime/env_spec.rb

@@ -360,36 +360,4 @@ RSpec.describe QA::Runtime::Env do
       end
     end
   end
-
-  describe '.test_resources_created_filepath' do
-    context 'when not in CI' do
-      before do
-        allow(described_class).to receive(:running_in_ci?).and_return(false)
-      end
-
-      it 'returns default path if QA_TEST_RESOURCES_CREATED_FILEPATH is not defined' do
-        stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil)
-
-        expect(described_class.test_resources_created_filepath).to include('tmp/test-resources.json')
-      end
-
-      it 'returns path if QA_TEST_RESOURCES_CREATED_FILEPATH is defined' do
-        stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', 'path/to_file')
-
-        expect(described_class.test_resources_created_filepath).to eq('path/to_file')
-      end
-    end
-
-    context 'when in CI' do
-      before do
-        allow(described_class).to receive(:running_in_ci?).and_return(true)
-        allow(SecureRandom).to receive(:hex).with(3).and_return('abc123')
-        stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil)
-      end
-
-      it 'returns path with random hex in file name' do
-        expect(described_class.test_resources_created_filepath).to include('tmp/test-resources-abc123.json')
-      end
-    end
-  end
 end

qa/spec/spec_helper.rb

@@ -69,7 +69,7 @@ RSpec.configure do |config|
 
   config.after(:suite) do |suite|
     # Write all test created resources to JSON file
-    QA::Tools::TestResourceDataProcessor.write_to_file
+    QA::Tools::TestResourceDataProcessor.write_to_file(suite.reporter.failed_examples.any?)
 
     # If requested, confirm that resources were used appropriately (e.g., not left with changes that interfere with
     # further reuse)

qa/spec/tools/test_resources_data_processor_spec.rb

@@ -43,18 +43,30 @@ RSpec.describe QA::Tools::TestResourceDataProcessor do
   end
 
   describe '.write_to_file' do
-    let(:resources_file) { Pathname.new(Faker::File.file_name(dir: 'tmp', ext: 'json')) }
+    using RSpec::Parameterized::TableSyntax
 
-    before do
-      stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', resources_file)
-
-      allow(File).to receive(:write)
+    where(:ci, :suite_failed, :file_path) do
+      true  | true  | 'root/tmp/failed-test-resources-random.json'
+      true  | false | 'root/tmp/test-resources-random.json'
+      false | true  | 'root/tmp/failed-test-resources.json'
+      false | false | 'root/tmp/test-resources.json'
     end
 
-    it 'writes applicable resources to file' do
-      processor.write_to_file
+    with_them do
+      let(:resources_file) { Pathname.new(file_path) }
+
+      before do
+        allow(QA::Runtime::Env).to receive(:running_in_ci?).and_return(ci)
+        allow(File).to receive(:write)
+        allow(QA::Runtime::Path).to receive(:qa_root).and_return('root')
+        allow(SecureRandom).to receive(:hex).with(any_args).and_return('random')
+      end
+
+      it 'writes applicable resources to file' do
+        processor.write_to_file(suite_failed)
 
-      expect(File).to have_received(:write).with(resources_file, JSON.pretty_generate(result))
+        expect(File).to have_received(:write).with(resources_file, JSON.pretty_generate(result))
+      end
     end
   end
 end