Integrate knapsack report fetching and generation in to QA framework

- Download report before setup
- Set knapsack config automatically from ci job name
- Rename and move regenerated report after suite finished
- Update upload method to handle multiple directories

.gitlab/ci/review-apps/qa.gitlab-ci.yml

@@ -26,35 +26,22 @@
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - cd qa
-    - if [ -n "$KNAPSACK_REPORT_PATH" ]; then
-        bundle exec rake knapsack:download;
-      fi
-  artifacts:
-    paths:
-      - qa/tmp
-    expire_in: 7 days
-    when: always
-
-.parallel-qa-base:
-  parallel: 5
-  variables:
-    KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
   script:
     - |
       bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
         -- \
         --color --format documentation \
         --format RspecJunitFormatter --out tmp/rspec.xml
-  after_script:
-    - if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
-        mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
-      fi
   artifacts:
     paths:
-      - qa/tmp # we can't merge list so need to include explicitly once more
-      - qa/knapsack/gcs/regenerated-*.json
+      - qa/tmp
     reports:
       junit: qa/tmp/rspec.xml
+    expire_in: 7 days
+    when: always
+
+.parallel-qa-base:
+  parallel: 5
 
 .allure-report-base:
   image:
@@ -79,17 +66,6 @@
         --ignore-missing-results \
         --color
 
-.knapsack-upload-base:
-  image:
-    name: ${QA_IMAGE}
-    entrypoint: [""]
-  stage: post-qa
-  allow_failure: true
-  before_script:
-    - cd qa
-  script:
-    - bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
-
 review-qa-smoke:
   extends:
     - .review-qa-base
@@ -97,8 +73,8 @@ review-qa-smoke:
   retry: 1  # This is confusing but this means "2 runs at max".
   variables:
     QA_RUN_TYPE: review-qa-smoke
-  script:
-    - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
+    QA_SCENARIO: Test::Instance::Smoke
+
 
 review-qa-reliable:
   extends:
@@ -109,7 +85,6 @@ review-qa-reliable:
   variables:
     QA_RUN_TYPE: review-qa-reliable
     QA_SCENARIO: Test::Instance::Reliable
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
 
 review-qa-all:
   extends:
@@ -119,7 +94,6 @@ review-qa-all:
   variables:
     QA_RUN_TYPE: review-qa-all
     QA_SCENARIO: Test::Instance::All
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
 
 review-performance:
   extends:
@@ -174,18 +148,15 @@ allure-report-qa-all:
     ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
     ALLURE_JOB_NAME: review-qa-all
 
-knapsack-report-qa-all:
-  extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-all
-  needs: ["review-qa-all"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
-
-knapsack-report-qa-reliable:
+knapsack-report:
   extends:
-    - .knapsack-upload-base
-    - .review:rules:knapsack-report-qa-reliable
-  needs: ["review-qa-reliable"]
-  variables:
-    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
+    - .review:rules:knapsack-report
+  image:
+    name: ${QA_IMAGE}
+    entrypoint: [""]
+  stage: post-qa
+  allow_failure: true
+  before_script:
+    - cd qa
+  script:
+    - bundle exec rake 'knapsack:upload[tmp/knapsack]'

.gitlab/ci/rules.gitlab-ci.yml

@@ -1689,20 +1689,10 @@
     - when: on_success
     - when: on_failure
 
-# Generate knapsack report on successful runs only
-# Reliable suite will pass most of the time so this should yield best distribution
-.review:rules:knapsack-report-qa-reliable:
+.review:rules:knapsack-report:
   rules:
     - if: '$KNAPSACK_GENERATE_REPORT == "true"'
-      when: on_success
-
-# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `knapsack-report-qa-all`.
-.review:rules:knapsack-report-qa-all:
-  rules:
-    - if: '$KNAPSACK_GENERATE_REPORT != "true"'
-      when: never
-    - when: on_success
-    - when: on_failure
+      when: always
 
 .review:rules:review-cleanup:
   rules:

qa/qa/runtime/env.rb

@@ -281,9 +281,7 @@ module QA
       end
 
       def knapsack?
-        return false unless ENV['CI_NODE_TOTAL'].to_i > 1
-
-        !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
+        running_in_ci? && ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true"
       end
 
       def ldap_username

qa/qa/scenario/template.rb

@@ -32,6 +32,11 @@ module QA
         #   Given *gitlab_address* = 'http://gitlab-abc123.test/' #=> http://about.gitlab-abc123.test/
         Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon
 
+        ##
+        # Setup knapsack and download latest report
+        #
+        Tools::KnapsackReport.configure! if Runtime::Env.knapsack?
+
         ##
         # Perform before hooks, which are different for CE and EE
         #

qa/qa/tools/knapsack_report.rb

@@ -9,46 +9,92 @@ module QA
       BUCKET = "knapsack-reports"
 
       class << self
+        def configure!
+          new.configure
+        end
+
+        def move
+          new.move_regenerated_report
+        end
+
         def download
           new.download_report
         end
 
-        def upload(glob)
-          new.upload_report(glob)
+        def upload(base_path)
+          new.upload_report(base_path)
         end
       end
 
-      def initialize
-        ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
-        ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!")
+      # Configure knapsack report
+      #
+      # * Setup variables
+      # * Fetch latest report
+      #
+      # @return [void]
+      def configure
+        Knapsack.logger = QA::Runtime::Logger.logger
+        Knapsack.report.config(
+          # temp compatibility before all pipelines migrate to automatically setting report path
+          test_file_pattern: ENV["KNAPSACK_TEST_FILE_PATTERN"] || "qa/specs/features/**/*_spec.rb",
+          report_path: ENV["KNAPSACK_REPORT_PATH"] || report_path
+        )
+
+        download_report
+      rescue StandardError => e
+        logger.warn("Failed to download latest knapsack report: #{e}")
+        logger.warn("Falling back to 'knapsack/master_report.json'")
+        Knapsack.report.config(report_path: "knapsack/master_report.json")
       end
 
       # Download knapsack report from gcs bucket
       #
       # @return [void]
       def download_report
-        logger.info("Downloading latest knapsack report '#{report_file}'")
+        logger.info("Downloading latest knapsack report '#{report_file}' to '#{report_path}'")
         file = client.get_object(BUCKET, report_file)
-
-        logger.info("Saving latest knapsack report to '#{report_path}'")
         File.write(report_path, file[:body])
       end
 
+      # Rename and move new regenerated report
+      #
+      # @return [void]
+      def move_regenerated_report
+        return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true"
+
+        path = "tmp/knapsack/#{report_name}"
+        FileUtils.mkdir_p(path)
+
+        # Use path from knapsack config in case of fallback to master_report.json
+        FileUtils.cp(Knapsack.report.report_path, "#{path}/#{ENV['CI_NODE_INDEX']}.json")
+      end
+
       # Merge and upload knapsack report to gcs bucket
       #
-      # @param [String] glob
+      # Will iterate over separate folders for regenerated reports created by move_regenerated_report method,
+      # merge them together and upload to GCS bucket
+      #
+      # @param [String] base_path
       # @return [void]
-      def upload_report(glob)
-        reports = Dir[glob]
-        return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty?
+      def upload_report(base_path)
+        report_dirs = Pathname.glob("#{base_path}/*").select(&:directory?)
+        return logger.error("Path '#{base_path}' did not contain any subfolders!") if report_dirs.empty?
 
-        report = reports
-          .map { |path| JSON.parse(File.read(path)) }
-          .reduce({}, :merge)
-        return logger.error("Knapsack generated empty report, skipping upload!") if report.empty?
+        report_dirs.each do |dir|
+          name = dir.basename.to_s
+          file = "#{name}.json"
+          jsons = dir.glob("*.json")
 
-        logger.info("Uploading latest knapsack report '#{report_file}'")
-        client.put_object(BUCKET, report_file, JSON.pretty_generate(report))
+          next logger.warn("Path #{name} did not contain any report json files, skipping upload!") if jsons.empty?
+
+          report = jsons
+            .map { |json| JSON.parse(File.read(json)) }
+            .reduce({}, :merge)
+          next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty?
+
+          logger.info("Uploading latest knapsack report '#{file}'")
+          client.put_object(BUCKET, file, JSON.pretty_generate(report))
+        end
       end
 
       private
@@ -66,22 +112,45 @@ module QA
       def client
         @client ||= Fog::Storage::Google.new(
           google_project: PROJECT,
-          google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"]
+          google_json_key_location: gcs_credentials
         )
       end
 
+      # Base path of knapsack report
+      #
+      # @return [String]
+      def report_base_path
+        @report_base_path ||= "knapsack/gcs"
+      end
+
       # Knapsack report path
       #
       # @return [String]
       def report_path
-        @report_path ||= ENV["KNAPSACK_REPORT_PATH"]
+        @report_path ||= "#{report_base_path}/#{report_file}"
       end
 
       # Knapsack report name
       #
       # @return [String]
       def report_file
-        @report_name ||= report_path.split("/").last
+        @report_file ||= "#{report_name}.json"
+      end
+
+      # Report name
+      #
+      # @return [String]
+      def report_name
+        @report_name ||= ENV["CI_JOB_NAME"]&.split(" ")&.first
+      end
+
+      # Path to GCS credentials json
+      #
+      # @return [String]
+      def gcs_credentials
+        @gcs_credentials ||= ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise(
+          "QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!"
+        )
       end
     end
   end

qa/spec/runtime/env_spec.rb

@@ -170,34 +170,30 @@ RSpec.describe QA::Runtime::Env do
 
   describe '.knapsack?' do
     before do
+      stub_env('CI', 'true')
       stub_env('CI_NODE_TOTAL', '2')
     end
 
-    it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do
+    it 'returns true if running in parallel CI run' do
       stub_env('KNAPSACK_GENERATE_REPORT', 'true')
 
       expect(described_class.knapsack?).to be_truthy
     end
 
-    it 'returns true if KNAPSACK_REPORT_PATH is defined' do
-      stub_env('KNAPSACK_REPORT_PATH', '/a/path')
-
-      expect(described_class.knapsack?).to be_truthy
+    it 'returns false if knapsack disabled' do
+      stub_env('NO_KNAPSACK', 'true')
+      expect(described_class.knapsack?).to be_falsey
     end
 
-    it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do
-      stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern')
-
-      expect(described_class.knapsack?).to be_truthy
-    end
+    it 'returns false if not running in parallel job' do
+      stub_env('CI_NODE_TOTAL', '1')
 
-    it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
       expect(described_class.knapsack?).to be_falsey
     end
 
-    it 'returns false if not running in parallel job' do
-      stub_env('CI_NODE_TOTAL', '1')
-      stub_env('KNAPSACK_GENERATE_REPORT', 'true')
+    it 'returns false if not running in ci' do
+      stub_env('CI', nil)
+      stub_env('CI_SERVER', nil)
 
       expect(described_class.knapsack?).to be_falsey
     end

qa/spec/spec_helper.rb

@@ -10,7 +10,7 @@ require 'active_support/core_ext/object/blank'
 require_relative 'qa_deprecation_toolkit_env'
 QaDeprecationToolkitEnv.configure!
 
-Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK']
+Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack?
 
 QA::Runtime::Browser.configure!
 QA::Runtime::AllureReport.configure!
@@ -69,6 +69,7 @@ RSpec.configure do |config|
     next if suite.reporter.failed_examples.present?
 
     QA::Resource::ReusableProject.remove_all_via_api!
+    QA::Tools::KnapsackReport.move if QA::Runtime::Env.knapsack?
   end
 
   config.expect_with :rspec do |expectations|

qa/tasks/knapsack.rake

@@ -10,8 +10,8 @@ namespace :knapsack do
   end
 
   desc "Merge and upload knapsack report"
-  task :upload, [:glob_pattern] do |_task, args|
-    QA::Tools::KnapsackReport.upload(args[:glob_pattern])
+  task :upload, [:base_path] do |_task, args|
+    QA::Tools::KnapsackReport.upload(args[:base_path])
   end
 end
 # rubocop:enable Rails/RakeEnvironment