Merge branch 'acunskis-knapsack-autogenerate' into 'master'

E2E: Generate knapsack report on scheduled runs

See merge request gitlab-org/gitlab!75479

.gitlab/ci/review-apps/qa.gitlab-ci.yml

@@ -26,6 +26,9 @@
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
     - cd qa
+    - if [ -n "$KNAPSACK_REPORT_PATH" ]; then
+        bundle exec rake knapsack:download;
+      fi
   artifacts:
     paths:
       - qa/tmp
@@ -34,15 +37,22 @@
 
 .parallel-qa-base:
   parallel: 5
+  variables:
+    KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
   script:
-    - export KNAPSACK_REPORT_PATH=knapsack/master_report.json
-    - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
     - |
       bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
         -- \
         --color --format documentation \
         --format RspecJunitFormatter --out tmp/rspec.xml
+  after_script:
+    - if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
+        mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
+      fi
   artifacts:
+    paths:
+      - qa/tmp # we can't merge list so need to include explicitly once more
+      - qa/knapsack/gcs/regenerated-*.json
     reports:
       junit: qa/tmp/rspec.xml
 
@@ -69,6 +79,16 @@
         --ignore-missing-results \
         --color
 
+.knapsack-upload-base:
+  image:
+    name: ${QA_IMAGE}
+    entrypoint: [""]
+  stage: post-qa
+  before_script:
+    - cd qa
+  script:
+    - bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
+
 review-qa-smoke:
   extends:
     - .review-qa-base
@@ -88,6 +108,7 @@ review-qa-reliable:
   variables:
     QA_RUN_TYPE: review-qa-reliable
     QA_SCENARIO: Test::Instance::Reliable
+    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
 
 review-qa-all:
   extends:
@@ -97,6 +118,7 @@ review-qa-all:
   variables:
     QA_RUN_TYPE: review-qa-all
     QA_SCENARIO: Test::Instance::All
+    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
 
 review-performance:
   extends:
@@ -141,3 +163,19 @@ allure-report-qa-all:
   variables:
     ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
     ALLURE_JOB_NAME: review-qa-all
+
+knapsack-report-qa-all:
+  extends:
+    - .knapsack-upload-base
+    - .review:rules:knapsack-report-qa-all
+  needs: ["review-qa-all"]
+  variables:
+    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
+
+knapsack-report-qa-reliable:
+  extends:
+    - .knapsack-upload-base
+    - .review:rules:knapsack-report-qa-reliable
+  needs: ["review-qa-reliable"]
+  variables:
+    KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json

.gitlab/ci/rules.gitlab-ci.yml

@@ -1621,6 +1621,10 @@
 
 .review:rules:review-qa-reliable:
   rules:
+    - <<: *if-dot-com-ee-2-hourly-schedule
+      allow_failure: true
+      variables:
+        KNAPSACK_GENERATE_REPORT: "true"
     - when: on_success
       allow_failure: true
 
@@ -1631,6 +1635,10 @@
       changes: *code-patterns
       when: manual
       allow_failure: true # manual jobs needs to be allowd to fail, otherwise they block the pipeline
+    - <<: *if-dot-com-ee-2-hourly-schedule
+      allow_failure: true
+      variables:
+        KNAPSACK_GENERATE_REPORT: "true"
     - when: on_success
       allow_failure: true
 
@@ -1646,6 +1654,20 @@
     - when: on_failure
       allow_failure: true
 
+# Generate knapsack report on successful runs only
+# Reliable suite will pass most of the time so this should yield best distribution
+.review:rules:knapsack-report-qa-reliable:
+  rules:
+    - <<: *if-dot-com-ee-2-hourly-schedule
+      when: on_success
+      allow_failure: true
+
+.review:rules:knapsack-report-qa-all:
+  rules:
+    - <<: *if-dot-com-ee-2-hourly-schedule
+      when: always
+      allow_failure: true
+
 .review:rules:review-cleanup:
   rules:
     - <<: *if-not-ee

qa/Gemfile

@@ -15,7 +15,7 @@ gem 'rest-client', '~> 2.1.0'
 gem 'rspec-retry', '~> 0.6.1', require: 'rspec/retry'
 gem 'rspec_junit_formatter', '~> 0.4.1'
 gem 'faker', '~> 2.19', '>= 2.19.0'
-gem 'knapsack', '~> 1.17'
+gem 'knapsack', '~> 4.0'
 gem 'parallel_tests', '~> 2.29'
 gem 'rotp', '~> 3.1.0'
 gem 'timecop', '~> 0.9.1'
@@ -27,6 +27,7 @@ gem 'zeitwerk', '~> 2.4'
 gem 'influxdb-client', '~> 1.17'
 gem 'terminal-table', '~> 1.8', require: false
 gem 'slack-notifier', '~> 2.4', require: false
+gem 'fog-google', '~> 1.17', require: false
 
 gem 'chemlab', '~> 0.9'
 gem 'chemlab-library-www-gitlab-com', '~> 0.1'

qa/Gemfile.lock

@@ -29,6 +29,7 @@ GEM
       uuid (>= 2.3, < 3)
     ast (2.4.2)
     binding_ninja (0.2.3)
+    builder (3.2.4)
     byebug (9.1.0)
     capybara (3.35.3)
       addressable
@@ -56,12 +57,14 @@ GEM
       adamantium (~> 0.2.0)
       equalizer (~> 0.0.9)
     concurrent-ruby (1.1.9)
+    declarative (0.0.20)
     deprecation_toolkit (1.5.1)
       activesupport (>= 4.2)
     diff-lcs (1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     equalizer (0.0.11)
+    excon (0.88.0)
     faker (2.19.0)
       i18n (>= 1.6, < 2)
     faraday (1.5.1)
@@ -85,6 +88,30 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
+    fog-core (2.1.0)
+      builder
+      excon (~> 0.58)
+      formatador (~> 0.2)
+      mime-types
+    fog-google (1.17.0)
+      fog-core (<= 2.1.0)
+      fog-json (~> 1.2)
+      fog-xml (~> 0.1.0)
+      google-apis-compute_v1 (~> 0.14)
+      google-apis-dns_v1 (~> 0.12)
+      google-apis-iamcredentials_v1 (~> 0.6)
+      google-apis-monitoring_v3 (~> 0.12)
+      google-apis-pubsub_v1 (~> 0.7)
+      google-apis-sqladmin_v1beta4 (~> 0.13)
+      google-apis-storage_v1 (~> 0.6)
+      google-cloud-env (~> 1.2)
+    fog-json (1.2.0)
+      fog-core
+      multi_json (~> 1.10)
+    fog-xml (0.1.4)
+      fog-core
+      nokogiri (>= 1.5.11, < 2.0.0)
+    formatador (0.3.0)
     gitlab (4.16.1)
       httparty (~> 0.14, >= 0.14.0)
       terminal-table (~> 1.5, >= 1.5.1)
@@ -94,6 +121,38 @@ GEM
       http (~> 5.0)
       nokogiri (~> 1.10)
       table_print (= 1.5.7)
+    google-apis-compute_v1 (0.21.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-core (0.4.1)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (>= 0.16.2, < 2.a)
+      httpclient (>= 2.8.1, < 3.a)
+      mini_mime (~> 1.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.a)
+      rexml
+      webrick
+    google-apis-dns_v1 (0.16.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-iamcredentials_v1 (0.8.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-monitoring_v3 (0.18.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-pubsub_v1 (0.10.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-sqladmin_v1beta4 (0.21.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-apis-storage_v1 (0.9.0)
+      google-apis-core (>= 0.4, < 2.a)
+    google-cloud-env (1.5.0)
+      faraday (>= 0.17.3, < 2.0)
+    googleauth (1.1.0)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (>= 0.16, < 2.a)
     http (5.0.4)
       addressable (~> 2.8)
       http-cookie (~> 1.0)
@@ -106,11 +165,13 @@ GEM
     httparty (0.20.0)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
+    httpclient (2.8.3)
     i18n (1.8.11)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     influxdb-client (1.17.0)
-    knapsack (1.17.1)
+    jwt (2.3.0)
+    knapsack (4.0.0)
       rake
     launchy (2.4.3)
       addressable (~> 2.3)
@@ -119,6 +180,7 @@ GEM
       rake (~> 13.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
+    memoist (0.16.2)
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
     method_source (0.9.0)
@@ -128,6 +190,7 @@ GEM
     mini_mime (1.1.0)
     mini_portile2 (2.6.1)
     minitest (5.14.4)
+    multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
     netrc (0.11.0)
@@ -138,6 +201,7 @@ GEM
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     oj (3.13.8)
+    os (1.1.4)
     parallel (1.19.2)
     parallel_tests (2.29.0)
       parallel
@@ -161,12 +225,17 @@ GEM
       rack (>= 1.0, < 3)
     rake (13.0.6)
     regexp_parser (2.1.1)
+    representable (3.1.1)
+      declarative (< 0.1.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
+      uber (< 0.2.0)
     require_all (3.0.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    retriable (3.1.2)
     rexml (3.2.5)
     rotp (3.1.0)
     rspec (3.10.0)
@@ -203,6 +272,11 @@ GEM
       childprocess (>= 0.5, < 5.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2)
+    signet (0.16.0)
+      addressable (~> 2.8)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     slack-notifier (2.4.0)
     systemu (2.6.5)
     table_print (1.5.7)
@@ -210,8 +284,10 @@ GEM
       unicode-display_width (~> 1.1, >= 1.1.1)
     thread_safe (0.3.6)
     timecop (0.9.1)
+    trailblazer-option (0.1.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
+    uber (0.1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8)
@@ -233,6 +309,7 @@ GEM
       nokogiri (~> 1.6)
       rubyzip (>= 1.3.0)
       selenium-webdriver (~> 4.0)
+    webrick (1.7.0)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.5.1)
@@ -250,9 +327,10 @@ DEPENDENCIES
   chemlab-library-www-gitlab-com (~> 0.1)
   deprecation_toolkit (~> 1.5.1)
   faker (~> 2.19, >= 2.19.0)
+  fog-google (~> 1.17)
   gitlab-qa
   influxdb-client (~> 1.17)
-  knapsack (~> 1.17)
+  knapsack (~> 4.0)
   octokit (~> 4.21)
   parallel (~> 1.19)
   parallel_tests (~> 2.29)

qa/Rakefile

 # frozen_string_literal: true
 # rubocop:disable Rails/RakeEnvironment
 
-load 'tasks/webdrivers.rake'
-load 'tasks/reliable_report.rake'
+Dir['tasks/*.rake'].each { |file| load file }
 
 require_relative 'qa/tools/revoke_all_personal_access_tokens'
 require_relative 'qa/tools/delete_subgroups'

qa/knapsack/gcs/.gitignore

+**
+
+!.gitignore

qa/qa/tools/knapsack_report.rb

+# frozen_string_literal: true
+
+require "fog/google"
+
+module QA
+  module Tools
+    class KnapsackReport
+      PROJECT = "gitlab-qa-resources"
+      BUCKET = "knapsack-reports"
+
+      class << self
+        def download
+          new.download_report
+        end
+
+        def upload(glob)
+          new.upload_report(glob)
+        end
+      end
+
+      def initialize
+        ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
+        ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!")
+      end
+
+      # Download knapsack report from gcs bucket
+      #
+      # @return [void]
+      def download_report
+        logger.info("Downloading latest knapsack report '#{report_file}'")
+        file = client.get_object(BUCKET, report_file)
+
+        logger.info("Saving latest knapsack report to '#{report_path}'")
+        File.write(report_path, file[:body])
+      end
+
+      # Merge and upload knapsack report to gcs bucket
+      #
+      # @param [String] glob
+      # @return [void]
+      def upload_report(glob)
+        reports = Dir[glob]
+        return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty?
+
+        report = reports
+          .map { |path| JSON.parse(File.read(path)) }
+          .reduce({}, :merge)
+        return logger.error("Knapsack generated empty report, skipping upload!") if report.empty?
+
+        logger.info("Uploading latest knapsack report '#{report_file}'")
+        client.put_object(BUCKET, report_file, JSON.pretty_generate(report))
+      end
+
+      private
+
+      # Logger instance
+      #
+      # @return [Logger]
+      def logger
+        @logger ||= Logger.new($stdout)
+      end
+
+      # GCS client
+      #
+      # @return [Fog::Storage::GoogleJSON]
+      def client
+        @client ||= Fog::Storage::Google.new(
+          google_project: PROJECT,
+          google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"]
+        )
+      end
+
+      # Knapsack report path
+      #
+      # @return [String]
+      def report_path
+        @report_path ||= ENV["KNAPSACK_REPORT_PATH"]
+      end
+
+      # Knapsack report name
+      #
+      # @return [String]
+      def report_file
+        @report_name ||= report_path.split("/").last
+      end
+    end
+  end
+end

qa/tasks/knapsack.rake

+# frozen_string_literal: true
+# rubocop:disable Rails/RakeEnvironment
+
+require_relative "../qa/tools/knapsack_report"
+
+namespace :knapsack do
+  desc "Download latest knapsack report"
+  task :download do
+    QA::Tools::KnapsackReport.download
+  end
+
+  desc "Merge and upload knapsack report"
+  task :upload, [:glob_pattern] do |_task, args|
+    QA::Tools::KnapsackReport.upload(args[:glob_pattern])
+  end
+end
+# rubocop:enable Rails/RakeEnvironment