Commit 064131ae authored by Lin Jen-Shin's avatar Lin Jen-Shin

Merge branch 'acunskis-knapsack-download' into 'master'

E2E: Integrate knapsack report setup in to QA framework

See merge request gitlab-org/gitlab!77316
parents 65f51f7d b2cb15ed
...@@ -26,35 +26,22 @@ ...@@ -26,35 +26,22 @@
- export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}" - echo "${CI_ENVIRONMENT_URL}"
- cd qa - cd qa
- if [ -n "$KNAPSACK_REPORT_PATH" ]; then
bundle exec rake knapsack:download;
fi
artifacts:
paths:
- qa/tmp
expire_in: 7 days
when: always
.parallel-qa-base:
parallel: 5
variables:
KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb"
script: script:
- | - |
bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \ bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \
-- \ -- \
--color --format documentation \ --color --format documentation \
--format RspecJunitFormatter --out tmp/rspec.xml --format RspecJunitFormatter --out tmp/rspec.xml
after_script:
- if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then
mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json;
fi
artifacts: artifacts:
paths: paths:
- qa/tmp # we can't merge list so need to include explicitly once more - qa/tmp
- qa/knapsack/gcs/regenerated-*.json
reports: reports:
junit: qa/tmp/rspec.xml junit: qa/tmp/rspec.xml
expire_in: 7 days
when: always
.parallel-qa-base:
parallel: 5
.allure-report-base: .allure-report-base:
image: image:
...@@ -79,17 +66,6 @@ ...@@ -79,17 +66,6 @@
--ignore-missing-results \ --ignore-missing-results \
--color --color
.knapsack-upload-base:
image:
name: ${QA_IMAGE}
entrypoint: [""]
stage: post-qa
allow_failure: true
before_script:
- cd qa
script:
- bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]'
review-qa-smoke: review-qa-smoke:
extends: extends:
- .review-qa-base - .review-qa-base
...@@ -97,8 +73,8 @@ review-qa-smoke: ...@@ -97,8 +73,8 @@ review-qa-smoke:
retry: 1 # This is confusing but this means "2 runs at max". retry: 1 # This is confusing but this means "2 runs at max".
variables: variables:
QA_RUN_TYPE: review-qa-smoke QA_RUN_TYPE: review-qa-smoke
script: QA_SCENARIO: Test::Instance::Smoke
- bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
review-qa-reliable: review-qa-reliable:
extends: extends:
...@@ -109,7 +85,6 @@ review-qa-reliable: ...@@ -109,7 +85,6 @@ review-qa-reliable:
variables: variables:
QA_RUN_TYPE: review-qa-reliable QA_RUN_TYPE: review-qa-reliable
QA_SCENARIO: Test::Instance::Reliable QA_SCENARIO: Test::Instance::Reliable
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json
review-qa-all: review-qa-all:
extends: extends:
...@@ -119,7 +94,6 @@ review-qa-all: ...@@ -119,7 +94,6 @@ review-qa-all:
variables: variables:
QA_RUN_TYPE: review-qa-all QA_RUN_TYPE: review-qa-all
QA_SCENARIO: Test::Instance::All QA_SCENARIO: Test::Instance::All
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
review-performance: review-performance:
extends: extends:
...@@ -174,18 +148,15 @@ allure-report-qa-all: ...@@ -174,18 +148,15 @@ allure-report-qa-all:
ALLURE_REPORT_PATH_PREFIX: gitlab-review-all ALLURE_REPORT_PATH_PREFIX: gitlab-review-all
ALLURE_JOB_NAME: review-qa-all ALLURE_JOB_NAME: review-qa-all
knapsack-report-qa-all: knapsack-report:
extends:
- .knapsack-upload-base
- .review:rules:knapsack-report-qa-all
needs: ["review-qa-all"]
variables:
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json
knapsack-report-qa-reliable:
extends: extends:
- .knapsack-upload-base - .review:rules:knapsack-report
- .review:rules:knapsack-report-qa-reliable image:
needs: ["review-qa-reliable"] name: ${QA_IMAGE}
variables: entrypoint: [""]
KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json stage: post-qa
allow_failure: true
before_script:
- cd qa
script:
- bundle exec rake 'knapsack:upload[tmp/knapsack/*/*.json]'
...@@ -1689,20 +1689,10 @@ ...@@ -1689,20 +1689,10 @@
- when: on_success - when: on_success
- when: on_failure - when: on_failure
# Generate knapsack report on successful runs only .review:rules:knapsack-report:
# Reliable suite will pass most of the time so this should yield best distribution
.review:rules:knapsack-report-qa-reliable:
rules: rules:
- if: '$KNAPSACK_GENERATE_REPORT == "true"' - if: '$KNAPSACK_GENERATE_REPORT == "true"'
when: on_success when: always
# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `knapsack-report-qa-all`.
.review:rules:knapsack-report-qa-all:
rules:
- if: '$KNAPSACK_GENERATE_REPORT != "true"'
when: never
- when: on_success
- when: on_failure
.review:rules:review-cleanup: .review:rules:review-cleanup:
rules: rules:
......
...@@ -281,9 +281,7 @@ module QA ...@@ -281,9 +281,7 @@ module QA
end end
def knapsack? def knapsack?
return false unless ENV['CI_NODE_TOTAL'].to_i > 1 ENV['CI_NODE_TOTAL'].to_i > 1 && ENV['NO_KNAPSACK'] != "true"
!!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
end end
def ldap_username def ldap_username
......
...@@ -32,6 +32,11 @@ module QA ...@@ -32,6 +32,11 @@ module QA
# Given *gitlab_address* = 'http://gitlab-abc123.test/' #=> http://about.gitlab-abc123.test/ # Given *gitlab_address* = 'http://gitlab-abc123.test/' #=> http://about.gitlab-abc123.test/
Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon Runtime::Scenario.define(:about_address, URI(-> { gitlab_address.host = "about.#{gitlab_address.host}"; gitlab_address }.call).to_s) # rubocop:disable Style/Semicolon
##
# Setup knapsack and download latest report
#
Tools::KnapsackReport.configure! if Runtime::Env.knapsack?
## ##
# Perform before hooks, which are different for CE and EE # Perform before hooks, which are different for CE and EE
# #
......
...@@ -19,13 +19,11 @@ module QA ...@@ -19,13 +19,11 @@ module QA
def paths_from_knapsack def paths_from_knapsack
allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
QA::Runtime::Logger.info '' QA::Runtime::Logger.info '==== Knapsack specs to execute ====='
QA::Runtime::Logger.info 'Report specs:' QA::Runtime::Logger.info 'Report specs:'
QA::Runtime::Logger.info allocator.report_node_tests.join(', ') QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
QA::Runtime::Logger.info ''
QA::Runtime::Logger.info 'Leftover specs:' QA::Runtime::Logger.info 'Leftover specs:'
QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ') QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
QA::Runtime::Logger.info ''
['--', allocator.node_tests] ['--', allocator.node_tests]
end end
......
...@@ -5,50 +5,81 @@ require "fog/google" ...@@ -5,50 +5,81 @@ require "fog/google"
module QA module QA
module Tools module Tools
class KnapsackReport class KnapsackReport
extend SingleForwardable
PROJECT = "gitlab-qa-resources" PROJECT = "gitlab-qa-resources"
BUCKET = "knapsack-reports" BUCKET = "knapsack-reports"
class << self def_delegators :new, :configure!, :move_regenerated_report, :download_report, :upload_report
def download
new.download_report
end
def upload(glob) # Configure knapsack report
new.upload_report(glob) #
end # * Setup variables
end # * Fetch latest report
#
# @return [void]
def configure!
ENV["KNAPSACK_TEST_FILE_PATTERN"] ||= "qa/specs/features/**/*_spec.rb"
ENV["KNAPSACK_REPORT_PATH"] = report_path
def initialize Knapsack.logger = QA::Runtime::Logger.logger
ENV["KNAPSACK_REPORT_PATH"] || raise("KNAPSACK_REPORT_PATH env var is required!")
ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("QA_KNAPSACK_REPORT_GCS_CREDENTIALS env var is required!") download_report
end end
# Download knapsack report from gcs bucket # Download knapsack report from gcs bucket
# #
# @return [void] # @return [void]
def download_report def download_report
logger.info("Downloading latest knapsack report '#{report_file}'") logger.debug("Downloading latest knapsack report for '#{report_name}' to '#{report_path}'")
file = client.get_object(BUCKET, report_file) file = client.get_object(BUCKET, report_file)
logger.info("Saving latest knapsack report to '#{report_path}'")
File.write(report_path, file[:body]) File.write(report_path, file[:body])
rescue StandardError => e
ENV["KNAPSACK_REPORT_PATH"] = "knapsack/master_report.json"
logger.warn("Failed to fetch latest knapsack report: #{e}")
logger.warn("Falling back to 'knapsack/master_report.json'")
end
# Rename and move new regenerated report to a separate folder used to indicate report name
#
# @return [void]
def move_regenerated_report
return unless ENV["KNAPSACK_GENERATE_REPORT"] == "true"
path = "tmp/knapsack/#{report_name}"
FileUtils.mkdir_p(path)
# Use path from knapsack config in case of fallback to master_report.json
FileUtils.cp(Knapsack.report.report_path, "#{path}/#{ENV['CI_NODE_INDEX']}.json")
end end
# Merge and upload knapsack report to gcs bucket # Merge and upload knapsack report to gcs bucket
# #
# Fetches all files defined in glob and uses parent folder as report name
#
# @param [String] glob # @param [String] glob
# @return [void] # @return [void]
def upload_report(glob) def upload_report(glob)
reports = Dir[glob] reports = Pathname.glob(glob).each_with_object(Hash.new { |hsh, key| hsh[key] = [] }) do |report, hash|
return logger.error("Pattern '#{glob}' did not match any files!") if reports.empty? next unless report.extname == ".json"
hash[report.parent.basename.to_s].push(report)
end
return logger.error("Glob '#{glob}' did not contain any valid report files!") if reports.empty?
reports.each do |name, jsons|
file = "#{name}.json"
report = reports report = jsons
.map { |path| JSON.parse(File.read(path)) } .map { |json| JSON.parse(File.read(json)) }
.reduce({}, :merge) .reduce({}, :merge)
return logger.error("Knapsack generated empty report, skipping upload!") if report.empty? next logger.warn("Knapsack generated empty report for '#{name}', skipping upload!") if report.empty?
logger.info("Uploading latest knapsack report '#{report_file}'") logger.info("Uploading latest knapsack report '#{file}'")
client.put_object(BUCKET, report_file, JSON.pretty_generate(report)) client.put_object(BUCKET, file, JSON.pretty_generate(report))
rescue StandardError => e
logger.error("Failed to upload knapsack report for '#{name}'. Error: #{e}")
end
end end
private private
...@@ -66,22 +97,48 @@ module QA ...@@ -66,22 +97,48 @@ module QA
def client def client
@client ||= Fog::Storage::Google.new( @client ||= Fog::Storage::Google.new(
google_project: PROJECT, google_project: PROJECT,
google_json_key_location: ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] google_json_key_location: gcs_credentials
) )
end end
# Base path of knapsack report
#
# @return [String]
def report_base_path
@report_base_path ||= "knapsack"
end
# Knapsack report path # Knapsack report path
# #
# @return [String] # @return [String]
def report_path def report_path
@report_path ||= ENV["KNAPSACK_REPORT_PATH"] @report_path ||= "#{report_base_path}/#{report_file}"
end end
# Knapsack report name # Knapsack report name
# #
# @return [String] # @return [String]
def report_file def report_file
@report_name ||= report_path.split("/").last @report_file ||= "#{report_name}.json"
end
# Report name
#
# Infer report name from ci job name
# Remove characters incompatible with gcs bucket naming from job names like ee:instance-parallel
#
# @return [String]
def report_name
@report_name ||= ENV["CI_JOB_NAME"].split(" ").first.tr(":", "-")
end
# Path to GCS credentials json
#
# @return [String]
def gcs_credentials
@gcs_credentials ||= ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise(
"QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable is required!"
)
end end
end end
end end
......
...@@ -173,31 +173,23 @@ RSpec.describe QA::Runtime::Env do ...@@ -173,31 +173,23 @@ RSpec.describe QA::Runtime::Env do
stub_env('CI_NODE_TOTAL', '2') stub_env('CI_NODE_TOTAL', '2')
end end
it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do it 'returns true if running in parallel CI run' do
stub_env('KNAPSACK_GENERATE_REPORT', 'true')
expect(described_class.knapsack?).to be_truthy expect(described_class.knapsack?).to be_truthy
end end
it 'returns true if KNAPSACK_REPORT_PATH is defined' do it 'returns false if knapsack disabled' do
stub_env('KNAPSACK_REPORT_PATH', '/a/path') stub_env('NO_KNAPSACK', 'true')
expect(described_class.knapsack?).to be_falsey
expect(described_class.knapsack?).to be_truthy
end end
it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do it 'returns false if not running in a parallel job' do
stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern') stub_env('CI_NODE_TOTAL', '1')
expect(described_class.knapsack?).to be_truthy
end
it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
expect(described_class.knapsack?).to be_falsey expect(described_class.knapsack?).to be_falsey
end end
it 'returns false if not running in parallel job' do it 'returns false if not running in ci' do
stub_env('CI_NODE_TOTAL', '1') stub_env('CI_NODE_TOTAL', nil)
stub_env('KNAPSACK_GENERATE_REPORT', 'true')
expect(described_class.knapsack?).to be_falsey expect(described_class.knapsack?).to be_falsey
end end
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
RSpec.describe QA::Scenario::Test::Integration::Github do RSpec.describe QA::Scenario::Test::Integration::Github do
describe '#perform' do describe '#perform' do
let(:env) { spy('Runtime::Env') } let(:env) { spy('Runtime::Env', knapsack?: false) }
before do before do
stub_const('QA::Runtime::Env', env) stub_const('QA::Runtime::Env', env)
......
...@@ -10,7 +10,7 @@ require 'active_support/core_ext/object/blank' ...@@ -10,7 +10,7 @@ require 'active_support/core_ext/object/blank'
require_relative 'qa_deprecation_toolkit_env' require_relative 'qa_deprecation_toolkit_env'
QaDeprecationToolkitEnv.configure! QaDeprecationToolkitEnv.configure!
Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK'] Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack?
QA::Runtime::Browser.configure! QA::Runtime::Browser.configure!
QA::Runtime::AllureReport.configure! QA::Runtime::AllureReport.configure!
...@@ -65,10 +65,10 @@ RSpec.configure do |config| ...@@ -65,10 +65,10 @@ RSpec.configure do |config|
end end
config.after(:suite) do |suite| config.after(:suite) do |suite|
# If any tests failed, leave the resources behind to help troubleshoot QA::Tools::KnapsackReport.move_regenerated_report if QA::Runtime::Env.knapsack?
next if suite.reporter.failed_examples.present?
QA::Resource::ReusableProject.remove_all_via_api! # If any tests failed, leave the resources behind to help troubleshoot
QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present?
end end
config.expect_with :rspec do |expectations| config.expect_with :rspec do |expectations|
......
...@@ -10,8 +10,8 @@ namespace :knapsack do ...@@ -10,8 +10,8 @@ namespace :knapsack do
end end
desc "Merge and upload knapsack report" desc "Merge and upload knapsack report"
task :upload, [:glob_pattern] do |_task, args| task :upload, [:glob] do |_task, args|
QA::Tools::KnapsackReport.upload(args[:glob_pattern]) QA::Tools::KnapsackReport.upload_report(args[:glob])
end end
end end
# rubocop:enable Rails/RakeEnvironment # rubocop:enable Rails/RakeEnvironment
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment