Commit 8f0ec8d2 authored by Andrejs Cunskis's avatar Andrejs Cunskis

Merge branch 'acunskis-spec-runtimes' into 'master'

Add task to report on long running spec files

See merge request gitlab-org/gitlab!78221
parents d4537b39 8b944797
# frozen_string_literal: true # frozen_string_literal: true
# rubocop:disable Rails/RakeEnvironment # rubocop:disable Rails/RakeEnvironment
Dir['tasks/*.rake'].each { |file| load file } require_relative "qa"
require_relative 'qa/tools/revoke_all_personal_access_tokens' Dir['tasks/*.rake'].each { |file| load file }
require_relative 'qa/tools/delete_subgroups'
require_relative 'qa/tools/generate_perf_testdata'
require_relative 'qa/tools/delete_test_ssh_keys'
require_relative 'qa/tools/initialize_gitlab_auth'
require_relative 'qa/tools/delete_projects'
desc "Revokes all personal access tokens" desc "Revokes all personal access tokens"
task :revoke_personal_access_tokens do task :revoke_personal_access_tokens do
......
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../qa'
# This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # This script deletes all projects directly under a group specified by ENV['TOP_LEVEL_GROUP_NAME']
# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')
......
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../qa'
# This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME'] # This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME']
# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group') # Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')
......
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../qa'
# This script deletes all selected test ssh keys for a specific user # This script deletes all selected test ssh keys for a specific user
# Keys can be selected by a string matching part of the key's title and by created date # Keys can be selected by a string matching part of the key's title and by created date
# - Specify `title_portion` to delete only keys that include the string provided # - Specify `title_portion` to delete only keys that include the string provided
......
# frozen_string_literal: true # frozen_string_literal: true
require 'yaml' require 'yaml'
require_relative '../../qa'
# This script generates testdata for Performance Testing. # This script generates testdata for Performance Testing.
# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS # Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
# This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing # This job creates a urls.txt which contains a hash of all the URLs needed for Performance Testing
......
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../qa'
module QA module QA
module Tools module Tools
# Task to set default password from Runtime::Env.default_password if not set already # Task to set default password from Runtime::Env.default_password if not set already
......
# frozen_string_literal: true
require "fog/google"
require "slack-notifier"
module QA
module Tools
class LongRunningSpecReporter
extend SingleForwardable
SLACK_CHANNEL = "#quality-reports"
PROJECT = "gitlab-qa-resources"
BUCKET = "knapsack-reports"
REPORT_NAME = "ee-instance-parallel.json"
RUNTIME_THRESHOLD = 300
def_delegator :new, :execute
# Find and report specs exceeding runtime threshold
#
# @return [void]
def execute
return puts("No long running specs detected, all good!") if long_running_specs.empty?
specs = long_running_specs.map { |k, v| "#{k}: #{(v / 60).round(2)} minutes" }.join("\n")
average = mean_runtime < 60 ? "#{mean_runtime.round(0)} seconds" : "#{(mean_runtime / 60).round(2)} minutes"
msg = <<~MSG
Following spec files are exceeding #{RUNTIME_THRESHOLD / 60} minute runtime threshold!
Current average spec runtime: #{average}.
MSG
puts("#{msg}\n#{specs}")
notifier.post(icon_emoji: ":time-out:", text: "#{msg}\n```#{specs}```")
end
private
# Average runtime of spec files
#
# @return [Number]
def mean_runtime
@mean_runtime ||= latest_report.values
.select { |v| v < RUNTIME_THRESHOLD }
.yield_self { |runtimes| runtimes.sum(0.0) / runtimes.length }
end
# Spec files exceeding runtime threshold
#
# @return [Hash]
def long_running_specs
@long_running_specs ||= latest_report.select { |k, v| v > RUNTIME_THRESHOLD }
end
# Latest knapsack report
#
# @return [Hash]
def latest_report
@latest_report ||= JSON.parse(client.get_object(BUCKET, REPORT_NAME)[:body])
end
# Slack notifier
#
# @return [Slack::Notifier]
def notifier
@notifier ||= Slack::Notifier.new(
slack_webhook_url,
channel: SLACK_CHANNEL,
username: "Spec Runtime Report"
)
end
# GCS client
#
# @return [Fog::Storage::GoogleJSON]
def client
@client ||= Fog::Storage::Google.new(
google_project: PROJECT,
**(File.exist?(gcs_json) ? { google_json_key_location: gcs_json } : { google_json_key_string: gcs_json })
)
end
# Slack webhook url
#
# @return [String]
def slack_webhook_url
@slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable")
end
# GCS credentials json
#
# @return [Hash]
def gcs_json
ENV["QA_KNAPSACK_REPORT_GCS_CREDENTIALS"] || raise("Missing QA_KNAPSACK_REPORT_GCS_CREDENTIALS env variable!")
end
end
end
end
# frozen_string_literal: true # frozen_string_literal: true
require_relative "../../qa"
require "influxdb-client" require "influxdb-client"
require "terminal-table" require "terminal-table"
require "slack-notifier" require "slack-notifier"
......
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../qa'
require 'net/protocol' require 'net/protocol'
# This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS # This script revokes all personal access tokens with the name of 'api-test-token' on the host specified by GITLAB_ADDRESS
# Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS # Required environment variables: GITLAB_USERNAME, GITLAB_PASSWORD and GITLAB_ADDRESS
# Run `rake revoke_personal_access_tokens` # Run `rake revoke_personal_access_tokens`
......
# frozen_string_literal: true
RSpec.describe QA::Tools::LongRunningSpecReporter do
include QA::Support::Helpers::StubEnv
subject(:reporter) { described_class.execute }
let(:gcs_client) { double("Fog::Storage::GoogleJSON", get_object: report) }
let(:slack_notifier) { double("Slack::Notifier", post: nil) }
before do
stub_env("SLACK_WEBHOOK", "slack_url")
stub_env("QA_KNAPSACK_REPORT_GCS_CREDENTIALS", "gcs_json")
allow(Fog::Storage::Google).to receive(:new)
.with(google_project: "gitlab-qa-resources", google_json_key_string: "gcs_json")
.and_return(gcs_client)
allow(Slack::Notifier).to receive(:new)
.with("slack_url", channel: "#quality-reports", username: "Spec Runtime Report")
.and_return(slack_notifier)
end
context "without specs exceeding runtime" do
let(:report) do
{
body: <<~JSON
{
"spec.rb": 5,
"spec_2.rb": 10
}
JSON
}
end
it "returns all good message" do
expect { reporter }.to output("No long running specs detected, all good!\n").to_stdout
end
end
context "with specs exceeding runtime" do
let(:report) do
{
body: <<~JSON
{
"spec.rb": 5.0,
"spec_2.rb": 320.0
}
JSON
}
end
let(:spec) { "spec_2.rb: 5.33 minutes" }
let(:message) do
<<~MSG
Following spec files are exceeding 5 minute runtime threshold!
Current average spec runtime: 5 seconds.
MSG
end
it "notifies on long running specs" do
expect { reporter }.to output("#{message}\n#{spec}\n").to_stdout
expect(slack_notifier).to have_received(:post).with(
icon_emoji: ":time-out:",
text: "#{message}\n```#{spec}```"
)
end
end
end
# frozen_string_literal: true # frozen_string_literal: true
# rubocop:disable Rails/RakeEnvironment # rubocop:disable Rails/RakeEnvironment
require_relative "../qa/tools/knapsack_report"
namespace :knapsack do namespace :knapsack do
desc "Download latest knapsack report" desc "Download latest knapsack report"
task :download do task :download do
...@@ -13,5 +11,10 @@ namespace :knapsack do ...@@ -13,5 +11,10 @@ namespace :knapsack do
task :upload, [:glob] do |_task, args| task :upload, [:glob] do |_task, args|
QA::Tools::KnapsackReport.upload_report(args[:glob]) QA::Tools::KnapsackReport.upload_report(args[:glob])
end end
desc "Report long running spec files"
task :notify_long_running_specs do
QA::Tools::LongRunningSpecReporter.execute
end
end end
# rubocop:enable Rails/RakeEnvironment # rubocop:enable Rails/RakeEnvironment
# frozen_string_literal: true # frozen_string_literal: true
# rubocop:disable Rails/RakeEnvironment # rubocop:disable Rails/RakeEnvironment
require_relative "../qa/tools/reliable_report"
desc "Fetch reliable and unreliable spec data and create report" desc "Fetch reliable and unreliable spec data and create report"
task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args| task :reliable_spec_report, [:range, :report_in_issue_and_slack] do |_task, args|
QA::Tools::ReliableReport.run(**args) QA::Tools::ReliableReport.run(**args)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment