Commit e9122bfd authored by Matija Čupić's avatar Matija Čupić

Add ci_template known events rake task

Adds a rake task that generates ci_templates.yml. This needs to be ran
whenever a new CI template gets added to make sure an event is defined
for the template inclusion.
parent 258d2f2e
......@@ -3,6 +3,7 @@
module Gitlab::UsageDataCounters
class CiTemplateUniqueCounter
REDIS_SLOT = 'ci_templates'
KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__)
# NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_`
TEMPLATE_TO_EVENT = {
......@@ -20,31 +21,26 @@ module Gitlab::UsageDataCounters
class << self
def track_unique_project_event(project_id:, template:, config_source:)
if event = unique_project_event(template, config_source)
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
end
end
private
return unless TEMPLATE_TO_EVENT[template] || Feature.enabled?(:track_all_ci_template_inclusions, default_enabled: :yaml)
def unique_project_event(template, config_source)
if TEMPLATE_TO_EVENT[template]
template_inclusion_name(config_source, TEMPLATE_TO_EVENT[template])
end
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(ci_template_event_name(template, config_source), values: project_id)
end
if Feature.enabled?(:track_all_ci_template_inclusions)
template_inclusion_name(config_source, template_to_event(template))
end
def ci_templates(relative_base = 'lib/gitlab/ci/templates')
Dir.glob('**/*.gitlab-ci.yml', base: Rails.root.join(relative_base))
end
def template_inclusion_name(config_source, name)
def ci_template_event_name(template_name, config_source)
prefix = 'implicit_' if config_source.to_s == 'auto_devops_source'
template_event_name = TEMPLATE_TO_EVENT[template_name] || template_to_event_name(template_name)
"p_#{REDIS_SLOT}_#{prefix}#{name}"
"p_#{REDIS_SLOT}_#{prefix}#{template_event_name}"
end
def template_to_event(template)
File.basename(template, '.gitlab-ci.yml').underscore
private
def template_to_event_name(template)
ActiveSupport::Inflector.parameterize(template.chomp('.gitlab-ci.yml'), separator: '_').underscore
end
end
end
......
......@@ -4,73 +4,586 @@
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops_build
# Explicit include:template pipeline events
- name: p_ci_templates_5_min_production_app
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops_deploy
- name: p_ci_templates_aws_cf_deploy_ec2
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_sast
- name: p_ci_templates_auto_devops_build
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_secret_detection
- name: p_ci_templates_auto_devops_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
# Explicit include:template pipeline events
- name: p_ci_templates_5_min_production_app
- name: p_ci_templates_auto_devops_deploy_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
# This part of the file is generated automatically by
# bin/rake gitlab:usage_data:generate_ci_template_events
#
# Do not edit it manually!
#
# The section above this should be removed once we roll out tracking all ci
# templates
# https://gitlab.com/gitlab-org/gitlab/-/issues/339684
- name: p_ci_templates_terraform_base_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_base
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dotnet
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_nodejs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_openshift
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_auto_devops
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_cf_deploy_ec2
- name: p_ci_templates_bash
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_deploy_ecs
- name: p_ci_templates_rust
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_auto_devops_build
- name: p_ci_templates_elixir
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_auto_devops_deploy
- name: p_ci_templates_clojure
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_auto_devops_deploy_latest
- name: p_ci_templates_crystal
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_getting_started
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_code_quality
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_load_performance_testing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_accessibility
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_failfast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_browser_performance
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_verify_browser_performance_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_grails
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_runner_validation
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_on_demand_scan
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_base_latest
- name: p_ci_templates_security_license_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_coverage_fuzzing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_api_fuzzing_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_secure_binaries
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_api
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_container_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dependency_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_api_fuzzing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_dast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_security_cluster_image_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_ios_fastlane
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_composer
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_c
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_python
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_android_fastlane
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_django
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_maven
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_flutter
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_workflows_branch_pipelines
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_workflows_mergerequest_pipelines
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_laravel
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_managed_cluster_applications
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_php
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_packer
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_mono
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_serverless
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_go
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_scala
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_latex
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_android
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_indeni_cloudrail
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_deploy_ecs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_cf_provision_and_deploy_ec2
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_aws_deploy_ecs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_gradle
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_chef
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_dast_default_branch_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_load_performance_testing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_helm_2to3
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_code_intelligence
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_code_quality
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_ecs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_ec2
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_build
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_browser_performance_testing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_test
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_deploy_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_browser_performance_testing_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_cf_provision
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_jobs_build_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_terraform_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_swift
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jekyll
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_harp
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_octopress
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_brunch
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_doxygen
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hyde
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_lektor
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jbake
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hexo
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_middleman
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_hugo
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_pelican
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_nanoc
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_swaggerui
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_jigsaw
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_metalsmith
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_gatsby
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_pages_html
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dart
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_docker
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_julia
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_npm
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_dotnet_core
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_5_minute_production_app
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_ruby
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_load_performance_testing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_helm_2to3
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_code_intelligence
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_code_quality
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_deploy_ecs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_deploy_ec2
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops_build
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_browser_performance_testing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_test
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_auto_devops_deploy_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_browser_performance_testing_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_cf_provision
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_jobs_build_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast_runner_validation
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast_on_demand_scan
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_license_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_coverage_fuzzing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_api_fuzzing_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_secure_binaries
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast_api
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_container_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dependency_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_api_fuzzing
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_dast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- name: p_ci_templates_implicit_security_cluster_image_scanning
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
......@@ -28,5 +28,33 @@ namespace :gitlab do
task generate_from_yaml: :environment do
puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data)
end
desc 'GitLab | UsageDataMetrics | Generate known_events/ci_templates.yml based on template definitions'
task generate_ci_template_events: :environment do
banner = <<~BANNER
# This file is generated automatically by
# bin/rake gitlab:usage_data:generate_ci_template_events
#
# Do not edit it manually!
BANNER
repository_includes = ci_template_includes_hash(:repository_source)
auto_devops_jobs_includes = ci_template_includes_hash(:auto_devops_source, 'Jobs')
auto_devops_security_includes = ci_template_includes_hash(:auto_devops_source, 'Security')
all_includes = [*repository_includes, *auto_devops_jobs_includes, *auto_devops_security_includes]
File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, ''))
end
def ci_template_includes_hash(source, template_directory = nil)
Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template|
{
'name' => Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name("#{template_directory}/#{template}", source),
'category' => 'ci_templates',
'redis_slot' => Gitlab::UsageDataCounters::CiTemplateUniqueCounter::REDIS_SLOT,
'aggregation' => 'weekly'
}
end
end
end
end
......@@ -77,12 +77,16 @@ RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do
let(:project_id) { 1 }
let(:config_source) { :repository_source }
Dir.glob('**/*.gitlab-ci.yml', base: Rails.root.join('lib/gitlab/ci/templates')) do |template|
described_class.ci_templates.each do |template|
next if described_class::TEMPLATE_TO_EVENT.key?(template)
it 'has an event defined' do
it "has an event defined for #{template}" do
expect do
described_class.track_unique_project_event(project_id: project_id, template: described_class.send(:template_to_event, template), config_source: config_source)
described_class.track_unique_project_event(
project_id: project_id,
template: template,
config_source: config_source
)
end.not_to raise_error
end
......@@ -104,7 +108,8 @@ RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do
end
it "tracks #{template}" do
expect(Gitlab::UsageDataCounters::HLLRedisCounter).to(receive(:track_event)).with("p_ci_templates_#{File.basename(template, '.gitlab-ci.yml').underscore}", values: project_id)
expected_template_event_name = described_class.ci_template_event_name(template, :repository_source)
expect(Gitlab::UsageDataCounters::HLLRedisCounter).to(receive(:track_event)).with(expected_template_event_name, values: project_id)
described_class.track_unique_project_event(project_id: project_id, template: template, config_source: config_source)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment