Commit 434ca1bd authored by Sean Arnold's avatar Sean Arnold Committed by Fabio Pitino

Track scoped_variables duration, use builder

- Use Variables builder to build scoped_variables.
- Migrate predefined_variables to builder

Changed: performance
parent a03f6bef
......@@ -780,6 +780,10 @@ module Ci
strong_memoize(:legacy_trigger) { trigger_requests.first }
end
def variables_builder
@variables_builder ||= ::Gitlab::Ci::Variables::Builder.new(self)
end
def persisted_variables
Gitlab::Ci::Variables::Collection.new.tap do |variables|
break variables unless persisted?
......@@ -1254,6 +1258,12 @@ module Ci
self.builds.latest.build_matchers(project)
end
def predefined_vars_in_builder_enabled?
strong_memoize(:predefined_vars_in_builder_enabled) do
Feature.enabled?(:ci_predefined_vars_in_builder, project, default_enabled: :yaml)
end
end
private
def add_message(severity, content)
......
......@@ -10,8 +10,10 @@ module Ci
# Variables in the environment name scope.
#
def scoped_variables(environment: expanded_environment_name, dependencies: true)
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.concat(predefined_variables)
track_duration do
variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies)
variables.concat(predefined_variables) unless pipeline.predefined_vars_in_builder_enabled?
variables.concat(project.predefined_variables)
variables.concat(pipeline.predefined_variables)
variables.concat(runner.predefined_variables) if runnable? && runner
......@@ -25,9 +27,23 @@ module Ci
variables.concat(trigger_request.user_variables) if trigger_request
variables.concat(pipeline.variables)
variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule
variables
end
end
def track_duration
start_time = ::Gitlab::Metrics::System.monotonic_time
result = yield
duration = ::Gitlab::Metrics::System.monotonic_time - start_time
::Gitlab::Ci::Pipeline::Metrics
.pipeline_builder_scoped_variables_histogram
.observe({}, duration.seconds)
result
end
##
# Variables that do not depend on the environment name.
#
......
---
name: ci_predefined_vars_in_builder
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72348
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/231300
milestone: '14.4'
type: development
group: group::pipeline authoring
default_enabled: false
......@@ -41,6 +41,7 @@ The following metrics are available:
| `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` |
| `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | |
| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation` |
| `gitlab_ci_pipeline_builder_scoped_variables_duration` | Histogram | 14.5 | Time in seconds it takes to create the scoped variables for a CI/CD job
| `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | |
| `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` |
| `job_waiter_started_total` | Counter | 12.9 | Number of batches of jobs started where a web request is waiting for the jobs to complete | `worker` |
......
......@@ -51,6 +51,15 @@ module Gitlab
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
def self.pipeline_builder_scoped_variables_histogram
name = :gitlab_ci_pipeline_builder_scoped_variables_duration
comment = 'Pipeline variables builder scoped_variables duration'
labels = {}
buckets = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5, 10, 30, 60, 120]
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
def self.pipeline_processing_events_counter
name = :gitlab_ci_pipeline_processing_events_total
comment = 'Total amount of pipeline processing events'
......
# frozen_string_literal: true
module Gitlab
module Ci
module Variables
class Builder
include ::Gitlab::Utils::StrongMemoize
def initialize(pipeline)
@pipeline = pipeline
end
def scoped_variables(job, environment:, dependencies:)
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled?
end
end
private
attr_reader :pipeline
def predefined_variables(job)
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.append(key: 'CI_JOB_NAME', value: job.name)
variables.append(key: 'CI_JOB_STAGE', value: job.stage)
variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action?
variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request
variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance)
variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s)
# legacy variables
variables.append(key: 'CI_BUILD_NAME', value: job.name)
variables.append(key: 'CI_BUILD_STAGE', value: job.stage)
variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request
variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action?
end
end
def ci_node_total_value(job)
parallel = job.options&.dig(:parallel)
parallel = parallel.dig(:total) if parallel.is_a?(Hash)
parallel || 1
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Variables::Builder do
let(:builder) { described_class.new(pipeline) }
let(:pipeline) { create(:ci_pipeline) }
let(:job) { create(:ci_build, pipeline: pipeline) }
describe '#scoped_variables' do
let(:environment) { job.expanded_environment_name }
let(:dependencies) { true }
subject { builder.scoped_variables(job, environment: environment, dependencies: dependencies) }
it 'returns the expected variables' do
keys = %w[CI_JOB_NAME
CI_JOB_STAGE
CI_NODE_TOTAL
CI_BUILD_NAME
CI_BUILD_STAGE]
subject.map { |env| env[:key] }.tap do |names|
expect(names).to include(*keys)
end
end
context 'feature flag disabled' do
before do
stub_feature_flags(ci_predefined_vars_in_builder: false)
end
it 'returns no variables' do
expect(subject.map { |env| env[:key] }).to be_empty
end
end
end
end
......@@ -2759,7 +2759,10 @@ RSpec.describe Ci::Build do
let(:job_dependency_var) { { key: 'job_dependency', value: 'value', public: true, masked: false } }
before do
allow(build).to receive(:predefined_variables) { [build_pre_var] }
allow_next_instance_of(Gitlab::Ci::Variables::Builder) do |builder|
allow(builder).to receive(:predefined_variables) { [build_pre_var] }
end
allow(build).to receive(:yaml_variables) { [build_yaml_var] }
allow(build).to receive(:persisted_variables) { [] }
allow(build).to receive(:job_jwt_variables) { [job_jwt_var] }
......@@ -3411,75 +3414,122 @@ RSpec.describe Ci::Build do
end
describe '#scoped_variables' do
context 'when build has not been persisted yet' do
let(:build) do
described_class.new(
name: 'rspec',
stage: 'test',
ref: 'feature',
project: project,
pipeline: pipeline,
scheduling_type: :stage
)
end
before do
pipeline.clear_memoization(:predefined_vars_in_builder_enabled)
end
let(:pipeline) { create(:ci_pipeline, project: project, ref: 'feature') }
it 'records a prometheus metric' do
histogram = double(:histogram)
expect(::Gitlab::Ci::Pipeline::Metrics).to receive(:pipeline_builder_scoped_variables_histogram)
.and_return(histogram)
it 'does not persist the build' do
expect(build).to be_valid
expect(build).not_to be_persisted
expect(histogram).to receive(:observe)
.with({}, a_kind_of(ActiveSupport::Duration))
build.scoped_variables
build.scoped_variables
end
expect(build).not_to be_persisted
end
shared_examples 'calculates scoped_variables' do
context 'when build has not been persisted yet' do
let(:build) do
described_class.new(
name: 'rspec',
stage: 'test',
ref: 'feature',
project: project,
pipeline: pipeline,
scheduling_type: :stage
)
end
it 'returns static predefined variables' do
keys = %w[CI_JOB_NAME
CI_COMMIT_SHA
CI_COMMIT_SHORT_SHA
CI_COMMIT_REF_NAME
CI_COMMIT_REF_SLUG
CI_JOB_STAGE]
let(:pipeline) { create(:ci_pipeline, project: project, ref: 'feature') }
variables = build.scoped_variables
it 'does not persist the build' do
expect(build).to be_valid
expect(build).not_to be_persisted
variables.map { |env| env[:key] }.tap do |names|
expect(names).to include(*keys)
build.scoped_variables
expect(build).not_to be_persisted
end
expect(variables)
.to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true, masked: false)
it 'returns static predefined variables' do
keys = %w[CI_JOB_NAME
CI_COMMIT_SHA
CI_COMMIT_SHORT_SHA
CI_COMMIT_REF_NAME
CI_COMMIT_REF_SLUG
CI_JOB_STAGE]
variables = build.scoped_variables
variables.map { |env| env[:key] }.tap do |names|
expect(names).to include(*keys)
end
expect(variables)
.to include(key: 'CI_COMMIT_REF_NAME', value: 'feature', public: true, masked: false)
end
it 'does not return prohibited variables' do
keys = %w[CI_JOB_ID
CI_JOB_URL
CI_JOB_TOKEN
CI_BUILD_ID
CI_BUILD_TOKEN
CI_REGISTRY_USER
CI_REGISTRY_PASSWORD
CI_REPOSITORY_URL
CI_ENVIRONMENT_URL
CI_DEPLOY_USER
CI_DEPLOY_PASSWORD]
build.scoped_variables.map { |env| env[:key] }.tap do |names|
expect(names).not_to include(*keys)
end
end
end
it 'does not return prohibited variables' do
keys = %w[CI_JOB_ID
CI_JOB_URL
CI_JOB_TOKEN
CI_BUILD_ID
CI_BUILD_TOKEN
CI_REGISTRY_USER
CI_REGISTRY_PASSWORD
CI_REPOSITORY_URL
CI_ENVIRONMENT_URL
CI_DEPLOY_USER
CI_DEPLOY_PASSWORD]
context 'with dependency variables' do
let!(:prepare) { create(:ci_build, name: 'prepare', pipeline: pipeline, stage_idx: 0) }
let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['prepare'] }) }
build.scoped_variables.map { |env| env[:key] }.tap do |names|
expect(names).not_to include(*keys)
let!(:job_variable) { create(:ci_job_variable, :dotenv_source, job: prepare) }
it 'inherits dependent variables' do
expect(build.scoped_variables.to_hash).to include(job_variable.key => job_variable.value)
end
end
end
context 'with dependency variables' do
let!(:prepare) { create(:ci_build, name: 'prepare', pipeline: pipeline, stage_idx: 0) }
let!(:build) { create(:ci_build, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['prepare'] }) }
it_behaves_like 'calculates scoped_variables'
let!(:job_variable) { create(:ci_job_variable, :dotenv_source, job: prepare) }
it 'delegates to the variable builders' do
expect_next_instance_of(Gitlab::Ci::Variables::Builder) do |builder|
expect(builder)
.to receive(:scoped_variables).with(build, hash_including(:environment, :dependencies))
.and_call_original
it 'inherits dependent variables' do
expect(build.scoped_variables.to_hash).to include(job_variable.key => job_variable.value)
expect(builder).to receive(:predefined_variables).and_call_original
end
build.scoped_variables
end
context 'when ci builder feature flag is disabled' do
before do
stub_feature_flags(ci_predefined_vars_in_builder: false)
end
it 'does not delegate to the variable builders' do
expect_next_instance_of(Gitlab::Ci::Variables::Builder) do |builder|
expect(builder).not_to receive(:predefined_variables)
end
build.scoped_variables
end
it_behaves_like 'calculates scoped_variables'
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment