Commit 818eed1b authored by Fabio Pitino's avatar Fabio Pitino

Merge branch 'ci-remove-no-matching-runner-failure-action' into 'master'

Drop builds with `ci_quota_exceeded` failure reason

See merge request gitlab-org/gitlab!63542
parents a4815394 51b849a2
......@@ -24,7 +24,7 @@ module Enums
project_deleted: 15,
ci_quota_exceeded: 16,
pipeline_loop_detected: 17,
no_matching_runner: 18,
no_matching_runner: 18, # not used anymore, but cannot be deleted because of old data
insufficient_bridge_permissions: 1_001,
downstream_bridge_project_not_found: 1_002,
invalid_bridge_trigger: 1_003,
......
......@@ -10,9 +10,10 @@ module Ci
end
def execute
DropNotRunnableBuildsService.new(pipeline).execute
Ci::ProcessPipelineService.new(pipeline).execute
end
end
end
end
::Ci::PipelineCreation::StartPipelineService.prepend_mod_with('Ci::PipelineCreation::StartPipelineService')
......@@ -3,7 +3,7 @@
module Ci
module PipelineCreation
class DropNotRunnableBuildsService
include Gitlab::Utils::StrongMemoize
include ::Gitlab::Utils::StrongMemoize
def initialize(pipeline)
@pipeline = pipeline
......@@ -16,40 +16,58 @@ module Ci
def execute
return unless ::Feature.enabled?(:ci_drop_new_builds_when_ci_quota_exceeded, project, default_enabled: :yaml)
return unless pipeline.created?
return unless project.shared_runners_enabled?
return unless project.ci_minutes_quota.minutes_used_up?
load_runners
validate_build_matchers
end
private
attr_reader :pipeline
attr_reader :instance_runners, :private_runners
delegate :project, to: :pipeline
def load_runners
@instance_runners, @private_runners = project
.all_runners
.active
.online
.runner_matchers
.partition(&:instance_type?)
def validate_build_matchers
build_ids = pipeline
.build_matchers
.filter_map { |matcher| matcher.build_ids if should_drop?(matcher) }
.flatten
drop_all_builds(build_ids, :ci_quota_exceeded)
end
def validate_build_matchers
pipeline.build_matchers.each do |build_matcher|
failure_reason = validate_build_matcher(build_matcher)
next unless failure_reason
def should_drop?(build_matcher)
matches_instance_runners_and_quota_used_up?(build_matcher) &&
!matches_private_runners?(build_matcher)
end
drop_all_builds(build_matcher.build_ids, failure_reason)
def matches_instance_runners_and_quota_used_up?(build_matcher)
instance_runners.any? do |matcher|
matcher.matches?(build_matcher) &&
!matcher.matches_quota?(build_matcher)
end
end
def validate_build_matcher(build_matcher)
return if matching_private_runners?(build_matcher)
return if matching_instance_runners?(build_matcher)
def matches_private_runners?(build_matcher)
private_runners.any? { |matcher| matcher.matches?(build_matcher) }
end
def instance_runners
strong_memoize(:instance_runners) do
runner_matchers.select(&:instance_type?)
end
end
def private_runners
strong_memoize(:private_runners) do
runner_matchers.reject(&:instance_type?)
end
end
matching_failure_reason(build_matcher)
def runner_matchers
strong_memoize(:runner_matchers) do
project.all_runners.active.online.runner_matchers
end
end
##
......@@ -58,34 +76,12 @@ module Ci
# transition to other states by `PipelineProcessWorker` running async.
#
def drop_all_builds(build_ids, failure_reason)
return if build_ids.empty?
pipeline.builds.id_in(build_ids).each do |build|
build.drop(failure_reason, skip_pipeline_processing: true)
build.drop!(failure_reason, skip_pipeline_processing: true)
end
end
def matching_private_runners?(build_matcher)
private_runners
.find { |matcher| matcher.matches?(build_matcher) }
.present?
end
def matching_instance_runners?(build_matcher)
instance_runners
.find { |matcher| matching_criteria(matcher, build_matcher) }
.present?
end
# Overridden in EE
def matching_criteria(runner_matcher, build_matcher)
runner_matcher.matches?(build_matcher)
end
# Overridden in EE
def matching_failure_reason(build_matcher)
:no_matching_runner
end
end
end
end
Ci::PipelineCreation::DropNotRunnableBuildsService.prepend_mod_with('Ci::PipelineCreation::DropNotRunnableBuildsService')
# frozen_string_literal: true
module EE
module Ci
module PipelineCreation
module DropNotRunnableBuildsService
extend ::Gitlab::Utils::Override
private
override :matching_criteria
def matching_criteria(runner_matcher, build_matcher)
super && runner_matcher.matches_quota?(build_matcher)
end
override :matching_failure_reason
def matching_failure_reason(build_matcher)
if build_matcher.project.shared_runners_enabled_but_unavailable?
:ci_quota_exceeded
else
:no_matching_runner
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Ci
module PipelineCreation
module StartPipelineService
extend ::Gitlab::Utils::Override
override :execute
def execute
::Ci::PipelineCreation::DropNotRunnableBuildsService.new(pipeline).execute
super
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::CreatePipelineService, :sidekiq_inline do
let_it_be(:namespace) { create(:namespace, :with_used_build_minutes_limit) }
let_it_be(:project) { create(:project, :repository, namespace: namespace) }
let_it_be(:user) { project.owner }
let_it_be(:instance_runner) { create(:ci_runner, :instance, :online) }
let(:service) do
described_class.new(project, user, { ref: 'refs/heads/master' })
end
let(:config) do
<<~EOY
job1:
stage: build
script:
- echo "deploy runner 123"
job2:
stage: test
script:
- echo "run on runner 123"
tags:
- "123"
EOY
end
before do
project.add_developer(user)
stub_ci_pipeline_yaml_file(config)
end
it 'drops builds that match shared runners' do
pipeline = create_pipeline!
job1 = pipeline.builds.find_by_name('job1')
job2 = pipeline.builds.find_by_name('job2')
expect(job1).to be_failed
expect(job1.failure_reason).to eq('ci_quota_exceeded')
expect(job2).not_to be_failed
end
context 'with private runners' do
let_it_be(:private_runner) do
create(:ci_runner, :project, :online, projects: [project])
end
it 'does not drop the builds' do
pipeline = create_pipeline!
job1 = pipeline.builds.find_by_name('job1')
job2 = pipeline.builds.find_by_name('job2')
expect(job1).not_to be_failed
expect(job2).not_to be_failed
end
end
def create_pipeline!
service.execute(:push)
end
end
......@@ -3,14 +3,21 @@
require 'spec_helper'
RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be_with_reload(:pipeline) do
create(:ci_pipeline, status: :created)
create(:ci_pipeline, project: project, status: :created)
end
let_it_be_with_reload(:job) do
create(:ci_build, project: pipeline.project, pipeline: pipeline)
end
let_it_be_with_reload(:job_with_tags) do
create(:ci_build, :tags, project: pipeline.project, pipeline: pipeline)
end
let_it_be(:instance_runner) do
create(:ci_runner,
:online,
......@@ -22,19 +29,28 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
describe '#execute' do
subject(:execute) { described_class.new(pipeline).execute }
shared_examples 'available CI quota' do
shared_examples 'jobs allowed to run' do
it 'does not drop the jobs' do
expect { execute }.not_to change { job.reload.status }
expect { execute }
.to not_change { job.reload.status }
.and not_change { job_with_tags.reload.status }
end
end
shared_examples 'limit exceeded' do
before do
allow(pipeline.project).to receive(:ci_minutes_quota)
.and_return(double('quota', minutes_used_up?: true))
end
it 'drops the job with ci_quota_exceeded reason' do
execute
job.reload
[job, job_with_tags].each(&:reload)
expect(job).to be_failed
expect(job.failure_reason).to eq('ci_quota_exceeded')
expect(job_with_tags).to be_pending
end
context 'when shared runners are disabled' do
......@@ -42,13 +58,39 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
pipeline.project.update!(shared_runners_enabled: false)
end
it 'drops the job with no_matching_runner reason' do
execute
job.reload
it_behaves_like 'jobs allowed to run'
end
context 'with project runners' do
let_it_be(:project_runner) do
create(:ci_runner, :online, runner_type: :project_type, projects: [project])
end
it_behaves_like 'jobs allowed to run'
end
context 'with group runners' do
let_it_be(:group_runner) do
create(:ci_runner, :online, runner_type: :group_type, groups: [group])
end
it_behaves_like 'jobs allowed to run'
end
context 'when the feature flag is disabled' do
before do
stub_feature_flags(ci_drop_new_builds_when_ci_quota_exceeded: false)
end
it_behaves_like 'jobs allowed to run'
end
expect(job).to be_failed
expect(job.failure_reason).to eq('no_matching_runner')
context 'when the pipeline status is running' do
before do
pipeline.update!(status: :running)
end
it_behaves_like 'jobs allowed to run'
end
end
......@@ -57,7 +99,7 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::PUBLIC)
end
it_behaves_like 'available CI quota'
it_behaves_like 'jobs allowed to run'
context 'when the CI quota is exceeded' do
before do
......@@ -65,9 +107,7 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
.and_return(double('quota', minutes_used_up?: true))
end
it 'does not drop the jobs' do
expect { execute }.not_to change { job.reload.status }
end
it_behaves_like 'jobs allowed to run'
end
end
......@@ -76,16 +116,8 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::INTERNAL)
end
it_behaves_like 'available CI quota'
context 'when the Ci quota is exceeded' do
before do
allow(pipeline.project).to receive(:ci_minutes_quota)
.and_return(double('quota', minutes_used_up?: true))
end
it_behaves_like 'limit exceeded'
end
it_behaves_like 'jobs allowed to run'
it_behaves_like 'limit exceeded'
end
context 'with private projects' do
......@@ -93,16 +125,8 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::PRIVATE)
end
it_behaves_like 'available CI quota'
context 'when the Ci quota is exceeded' do
before do
allow(pipeline.project).to receive(:ci_minutes_quota)
.and_return(double('quota', minutes_used_up?: true))
end
it_behaves_like 'limit exceeded'
end
it_behaves_like 'jobs allowed to run'
it_behaves_like 'limit exceeded'
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::PipelineCreation::StartPipelineService do
let(:pipeline) { build(:ci_pipeline) }
subject(:service) { described_class.new(pipeline) }
describe '#execute' do
it 'calls the pipeline runners matching validation service' do
expect(Ci::PipelineCreation::DropNotRunnableBuildsService)
.to receive(:new)
.with(pipeline)
.and_return(double('service', execute: true))
service.execute
end
end
end
......@@ -29,48 +29,22 @@ RSpec.describe Ci::ProcessPipelineService, '#execute' do
stub_ci_pipeline_to_return_yaml_file
end
context 'when there is a runner available' do
let_it_be(:runner) do
create(:ci_runner, :online, tag_list: %w[ruby postgres mysql])
end
it 'creates a downstream cross-project pipeline' do
service.execute
Sidekiq::Worker.drain_all
it 'creates a downstream cross-project pipeline' do
service.execute
Sidekiq::Worker.drain_all
expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
update_build_status(:test, :success)
Sidekiq::Worker.drain_all
update_build_status(:test, :success)
Sidekiq::Worker.drain_all
expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
expect(downstream.ci_pipelines).to be_one
expect(downstream.ci_pipelines.first).to be_pending
expect(downstream.builds).not_to be_empty
expect(downstream.builds.first.variables)
.to include(key: 'BRIDGE', value: 'cross', public: false, masked: false)
end
end
context 'with no runners' do
it 'creates a failed downstream cross-project pipeline' do
service.execute
Sidekiq::Worker.drain_all
expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
update_build_status(:test, :success)
Sidekiq::Worker.drain_all
expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
expect(downstream.ci_pipelines).to be_one
expect(downstream.ci_pipelines.first).to be_failed
expect(downstream.builds).not_to be_empty
expect(downstream.builds).to all be_failed
expect(downstream.builds.map(&:failure_reason)).to all eq('no_matching_runner')
end
expect(downstream.ci_pipelines).to be_one
expect(downstream.ci_pipelines.first).to be_pending
expect(downstream.builds).not_to be_empty
expect(downstream.builds.first.variables)
.to include(key: 'BRIDGE', value: 'cross', public: false, masked: false)
end
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::InitialPipelineProcessWorker do
describe '#perform' do
let_it_be(:namespace) { create(:namespace, :with_used_build_minutes_limit) }
let_it_be(:project) { create(:project, namespace: namespace) }
let_it_be_with_reload(:pipeline) do
create(:ci_pipeline, :with_job, project: project, status: :created)
end
let_it_be(:instance_runner) { create(:ci_runner, :instance, :online) }
include_examples 'an idempotent worker' do
let(:job_args) { pipeline.id }
context 'when the project is out of CI minutes' do
it 'marks the pipeline as failed' do
expect(pipeline).to be_created
subject
expect(pipeline.reload).to be_failed
end
end
end
end
end
# frozen_string_literal: true
module QA
# TODO: Remove `:requires_admin` meta when the feature flag is removed
RSpec.describe 'Verify', :runner, :requires_admin do
RSpec.describe 'Verify', :runner do
describe 'Pipeline creation and processing' do
let(:executor) { "qa-runner-#{Time.now.to_i}" }
let(:max_wait) { 30 }
let(:feature_flag) { :ci_drop_new_builds_when_ci_quota_exceeded }
let(:project) do
Resource::Project.fabricate_via_api! do |project|
......@@ -28,8 +26,6 @@ module QA
it 'users creates a pipeline which gets processed', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1849' do
# TODO: Convert back to :smoke once proved to be stable. Related issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300909
tags_mismatch_status = Runtime::Feature.enabled?(feature_flag, project: project) ? :failed : :pending
Flow::Login.sign_in
Resource::Repository::Commit.fabricate_via_api! do |commit|
......@@ -75,7 +71,7 @@ module QA
{
'test-success': :passed,
'test-failure': :failed,
'test-tags-mismatch': tags_mismatch_status,
'test-tags-mismatch': :pending,
'test-artifacts': :passed
}.each do |job, status|
Page::Project::Pipeline::Show.perform do |pipeline|
......
......@@ -202,37 +202,21 @@ RSpec.describe Ci::CreatePipelineService do
YAML
end
context 'when there are runners matching the builds' do
before do
create(:ci_runner, :online)
end
it 'creates a pipeline with build_a and test_b pending; deploy_b manual', :sidekiq_inline do
processables = pipeline.processables
build_a = processables.find { |processable| processable.name == 'build_a' }
test_a = processables.find { |processable| processable.name == 'test_a' }
test_b = processables.find { |processable| processable.name == 'test_b' }
deploy_a = processables.find { |processable| processable.name == 'deploy_a' }
deploy_b = processables.find { |processable| processable.name == 'deploy_b' }
expect(pipeline).to be_created_successfully
expect(build_a.status).to eq('pending')
expect(test_a.status).to eq('created')
expect(test_b.status).to eq('pending')
expect(deploy_a.status).to eq('created')
expect(deploy_b.status).to eq('manual')
end
end
context 'when there are no runners matching the builds' do
it 'creates a pipeline but all jobs failed', :sidekiq_inline do
processables = pipeline.processables
expect(pipeline).to be_created_successfully
expect(processables).to all be_failed
expect(processables.map(&:failure_reason)).to all eq('no_matching_runner')
end
it 'creates a pipeline with build_a and test_b pending; deploy_b manual', :sidekiq_inline do
processables = pipeline.processables
build_a = processables.find { |processable| processable.name == 'build_a' }
test_a = processables.find { |processable| processable.name == 'test_a' }
test_b = processables.find { |processable| processable.name == 'test_b' }
deploy_a = processables.find { |processable| processable.name == 'deploy_a' }
deploy_b = processables.find { |processable| processable.name == 'deploy_b' }
expect(pipeline).to be_created_successfully
expect(build_a.status).to eq('pending')
expect(test_a.status).to eq('created')
expect(test_b.status).to eq('pending')
expect(deploy_a.status).to eq('created')
expect(deploy_b.status).to eq('manual')
end
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
let_it_be(:group) { create(:group) }
let_it_be(:project) { create(:project, group: group) }
let_it_be_with_reload(:pipeline) do
create(:ci_pipeline, project: project, status: :created)
end
let_it_be_with_reload(:job) do
create(:ci_build, project: project, pipeline: pipeline)
end
describe '#execute' do
subject(:execute) { described_class.new(pipeline).execute }
shared_examples 'jobs allowed to run' do
it 'does not drop the jobs' do
expect { execute }.not_to change { job.reload.status }
end
end
context 'when the feature flag is disabled' do
before do
stub_feature_flags(ci_drop_new_builds_when_ci_quota_exceeded: false)
end
it_behaves_like 'jobs allowed to run'
end
context 'when the pipeline status is running' do
before do
pipeline.update!(status: :running)
end
it_behaves_like 'jobs allowed to run'
end
context 'when there are no runners available' do
let_it_be(:offline_project_runner) do
create(:ci_runner, runner_type: :project_type, projects: [project])
end
it 'drops the job' do
execute
job.reload
expect(job).to be_failed
expect(job.failure_reason).to eq('no_matching_runner')
end
end
context 'with project runners' do
let_it_be(:project_runner) do
create(:ci_runner, :online, runner_type: :project_type, projects: [project])
end
it_behaves_like 'jobs allowed to run'
end
context 'with group runners' do
let_it_be(:group_runner) do
create(:ci_runner, :online, runner_type: :group_type, groups: [group])
end
it_behaves_like 'jobs allowed to run'
end
context 'with instance runners' do
let_it_be(:instance_runner) do
create(:ci_runner, :online, runner_type: :instance_type)
end
it_behaves_like 'jobs allowed to run'
end
end
end
......@@ -8,15 +8,6 @@ RSpec.describe Ci::PipelineCreation::StartPipelineService do
subject(:service) { described_class.new(pipeline) }
describe '#execute' do
it 'calls the pipeline runners matching validation service' do
expect(Ci::PipelineCreation::DropNotRunnableBuildsService)
.to receive(:new)
.with(pipeline)
.and_return(double('service', execute: true))
service.execute
end
it 'calls the pipeline process service' do
expect(Ci::ProcessPipelineService)
.to receive(:new)
......
......@@ -11,26 +11,12 @@ RSpec.describe Ci::InitialPipelineProcessWorker do
include_examples 'an idempotent worker' do
let(:job_args) { pipeline.id }
context 'when there are runners available' do
before do
create(:ci_runner, :online)
end
it 'marks the pipeline as pending' do
expect(pipeline).to be_created
subject
expect(pipeline.reload).to be_pending
end
end
it 'marks the pipeline as failed' do
it 'marks the pipeline as pending' do
expect(pipeline).to be_created
subject
expect(pipeline.reload).to be_failed
expect(pipeline.reload).to be_pending
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment