Commit 7f6056a0 authored by Furkan Ayhan's avatar Furkan Ayhan

Make PipelineProcessWorker idempotent with until_executing deduplication

This worker does nothing but calling
Ci::PipelineProcessing::AtomicProcessingService.
And this service is an idempotent service.
It does nothing unless the pipeline needs_processing?

This change is behind a FF ci_idempotent_pipeline_process_worker
parent f3528698
...@@ -1620,7 +1620,7 @@ ...@@ -1620,7 +1620,7 @@
:urgency: :high :urgency: :high
:resource_boundary: :unknown :resource_boundary: :unknown
:weight: 5 :weight: 5
:idempotent: :idempotent: true
:tags: [] :tags: []
- :name: pipeline_processing:stage_update - :name: pipeline_processing:stage_update
:worker_name: StageUpdateWorker :worker_name: StageUpdateWorker
......
# frozen_string_literal: true # frozen_string_literal: true
class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker class PipelineProcessWorker
include ApplicationWorker include ApplicationWorker
sidekiq_options retry: 3 sidekiq_options retry: 3
...@@ -10,7 +10,9 @@ class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker ...@@ -10,7 +10,9 @@ class PipelineProcessWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :continuous_integration feature_category :continuous_integration
urgency :high urgency :high
loggable_arguments 1 loggable_arguments 1
data_consistency :delayed, feature_flag: :load_balancing_for_pipeline_process_worker
idempotent!
deduplicate :until_executing, feature_flag: :ci_idempotent_pipeline_process_worker
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def perform(pipeline_id) def perform(pipeline_id)
......
--- ---
name: load_balancing_for_pipeline_process_worker name: ci_idempotent_pipeline_process_worker
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61766 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62410
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/330960 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/332963
milestone: '13.12' milestone: '14.0'
type: development type: development
group: group::pipeline execution group: group::pipeline authoring
default_enabled: false default_enabled: false
...@@ -3,10 +3,44 @@ ...@@ -3,10 +3,44 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe PipelineProcessWorker do RSpec.describe PipelineProcessWorker do
let_it_be(:pipeline) { create(:ci_pipeline) }
include_examples 'an idempotent worker' do
let(:pipeline) { create(:ci_pipeline, :created) }
let(:job_args) { [pipeline.id] }
before do
create(:ci_build, :created, pipeline: pipeline)
end
it 'processes the pipeline' do
expect(pipeline.status).to eq('created')
expect(pipeline.processables.pluck(:status)).to contain_exactly('created')
subject
expect(pipeline.reload.status).to eq('pending')
expect(pipeline.processables.pluck(:status)).to contain_exactly('pending')
subject
expect(pipeline.reload.status).to eq('pending')
expect(pipeline.processables.pluck(:status)).to contain_exactly('pending')
end
end
context 'when the FF ci_idempotent_pipeline_process_worker is disabled' do
before do
stub_feature_flags(ci_idempotent_pipeline_process_worker: false)
end
it 'is not deduplicated' do
expect(described_class).not_to be_deduplication_enabled
end
end
describe '#perform' do describe '#perform' do
context 'when pipeline exists' do context 'when pipeline exists' do
let(:pipeline) { create(:ci_pipeline) }
it 'processes pipeline' do it 'processes pipeline' do
expect_any_instance_of(Ci::ProcessPipelineService).to receive(:execute) expect_any_instance_of(Ci::ProcessPipelineService).to receive(:execute)
...@@ -16,14 +50,9 @@ RSpec.describe PipelineProcessWorker do ...@@ -16,14 +50,9 @@ RSpec.describe PipelineProcessWorker do
context 'when pipeline does not exist' do context 'when pipeline does not exist' do
it 'does not raise exception' do it 'does not raise exception' do
expect { described_class.new.perform(123) } expect { described_class.new.perform(non_existing_record_id) }
.not_to raise_error .not_to raise_error
end end
end end
it_behaves_like 'worker with data consistency',
described_class,
feature_flag: :load_balancing_for_pipeline_process_worker,
data_consistency: :delayed
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment