Enable load balancing in ExpirePipelineCacheWorker

By marking it delayed. This will enable the use
of read replicas as long as they're caught up.

app/workers/all_queues.yml

@@ -1428,7 +1428,7 @@
   :urgency: :high
   :resource_boundary: :cpu
   :weight: 3
-  :idempotent: true
+  :idempotent:
   :tags: []
 - :name: pipeline_creation:create_pipeline
   :worker_name: CreatePipelineWorker

app/workers/expire_pipeline_cache_worker.rb

 # frozen_string_literal: true
 
+# rubocop: disable Scalability/IdempotentWorker
 class ExpirePipelineCacheWorker
   include ApplicationWorker
 
@@ -9,8 +10,12 @@ class ExpirePipelineCacheWorker
   queue_namespace :pipeline_cache
   urgency :high
   worker_resource_boundary :cpu
+  data_consistency :delayed, feature_flag: :load_balancing_for_expire_pipeline_cache_worker
 
-  idempotent!
+  # This worker _should_ be idempotent, but due to us moving this to data_consistency :delayed
+  # and an ongoing incompatibility between the two switches, we need to disable this.
+  # Uncomment once https://gitlab.com/gitlab-org/gitlab/-/issues/325291 is resolved
+  # idempotent!
 
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(pipeline_id)
@@ -21,3 +26,4 @@ class ExpirePipelineCacheWorker
   end
   # rubocop: enable CodeReuse/ActiveRecord
 end
+# rubocop:enable Scalability/IdempotentWorker

config/feature_flags/development/load_balancing_for_expire_pipeline_cache_worker.yml

+---
+name: load_balancing_for_expire_pipeline_cache_worker
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62073
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/331360
+milestone: '14.0'
+type: development
+group: group::memory
+default_enabled: false

spec/workers/expire_pipeline_cache_worker_spec.rb

@@ -42,8 +42,15 @@ RSpec.describe ExpirePipelineCacheWorker do
       subject.perform(617748)
     end
 
+    skip "with https://gitlab.com/gitlab-org/gitlab/-/issues/325291 resolved" do
       it_behaves_like 'an idempotent worker' do
         let(:job_args) { [pipeline.id] }
       end
     end
+
+    it_behaves_like 'worker with data consistency',
+                  described_class,
+                  feature_flag: :load_balancing_for_expire_pipeline_cache_worker,
+                  data_consistency: :delayed
+  end
 end