Merge branch 'ab-60524-high-frequency-query' into 'master'

Improve scheduling of mirror updates to reduce frequency of database queries

See merge request gitlab-org/gitlab-ee!11217

ee/app/workers/repository_update_mirror_worker.rb

@@ -35,8 +35,10 @@ class RepositoryUpdateMirrorWorker
     fail_mirror(project, ex.message)
     raise UpdateError, "#{ex.class}: #{ex.message}"
   ensure
-    if !lease.exists? && Gitlab::Mirror.reschedule_immediately? && lease.try_obtain
-      UpdateAllMirrorsWorker.perform_async
+    unless Feature.enabled?(:update_all_mirrors_worker_rescheduling)
+      if !lease.exists? && lease.try_obtain && Gitlab::Mirror.reschedule_immediately?
+        UpdateAllMirrorsWorker.perform_async
+      end
     end
   end
 

ee/app/workers/update_all_mirrors_worker.rb

@@ -7,14 +7,26 @@ class UpdateAllMirrorsWorker
   LEASE_TIMEOUT = 5.minutes
   SCHEDULE_WAIT_TIMEOUT = 4.minutes
   LEASE_KEY = 'update_all_mirrors'.freeze
+  RESCHEDULE_WAIT = 10.seconds
 
   def perform
-    lease_uuid = try_obtain_lease
-    return unless lease_uuid
+    scheduling_ran = with_lease do
+      schedule_mirrors!
+    end
+
+    # If we didn't get the lease, exit early
+    return unless scheduling_ran
 
-    schedule_mirrors!
+    if Feature.enabled?(:update_all_mirrors_worker_rescheduling)
+      # Wait to give some jobs a chance to complete
+      Kernel.sleep(RESCHEDULE_WAIT)
 
-    cancel_lease(lease_uuid)
+      # If there's capacity left now (some jobs completed),
+      # reschedule this job to enqueue more work.
+      #
+      # This is in addition to the regular (cron-like) scheduling of this job.
+      reschedule_if_capacity_left
+    end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
@@ -49,6 +61,22 @@ class UpdateAllMirrorsWorker
 
   private
 
+  def reschedule_if_capacity_left
+    return unless Gitlab::Mirror.reschedule_immediately?
+
+    UpdateAllMirrorsWorker.perform_async
+  end
+
+  def with_lease
+    if lease_uuid = try_obtain_lease
+      yield
+    end
+
+    lease_uuid
+  ensure
+    cancel_lease(lease_uuid) if lease_uuid
+  end
+
   def try_obtain_lease
     ::Gitlab::ExclusiveLease.new(LEASE_KEY, timeout: LEASE_TIMEOUT).try_obtain
   end

ee/changelogs/unreleased/ab-60524-high-frequency-query.yml

+---
+title: Swap conditions to reduce frequency of database query
+merge_request: 11217
+author:
+type: performance

ee/spec/workers/repository_update_mirror_worker_spec.rb

@@ -54,6 +54,8 @@ describe RepositoryUpdateMirrorWorker do
     context 'reschedule mirrors' do
       before do
         allow_any_instance_of(Projects::UpdateMirrorService).to receive(:execute).and_return(status: :success)
+
+        stub_feature_flags(update_all_mirrors_worker_rescheduling: false)
       end
 
       context 'when we obtain the lease' do

ee/spec/workers/update_all_mirrors_worker_spec.rb

@@ -23,6 +23,34 @@ describe UpdateAllMirrorsWorker do
 
       worker.perform
     end
+
+    context 'with update_all_mirrors_worker_rescheduling feature' do
+      before do
+        stub_feature_flags(update_all_mirrors_worker_rescheduling: true)
+      end
+
+      it 'sleeps a bit after scheduling mirrors' do
+        expect(Kernel).to receive(:sleep).with(described_class::RESCHEDULE_WAIT)
+
+        worker.perform
+      end
+
+      it 'reschedules the job if capacity is left' do
+        allow(Gitlab::Mirror).to receive(:reschedule_immediately?).and_return(true)
+
+        expect(described_class).to receive(:perform_async)
+
+        worker.perform
+      end
+
+      it 'does not reschedule the job if no capacity left' do
+        allow(Gitlab::Mirror).to receive(:reschedule_immediately?).and_return(false)
+
+        expect(described_class).not_to receive(:perform_async)
+
+        worker.perform
+      end
+    end
   end
 
   describe '#schedule_mirrors!' do