Geo: Decrease frequency of project shard scheduler

If last time we checked, there were no pending resources, we set a back
off time to decrease the frequency of pool

ee/app/workers/geo/repository_shard_sync_worker.rb

@@ -4,16 +4,32 @@ module Geo
 
     attr_accessor :shard_name
 
+    BACKOFF_TIME = 5.minutes
+
     def perform(shard_name)
       @shard_name = shard_name
 
       return unless Gitlab::ShardHealthCache.healthy_shard?(shard_name)
 
+      return if should_be_skipped?
+
       super()
     end
 
     private
 
+    def skip_shard_key
+      "#{self.class.name.underscore}:shard:#{shard_name}:skip"
+    end
+
+    def should_be_skipped?
+      Rails.cache.read(skip_shard_key)
+    end
+
+    def set_backoff_time!
+      Rails.cache.write(skip_shard_key, true, expires_in: BACKOFF_TIME.minutes)
+    end
+
     def worker_metadata
       { shard: shard_name }
     end
@@ -54,11 +70,15 @@ module Geo
       resources = find_project_ids_not_synced(batch_size: db_retrieve_batch_size)
       remaining_capacity = db_retrieve_batch_size - resources.size
 
-      if remaining_capacity.zero?
+      pending_resources = if remaining_capacity.zero?
                             resources
                           else
                             resources + find_project_ids_updated_recently(batch_size: remaining_capacity)
                           end
+
+      set_backoff_time! if pending_resources.empty?
+
+      pending_resources
     end
 
     # rubocop: disable CodeReuse/ActiveRecord

ee/app/workers/geo/scheduler/scheduler_worker.rb

@@ -38,6 +38,7 @@ module Geo
           begin
             reason = loop do
               break :node_disabled unless node_enabled?
+              break :skipped       if should_be_skipped?
 
               update_jobs_in_progress
               update_pending_resources
@@ -69,6 +70,10 @@ module Geo
 
       private
 
+      def should_be_skipped?
+        false
+      end
+
       # Subclasses should override this method to provide additional metadata
       # in log messages
       def worker_metadata

ee/changelogs/unreleased/5748-geo-decrease-frequency-of-project-shard-schedulers-when-few-projects-to-schedule.yml

+---
+title: 'Geo: Decrease frequency of project shard schedulers when few projects to schedule'
+merge_request: 7287
+author:
+type: changed

ee/spec/workers/geo/repository_shard_sync_worker_spec.rb

@@ -9,18 +9,19 @@ describe Geo::RepositoryShardSyncWorker, :geo, :delete, :clean_gitlab_redis_cach
   let!(:primary) { create(:geo_node, :primary) }
   let!(:secondary) { create(:geo_node) }
 
-  let!(:restricted_group) { create(:group) }
-
-  let!(:unsynced_project_in_restricted_group) { create(:project, group: restricted_group) }
-  let!(:unsynced_project) { create(:project) }
-
   let(:shard_name) { Gitlab.config.repositories.storages.keys.first }
+  let(:cache_key) { "#{described_class.name.underscore}:shard:#{shard_name}:skip" }
 
   before do
     stub_current_geo_node(secondary)
   end
 
   shared_examples '#perform' do |skip_tests|
+    let!(:restricted_group) { create(:group) }
+
+    let!(:unsynced_project_in_restricted_group) { create(:project, group: restricted_group) }
+    let!(:unsynced_project) { create(:project) }
+
     before do
       skip('FDW is not configured') if skip_tests
     end
@@ -237,6 +238,23 @@ describe Geo::RepositoryShardSyncWorker, :geo, :delete, :clean_gitlab_redis_cach
         Sidekiq::Testing.inline! { subject.perform(shard_name) }
       end
     end
+
+    it 'sets the back off time when there no pending items' do
+      create(:geo_project_registry, :synced, project: unsynced_project_in_restricted_group)
+      create(:geo_project_registry, :synced, project: unsynced_project)
+
+      expect(Rails.cache).to receive(:write).with(cache_key, true, expires_in: 18000).once
+
+      subject.perform(shard_name)
+    end
+
+    it 'does not perform Geo::ProjectSyncWorker when the backoff time is set' do
+      expect(Rails.cache).to receive(:read).with(cache_key).and_return(true)
+
+      expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
+
+      subject.perform(shard_name)
+    end
   end
 
   describe 'when PostgreSQL FDW is available', :geo do