Respect the next retry time when re-verifying failed repositories

Scope the queries to retrieve failed repository verifications
by (repository|wiki)_retry_at < Time.now to respect the next
retry attempt, otherwise we retry them before the
(repository|wiki)_retry_at.

ee/app/finders/geo/repository_verification_finder.rb

@@ -79,8 +79,10 @@ module Geo
         projects_table
           .join(repository_state_table).on(project_id_matcher)
           .project(projects_table[:id], repository_state_table["#{type}_retry_at"])
-          .where(repository_state_table["last_#{type}_verification_failure"].not_eq(nil))
-          .take(batch_size)
+          .where(
+            repository_state_table["#{type}_retry_at"].lt(Time.now)
+              .and(repository_state_table["last_#{type}_verification_failure"].not_eq(nil))
+          ).take(batch_size)
 
       query = apply_shard_restriction(query) if shard_name.present?
       query

ee/changelogs/unreleased/8643-geo-fix-queries-to-retrieve-failed-verifications.yml

+---
+title: Geo - Respect the next retry time when re-verifying failed repositories
+merge_request: 8661
+author:
+type: fixed

ee/spec/factories/project_repository_states.rb

@@ -6,7 +6,7 @@ FactoryBot.define do
       repository_verification_checksum nil
       last_repository_verification_failure 'Could not calculate the checksum'
       repository_retry_count 1
-      repository_retry_at { Time.now + 5.minutes }
+      repository_retry_at { 5.minutes.ago }
     end
 
     trait :repository_outdated do
@@ -25,7 +25,7 @@ FactoryBot.define do
       wiki_verification_checksum nil
       last_wiki_verification_failure 'Could not calculate the checksum'
       wiki_retry_count 1
-      wiki_retry_at { Time.now + 5.minutes }
+      wiki_retry_at { 5.minutes.ago }
     end
 
     trait :wiki_outdated do

ee/spec/finders/geo/repository_verification_finder_spec.rb

@@ -4,13 +4,19 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
   set(:project) { create(:project) }
 
   describe '#find_failed_repositories' do
-    it 'returns projects where repository verification failed' do
+    it 'returns projects where next retry attempt is in the past' do
       create(:repository_state, :repository_failed, :wiki_verified, project: project)
 
       expect(subject.find_failed_repositories(batch_size: 10))
         .to match_array(project)
     end
 
+    it 'does not return projects where next retry attempt is in the future' do
+      create(:repository_state, :repository_failed, :wiki_verified, repository_retry_at: 5.minutes.from_now)
+
+      expect(subject.find_failed_repositories(batch_size: 10)).to be_empty
+    end
+
     it 'does not return projects where repository verification is outdated' do
       create(:repository_state, :repository_outdated, project: project)
 
@@ -25,10 +31,10 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
 
     it 'returns projects ordered by next retry time' do
       next_project = create(:project)
-      create(:repository_state, :repository_failed, repository_retry_at: 1.hour.from_now, project: project)
-      create(:repository_state, :repository_failed, repository_retry_at: 30.minutes.from_now, project: next_project)
+      create(:repository_state, :repository_failed, repository_retry_at: 1.hour.ago, project: project)
+      create(:repository_state, :repository_failed, repository_retry_at: 30.minutes.ago, project: next_project)
 
-      expect(subject.find_failed_repositories(batch_size: 10)).to eq [next_project, project]
+      expect(subject.find_failed_repositories(batch_size: 10)).to eq [project, next_project]
     end
 
     context 'with shard restriction' do
@@ -47,13 +53,19 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
   end
 
   describe '#find_failed_wikis' do
-    it 'returns projects where wiki verification failed' do
+    it 'returns projects where next retry attempt is in the past' do
       create(:repository_state, :repository_verified, :wiki_failed, project: project)
 
       expect(subject.find_failed_wikis(batch_size: 10))
         .to match_array(project)
     end
 
+    it 'does not return projects where next retry attempt is in the future' do
+      create(:repository_state, :repository_verified, :wiki_failed, wiki_retry_at: 5.minutes.from_now)
+
+      expect(subject.find_failed_wikis(batch_size: 10)).to be_empty
+    end
+
     it 'does not return projects where wiki verification is outdated' do
       create(:repository_state, :wiki_outdated, project: project)
 
@@ -68,10 +80,10 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
 
     it 'returns projects ordered by next retry time' do
       next_project = create(:project)
-      create(:repository_state, :wiki_failed, wiki_retry_at: 1.hour.from_now, project: project)
-      create(:repository_state, :wiki_failed, wiki_retry_at: 30.minutes.from_now, project: next_project)
+      create(:repository_state, :wiki_failed, wiki_retry_at: 1.hour.ago, project: project)
+      create(:repository_state, :wiki_failed, wiki_retry_at: 30.minutes.ago, project: next_project)
 
-      expect(subject.find_failed_wikis(batch_size: 10)).to eq [next_project, project]
+      expect(subject.find_failed_wikis(batch_size: 10)).to eq [project, next_project]
     end
 
     context 'with shard restriction' do