Split Geo::RepositoryVerificationFinder#find_failed_projects per type

ee/app/finders/geo/repository_verification_finder.rb

@@ -4,13 +4,22 @@ module Geo
       @shard_name = shard_name
     end
 
-    def find_failed_projects(batch_size:)
-      query = build_query_to_find_failed_projects(batch_size: batch_size)
-      cte   = Gitlab::SQL::CTE.new(:failed_projects, query)
+    def find_failed_repositories(batch_size:)
+      query = build_query_to_find_failed_projects(type: :repository, batch_size: batch_size)
+      cte   = Gitlab::SQL::CTE.new(:failed_repositories, query)
 
       Project.with(cte.to_arel)
              .from(cte.alias_to(projects_table))
-             .order(last_repository_updated_at_asc)
+             .order("projects.repository_retry_at ASC")
+    end
+
+    def find_failed_wikis(batch_size:)
+      query = build_query_to_find_failed_projects(type: :wiki, batch_size: batch_size)
+      cte   = Gitlab::SQL::CTE.new(:failed_wikis, query)
+
+      Project.with(cte.to_arel)
+             .from(cte.alias_to(projects_table))
+             .order("projects.wiki_retry_at ASC")
     end
 
     def find_outdated_projects(batch_size:)
@@ -54,12 +63,12 @@ module Geo
 
     attr_reader :shard_name
 
-    def build_query_to_find_failed_projects(batch_size:)
+    def build_query_to_find_failed_projects(type:, batch_size:)
       query =
         projects_table
           .join(repository_state_table).on(project_id_matcher)
-          .project(projects_table[:id], projects_table[:last_repository_updated_at])
-          .where(repository_failed.or(wiki_failed))
+          .project(projects_table[:id], repository_state_table["#{type}_retry_at"])
+          .where(repository_state_table["last_#{type}_verification_failure"].not_eq(nil))
           .take(batch_size)
 
       query = apply_shard_restriction(query) if shard_name.present?
@@ -97,19 +106,11 @@ module Geo
         .join_sources
     end
 
-    def repository_failed
-      repository_state_table[:last_repository_verification_failure].not_eq(nil)
-    end
-
     def repository_outdated
       repository_state_table[:repository_verification_checksum].eq(nil)
         .and(repository_state_table[:last_repository_verification_failure].eq(nil))
     end
 
-    def wiki_failed
-      repository_state_table[:last_wiki_verification_failure].not_eq(nil)
-    end
-
     def wiki_outdated
       repository_state_table[:wiki_verification_checksum].eq(nil)
         .and(repository_state_table[:last_wiki_verification_failure].eq(nil))

ee/app/workers/geo/repository_verification/primary/shard_worker.rb

@@ -60,7 +60,18 @@ module Geo
         end
 
         def find_failed_project_ids(batch_size:)
-          finder.find_failed_projects(batch_size: batch_size).pluck(:id)
+          repositories_ids = find_failed_repositories_ids(batch_size: batch_size)
+          wiki_ids = find_failed_wiki_ids(batch_size: batch_size)
+
+          take_batch(repositories_ids, wiki_ids, batch_size: batch_size)
+        end
+
+        def find_failed_repositories_ids(batch_size:)
+          finder.find_failed_repositories(batch_size: batch_size).pluck(:id)
+        end
+
+        def find_failed_wiki_ids(batch_size:)
+          finder.find_failed_wikis(batch_size: batch_size).pluck(:id)
         end
       end
     end

ee/spec/finders/geo/repository_verification_finder_spec.rb

@@ -3,53 +3,75 @@ require 'spec_helper'
 describe Geo::RepositoryVerificationFinder, :postgresql do
   set(:project) { create(:project) }
 
-  describe '#find_failed_projects' do
+  describe '#find_failed_repositories' do
     it 'returns projects where repository verification failed' do
       create(:repository_state, :repository_failed, :wiki_verified, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10))
+      expect(subject.find_failed_repositories(batch_size: 10))
         .to match_array(project)
     end
 
     it 'does not return projects where repository verification is outdated' do
       create(:repository_state, :repository_outdated, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10)).to be_empty
+      expect(subject.find_failed_repositories(batch_size: 10)).to be_empty
     end
 
     it 'does not return projects where repository verification is pending' do
       create(:repository_state, :wiki_verified, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10)).to be_empty
+      expect(subject.find_failed_repositories(batch_size: 10)).to be_empty
     end
 
+    it 'returns projects ordered by next retry time' do
+      next_project = create(:project)
+      create(:repository_state, :repository_failed, repository_retry_at: 1.hour.from_now, project: project)
+      create(:repository_state, :repository_failed, repository_retry_at: 30.minutes.from_now, project: next_project)
+
+      expect(subject.find_failed_repositories(batch_size: 10)).to eq [next_project, project]
+    end
+
+    context 'with shard restriction' do
+      subject { described_class.new(shard_name: project.repository_storage) }
+
+      it 'does not return projects on other shards' do
+        project_other_shard = create(:project)
+        project_other_shard.update_column(:repository_storage, 'other')
+        create(:repository_state, :repository_failed, project: project)
+        create(:repository_state, :repository_failed, project: project_other_shard)
+
+        expect(subject.find_failed_repositories(batch_size: 10))
+          .to match_array(project)
+      end
+    end
+  end
+
+  describe '#find_failed_wikis' do
     it 'returns projects where wiki verification failed' do
       create(:repository_state, :repository_verified, :wiki_failed, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10))
+      expect(subject.find_failed_wikis(batch_size: 10))
         .to match_array(project)
     end
 
-    it 'does non return projects where wiki verification is outdated' do
+    it 'does not return projects where wiki verification is outdated' do
       create(:repository_state, :wiki_outdated, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10)).to be_empty
+      expect(subject.find_failed_wikis(batch_size: 10)).to be_empty
     end
 
     it 'does not return projects where wiki verification is pending' do
       create(:repository_state, :repository_verified, project: project)
 
-      expect(subject.find_failed_projects(batch_size: 10)).to be_empty
+      expect(subject.find_failed_wikis(batch_size: 10)).to be_empty
     end
 
-    it 'returns less active projects first' do
-      less_active_project = create(:project)
-      create(:repository_state, :repository_failed, project: project)
-      create(:repository_state, :repository_failed, project: less_active_project)
-      project.update_column(:last_repository_updated_at, 30.minutes.ago)
-      less_active_project.update_column(:last_repository_updated_at, 2.days.ago)
+    it 'returns projects ordered by next retry time' do
+      next_project = create(:project)
+      create(:repository_state, :wiki_failed, wiki_retry_at: 1.hour.from_now, project: project)
+      create(:repository_state, :wiki_failed, wiki_retry_at: 30.minutes.from_now, project: next_project)
 
-      expect(subject.find_failed_projects(batch_size: 10)).to eq [less_active_project, project]
+      expect(subject.find_failed_wikis(batch_size: 10)).to eq [next_project, project]
     end
 
     context 'with shard restriction' do
@@ -58,10 +80,10 @@ describe Geo::RepositoryVerificationFinder, :postgresql do
       it 'does not return projects on other shards' do
         project_other_shard = create(:project)
         project_other_shard.update_column(:repository_storage, 'other')
-        create(:repository_state, :repository_failed, project: project)
-        create(:repository_state, :repository_failed, project: project_other_shard)
+        create(:repository_state, :wiki_failed, project: project)
+        create(:repository_state, :wiki_failed, project: project_other_shard)
 
-        expect(subject.find_failed_projects(batch_size: 10))
+        expect(subject.find_failed_wikis(batch_size: 10))
           .to match_array(project)
       end
     end