Resolve "Geo: Checksum failures often due to repository not existing on disk"

ee/app/services/geo/repository_verify_secondary_service.rb

@@ -40,17 +40,23 @@ module Geo
     end
 
     def verify_checksum
-      checksum = repository.checksum
+      checksum = calculate_checksum
 
       if mismatch?(checksum)
         update_registry!(failure: "#{type.to_s.capitalize} checksum mismatch: #{repository.disk_path}")
       else
         update_registry!(checksum: checksum)
       end
-    rescue ::Gitlab::Git::Repository::NoRepository, ::Gitlab::Git::Repository::ChecksumError, Timeout::Error => e
+    rescue ::Gitlab::Git::Repository::ChecksumError, Timeout::Error => e
       update_registry!(failure: "Error verifying #{type.to_s.capitalize} checksum: #{repository.disk_path}", exception: e)
     end
 
+    def calculate_checksum
+      repository.checksum
+    rescue Gitlab::Git::Repository::NoRepository
+      Gitlab::Git::Repository::EMPTY_REPOSITORY_CHECKSUM
+    end
+
     def mismatch?(checksum)
       primary_checksum != checksum
     end

ee/app/workers/geo/repository_verification/primary/single_worker.rb

@@ -35,6 +35,8 @@ module Geo
 
         def calculate_checksum(type, repository)
           update_repository_state!(type, checksum: repository.checksum)
+        rescue Gitlab::Git::Repository::NoRepository
+          update_repository_state!(type, checksum: Gitlab::Git::Repository::EMPTY_REPOSITORY_CHECKSUM)
         rescue => e
           log_error('Error calculating the repository checksum', e, type: type)
           update_repository_state!(type, failure: e.message)

ee/changelogs/unreleased/5576-geo-checksum-failures-often-due-to-repository-not-existing-on-disk.yml

+---
+title: Geo - Returns a dummy checksum when there is no repository on disk
+merge_request:
+author:
+type: fixed

ee/spec/services/geo/repository_verify_secondary_service_spec.rb

@@ -54,6 +54,19 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
         .from(nil).to('my_checksum')
     end
 
+    it 'does not mark the verification as failed when there is no repo' do
+      allow(repository).to receive(:checksum).and_raise(Gitlab::Git::Repository::NoRepository)
+
+      repository_state.assign_attributes("#{type}_verification_checksum" => '0000000000000000000000000000000000000000')
+
+      service.execute
+
+      expect(registry.reload).to have_attributes(
+        "#{type}_verification_checksum_sha" => '0000000000000000000000000000000000000000',
+        "last_#{type}_verification_failure" => nil
+      )
+    end
+
     it 'keeps track of failure when the checksum mismatch' do
       expect(repository).to receive(:checksum).and_return('other_checksum')
 

ee/spec/workers/geo/repository_verification/primary/single_worker_spec.rb

@@ -100,7 +100,7 @@ describe Geo::RepositoryVerification::Primary::SingleWorker, :postgresql, :clean
           repository_verification_checksum: 'f123',
           wiki_verification_checksum: 'e123')
 
-      subject.perform(project_with_repositories.id - 1.hour)
+      subject.perform(project_with_repositories.id)
 
       expect(repository_state.reload).to have_attributes(
         repository_verification_checksum: 'f123',
@@ -121,14 +121,42 @@ describe Geo::RepositoryVerification::Primary::SingleWorker, :postgresql, :clean
       )
     end
 
-    it 'keeps track of failures when calculating the repository checksum' do
+    it 'does not mark the calculating as failed when there is no repo' do
       subject.perform(project_without_repositories.id)
 
       expect(project_without_repositories.repository_state).to have_attributes(
+        repository_verification_checksum: '0000000000000000000000000000000000000000',
+        last_repository_verification_failure: nil,
+        wiki_verification_checksum: '0000000000000000000000000000000000000000',
+        last_wiki_verification_failure: nil
+      )
+    end
+
+    it 'keeps track of failures when calculating the repository checksum' do
+      repository = double
+
+      allow(Repository).to receive(:new).with(
+        project_with_repositories.full_path,
+        project_with_repositories,
+        disk_path: project_with_repositories.disk_path
+      ).and_return(repository)
+
+      allow(Repository).to receive(:new).with(
+        project_with_repositories.wiki.full_path,
+        project_with_repositories,
+        disk_path: project_with_repositories.wiki.disk_path,
+        is_wiki: true
+      ).and_return(repository)
+
+      allow(repository).to receive(:checksum).twice.and_raise('Something went wrong')
+
+      subject.perform(project_with_repositories.id)
+
+      expect(project_with_repositories.repository_state).to have_attributes(
         repository_verification_checksum: nil,
-        last_repository_verification_failure: /not a git repository/,
+        last_repository_verification_failure: 'Something went wrong',
         wiki_verification_checksum: nil,
-        last_wiki_verification_failure: /not a git repository/
+        last_wiki_verification_failure: 'Something went wrong'
       )
     end
   end