[Geo] Don't mark sync as successful if repo does not exist

A secondary node marks every sync as successful if the repository
does not exist. While it's OK for the majority of the cases it
does not take into account some possible problems with
NFS servers availability.
To make more precise decisions we could make use of repository checksum.
If one exists but the repo doesn't, it's an indicator that
something is wrong with the infrastructure.

ee/app/services/geo/base_sync_service.rb

@@ -264,5 +264,13 @@ module Geo
     def new_repository?
       @new_repository
     end
+
+    # If repository has a verification checksum, we can assume that it existed on the primary
+    def repository_presumably_exists_on_primary?
+      return false unless project.repository_state
+
+      checksum = project.repository_state.public_send("#{type}_verification_checksum") # rubocop:disable GitlabSecurity/PublicSend
+      checksum && checksum != Gitlab::Git::Repository::EMPTY_REPOSITORY_CHECKSUM
+    end
   end
 end

ee/app/services/geo/repository_sync_service.rb

@@ -12,10 +12,14 @@ module Geo
       mark_sync_as_successful
     rescue Gitlab::Shell::Error, Gitlab::Git::BaseError => e
       # In some cases repository does not exist, the only way to know about this is to parse the error text.
-      # If it does not exist we should consider it as successfully downloaded.
       if e.message.include? Gitlab::GitAccess::ERROR_MESSAGES[:no_repo]
+        if repository_presumably_exists_on_primary?
+          log_info('Repository is not found, but it seems to exist on the primary')
+          fail_registry!('Repository is not found', e)
+        else
           log_info('Repository is not found, marking it as successfully synced')
           mark_sync_as_successful(missing_on_primary: true)
+        end
       else
         fail_registry!('Error syncing repository', e)
       end

ee/app/services/geo/wiki_sync_service.rb

@@ -14,8 +14,13 @@ module Geo
       # In some cases repository does not exist, the only way to know about this is to parse the error text.
       # If it does not exist we should consider it as successfully downloaded.
       if e.message.include? Gitlab::GitAccess::ERROR_MESSAGES[:no_repo]
-        log_info('Wiki repository is not found, marking it as successfully synced')
+        if repository_presumably_exists_on_primary?
+          log_info('Wiki is not found, but it seems to exist on the primary')
+          fail_registry!('Wiki is not found', e)
+        else
+          log_info('Wiki is not found, marking it as successfully synced')
           mark_sync_as_successful(missing_on_primary: true)
+        end
       else
         fail_registry!('Error syncing wiki repository', e)
       end

ee/changelogs/unreleased/5067-geo-don-t-mark-sync-as-successful-if-repo-does-not-exist-because-of-some-problems.yml

+---
+title: "[Geo] Don't mark sync as successful if repo does not exist because of some
+  problems"
+merge_request: 10578
+author:
+type: changed

ee/spec/services/geo/repository_sync_service_spec.rb

@@ -153,6 +153,24 @@ describe Geo::RepositorySyncService do
       expect(Geo::ProjectRegistry.last.resync_repository).to be true
     end
 
+    context 'repository presumably exists on primary' do
+      it 'increases retry count if no repository found' do
+        registry = create(:geo_project_registry, project: project)
+        create(:repository_state, :repository_verified, project: project)
+
+        allow(repository).to receive(:fetch_as_mirror)
+          .with(url_to_repo, remote_name: 'geo', forced: true)
+          .and_raise(Gitlab::Shell::Error.new(Gitlab::GitAccess::ERROR_MESSAGES[:no_repo]))
+
+        subject.execute
+
+        expect(registry.reload).to have_attributes(
+          resync_repository: true,
+          repository_retry_count: 1
+        )
+      end
+    end
+
     context 'tracking database' do
       context 'temporary repositories' do
         include_examples 'cleans temporary repositories' do

ee/spec/services/geo/wiki_sync_service_spec.rb

@@ -118,6 +118,24 @@ RSpec.describe Geo::WikiSyncService do
       expect(Geo::ProjectRegistry.last.resync_wiki).to be true
     end
 
+    context 'wiki repository presumably exists on primary' do
+      it 'increases retry count if no wiki repository found' do
+        registry = create(:geo_project_registry, project: project)
+        create(:repository_state, :wiki_verified, project: project)
+
+        allow(repository).to receive(:fetch_as_mirror)
+          .with(url_to_repo, remote_name: 'geo', forced: true)
+          .and_raise(Gitlab::Shell::Error.new(Gitlab::GitAccess::ERROR_MESSAGES[:no_repo]))
+
+        subject.execute
+
+        expect(registry.reload).to have_attributes(
+          resync_wiki: true,
+          wiki_retry_count: 1
+        )
+      end
+    end
+
     context 'tracking database' do
       context 'temporary repositories' do
         include_examples 'cleans temporary repositories' do