Check which repositories should be synced

app/services/geo/repository_sync_service.rb

@@ -12,8 +12,8 @@ module Geo
     def execute
       try_obtain_lease do
         log('Started repository sync')
-        started_at, finished_at = fetch_repositories
-        update_registry(started_at, finished_at)
+        sync_project_repository
+        sync_wiki_repository
         log('Finished repository sync')
       end
     rescue ActiveRecord::RecordNotFound
@@ -26,14 +26,51 @@ module Geo
       @project ||= Project.find(project_id)
     end
 
-    def fetch_repositories
+    def registry
+      @registry ||= Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id)
+    end
+
+    def sync_project_repository
+      return unless sync_repository?
+
+      started_at, finished_at = fetch_project_repository
+      update_registry(:repository, started_at, finished_at)
+      expire_repository_caches
+    end
+
+    def sync_repository?
+      return true if registry.resync_repository?
+      return true if registry.last_repository_successful_sync_at.nil?
+      return true if registry.last_repository_synced_at.nil?
+
+      false
+    end
+
+    def sync_wiki_repository
+      return unless sync_wiki?
+
+      started_at, finished_at = fetch_wiki_repository
+      update_registry(:wiki, started_at, finished_at)
+    end
+
+    def sync_wiki?
+      return true if registry.resync_wiki?
+      return true if registry.last_wiki_successful_sync_at.nil?
+      return true if registry.last_wiki_synced_at.nil?
+
+      false
+    end
+
+    def fetch_project_repository
+      return unless sync_repository?
+
+      log('Fetching project repository')
       started_at  = DateTime.now
       finished_at = nil
 
       begin
-        fetch_project_repository
-        fetch_wiki_repository
-        expire_repository_caches
+        project.create_repository unless project.repository_exists?
+        project.repository.fetch_geo_mirror(ssh_url_to_repo)
 
         finished_at = DateTime.now
       rescue Gitlab::Shell::Error => e
@@ -47,18 +84,25 @@ module Geo
       [started_at, finished_at]
     end
 
-    def fetch_project_repository
-      log('Fetching project repository')
-      project.create_repository unless project.repository_exists?
-      project.repository.fetch_geo_mirror(ssh_url_to_repo)
-    end
-
     def fetch_wiki_repository
-      # Second .wiki call returns a Gollum::Wiki, and it will always create the physical repository when not found
-      if project.wiki.wiki.exist?
-        log('Fetching wiki repository')
-        project.wiki.repository.fetch_geo_mirror(ssh_url_to_wiki)
+      return unless sync_wiki?
+
+      log('Fetching wiki repository')
+      started_at  = DateTime.now
+      finished_at = nil
+
+      begin
+        # Second .wiki call returns a Gollum::Wiki, and it will always create the physical repository when not found
+        if project.wiki.wiki.exist?
+          project.wiki.repository.fetch_geo_mirror(ssh_url_to_wiki)
+        end
+
+        finished_at = DateTime.now
+      rescue Gitlab::Shell::Error => e
+        Rails.logger.error("#{self.class.name}: Error syncing wiki repository for project #{project.path_with_namespace}: #{e}")
       end
+
+      [started_at, finished_at]
     end
 
     def expire_repository_caches
@@ -84,11 +128,15 @@ module Geo
       Gitlab::ExclusiveLease.cancel(lease_key, repository_lease)
     end
 
-    def update_registry(started_at, finished_at)
-      log('Updating repository sync information')
-      registry = Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id)
-      registry.last_repository_synced_at = started_at
-      registry.last_repository_successful_sync_at = finished_at if finished_at
+    def update_registry(type, started_at, finished_at)
+      log('Updating #{type} sync information')
+      registry.public_send("last_#{type}_synced_at=", started_at)
+
+      if finished_at
+        registry.public_send("last_#{type}_successful_sync_at=", started_at)
+        registry.public_send("resync_#{type}=", false)
+      end
+
       registry.save
     end
 

spec/services/geo/repository_sync_service_spec.rb

@@ -16,7 +16,7 @@ describe Geo::RepositorySyncService, services: true do
   end
 
   describe '#execute' do
-    context 'when repository is empty' do
+    context 'when project have never been synced' do
       let(:project) { create(:project_empty_repo) }
 
       it 'fetches project repositories' do
@@ -55,11 +55,11 @@ describe Geo::RepositorySyncService, services: true do
       end
 
       context 'tracking database' do
-        it 'tracks repository sync' do
+        it 'create a new registry' do
           expect { subject.execute }.to change(Geo::ProjectRegistry, :count).by(1)
         end
 
-        it 'stores last_repository_successful_sync_at when succeed' do
+        it 'sets last_repository_successful_sync_at when repository sync succeed' do
           subject.execute
 
           registry = Geo::ProjectRegistry.find_by(project_id: project.id)
@@ -67,8 +67,8 @@ describe Geo::RepositorySyncService, services: true do
           expect(registry.last_repository_successful_sync_at).not_to be_nil
         end
 
-        it 'reset last_repository_successful_sync_at when fail' do
-          allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { raise Gitlab::Shell::Error }
+        it 'resets last_repository_successful_sync_at when repository sync fail' do
+          allow_any_instance_of(Repository).to receive(:fetch_geo_mirror).with(/#{project.path_with_namespace}\.git/) { raise Gitlab::Shell::Error }
 
           subject.execute
 
@@ -76,58 +76,87 @@ describe Geo::RepositorySyncService, services: true do
 
           expect(registry.last_repository_successful_sync_at).to be_nil
         end
+
+        it 'sets last_wiki_successful_sync_at when wiki sync succeed' do
+          subject.execute
+
+          registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+
+          expect(registry.last_wiki_successful_sync_at).not_to be_nil
+        end
+
+        it 'resets last_wiki_successful_sync_at when wiki sync fail' do
+          allow_any_instance_of(Repository).to receive(:fetch_geo_mirror).with(/#{project.path_with_namespace}\.wiki.git/) { raise Gitlab::Shell::Error }
+
+          subject.execute
+
+          registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+
+          expect(registry.last_wiki_successful_sync_at).to be_nil
+        end
       end
     end
 
-    context 'when repository exists and is not empty' do
+    context 'when project have been synced' do
       let(:project) { create(:project) }
+      let(:last_repository_synced_at) { 5.days.ago }
+      let(:last_wiki_synced_at) { 4.days.ago }
 
-      it 'fetches project repositories' do
-        fetch_count = 0
+      let!(:registry) do
+        Geo::ProjectRegistry.create(
+          project: project,
+          last_repository_synced_at: last_repository_synced_at,
+          last_repository_successful_sync_at: last_repository_synced_at,
+          last_wiki_synced_at: last_wiki_synced_at,
+          last_wiki_successful_sync_at: last_wiki_synced_at,
+          resync_repository: false,
+          resync_wiki: false
+        )
+      end
 
-        allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) do
-          fetch_count += 1
-        end
+      it 'does not fetch project repositories' do
+        expect_any_instance_of(Repository).not_to receive(:fetch_geo_mirror)
 
         subject.execute
-
-        expect(fetch_count).to eq 2
       end
 
       context 'tracking database' do
-        it 'tracks repository sync' do
-          expect { subject.execute }.to change(Geo::ProjectRegistry, :count).by(1)
+        it 'does not create a new registry' do
+          expect { subject.execute }.not_to change(Geo::ProjectRegistry, :count)
         end
 
-        it 'stores last_repository_successful_sync_at when succeed' do
+        it 'does not update last_repository_successful_sync_at' do
           subject.execute
 
-          registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+          registry.reload
 
-          expect(registry.last_repository_successful_sync_at).not_to be_nil
+          expect(registry.last_repository_synced_at).to eq last_repository_synced_at
+          expect(registry.last_repository_successful_sync_at).to eq last_repository_synced_at
         end
 
-        it 'reset last_repository_successful_sync_at when fail' do
-          allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { raise Gitlab::Shell::Error }
-
+        it 'does not update last_wiki_successful_sync_at' do
           subject.execute
 
-          registry = Geo::ProjectRegistry.find_by(project_id: project.id)
+          registry.reload
 
-          expect(registry.last_repository_successful_sync_at).to be_nil
+          expect(registry.last_wiki_synced_at).to eq last_wiki_synced_at
+          expect(registry.last_wiki_successful_sync_at).to eq last_wiki_synced_at
         end
       end
     end
 
-    context 'when repository was synced successfully' do
+    context 'when last attempt to sync project repositories failed' do
       let(:project) { create(:project) }
-      let(:last_repository_synced_at) { 5.days.ago }
 
       let!(:registry) do
         Geo::ProjectRegistry.create(
           project: project,
-          last_repository_synced_at: last_repository_synced_at,
-          last_repository_successful_sync_at: last_repository_synced_at
+          last_repository_synced_at: DateTime.now,
+          last_repository_successful_sync_at: nil,
+          last_wiki_synced_at: DateTime.now,
+          last_wiki_successful_sync_at: nil,
+          resync_repository: false,
+          resync_wiki: false
         )
       end
 
@@ -144,11 +173,49 @@ describe Geo::RepositorySyncService, services: true do
       end
 
       context 'tracking database' do
-        it 'does not create a new registry' do
-          expect { subject.execute }.not_to change(Geo::ProjectRegistry, :count)
+        it 'sets last_repository_successful_sync_at' do
+          subject.execute
+
+          registry.reload
+
+          expect(registry.last_repository_successful_sync_at).not_to be_nil
+        end
+
+        it 'sets last_wiki_successful_sync_at' do
+          subject.execute
+
+          registry.reload
+
+          expect(registry.last_wiki_successful_sync_at).not_to be_nil
         end
+      end
+    end
+
+    context 'when project repository is dirty' do
+      let(:project) { create(:project) }
+      let(:last_repository_synced_at) { 5.days.ago }
+      let(:last_wiki_synced_at) { 4.days.ago }
+
+      let!(:registry) do
+        Geo::ProjectRegistry.create(
+          project: project,
+          last_repository_synced_at: last_repository_synced_at,
+          last_repository_successful_sync_at: last_repository_synced_at,
+          last_wiki_synced_at: last_wiki_synced_at,
+          last_wiki_successful_sync_at: last_wiki_synced_at,
+          resync_repository: true,
+          resync_wiki: false
+        )
+      end
+
+      it 'fetches project repository' do
+        expect_any_instance_of(Repository).to receive(:fetch_geo_mirror).once
 
-        it 'updates registry when succeed' do
+        subject.execute
+      end
+
+      context 'tracking database' do
+        it 'updates last_repository_successful_sync_at' do
           subject.execute
 
           registry.reload
@@ -157,51 +224,73 @@ describe Geo::RepositorySyncService, services: true do
           expect(registry.last_repository_successful_sync_at).to be_within(1.minute).of(Time.now)
         end
 
-        it 'does not update registry last_repository_successful_sync_at when fail' do
-          allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) { raise Gitlab::Shell::Error }
+        it 'does not update last_wiki_successful_sync_at' do
+          subject.execute
+
+          registry.reload
 
+          expect(registry.last_wiki_synced_at).to eq last_wiki_synced_at
+          expect(registry.last_wiki_successful_sync_at).to eq last_wiki_synced_at
+        end
+
+        it 'resets resync_repository' do
           subject.execute
 
           registry.reload
 
-          expect(registry.last_repository_synced_at).to be_within(1.minute).of(Time.now)
-          expect(registry.last_repository_successful_sync_at).to be_within(1.minute).of(last_repository_synced_at)
+          expect(registry.resync_repository).to be false
         end
       end
     end
 
-    context 'when last attempt to sync the repository failed' do
+    context 'when project wiki is dirty' do
       let(:project) { create(:project) }
+      let(:last_repository_synced_at) { 5.days.ago }
+      let(:last_wiki_synced_at) { 4.days.ago }
 
       let!(:registry) do
         Geo::ProjectRegistry.create(
           project: project,
-          last_repository_synced_at: DateTime.now,
-          last_repository_successful_sync_at: nil
+          last_repository_synced_at: last_repository_synced_at,
+          last_repository_successful_sync_at: last_repository_synced_at,
+          last_wiki_synced_at: last_wiki_synced_at,
+          last_wiki_successful_sync_at: last_wiki_synced_at,
+          resync_repository: false,
+          resync_wiki: true
         )
       end
 
-      it 'fetches project repositories' do
-        fetch_count = 0
-
-        allow_any_instance_of(Repository).to receive(:fetch_geo_mirror) do
-          fetch_count += 1
-        end
+      it 'fetches wiki repository' do
+        expect_any_instance_of(Repository).to receive(:fetch_geo_mirror).once
 
         subject.execute
-
-        expect(fetch_count).to eq 2
       end
 
       context 'tracking database' do
-        it 'does not create a new registry' do
-          expect { subject.execute }.not_to change(Geo::ProjectRegistry, :count)
+        it 'updates last_wiki_successful_sync_at' do
+          subject.execute
+
+          registry.reload
+
+          expect(registry.last_wiki_synced_at).to be_within(1.minute).of(Time.now)
+          expect(registry.last_wiki_successful_sync_at).to be_within(1.minute).of(Time.now)
         end
 
-        it 'updates last_repository_successful_sync_at' do
+        it 'does not update last_repository_successful_sync_at' do
           subject.execute
 
-          expect(registry.reload.last_repository_successful_sync_at).not_to be_nil
+          registry.reload
+
+          expect(registry.last_repository_synced_at).to eq last_repository_synced_at
+          expect(registry.last_repository_successful_sync_at).to eq last_repository_synced_at
+        end
+
+        it 'resets resync_wiki' do
+          subject.execute
+
+          registry.reload
+
+          expect(registry.resync_wiki).to be false
         end
       end
     end