Merge branch 'ce-to-ee-2017-10-13' of gitlab.com:gitlab-org/gitlab-ee into ce_upstream

app/models/geo/project_registry.rb

@@ -20,19 +20,29 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
       .where(resync_repository: false, resync_wiki: false)
   end
 
-  def resync_repository?
-    resync_repository || last_repository_successful_sync_at.nil?
+  def repository_sync_due?(scheduled_time)
+    never_synced_repository? || repository_sync_needed?(scheduled_time)
   end
 
-  def resync_wiki?
-    resync_wiki || last_wiki_successful_sync_at.nil?
+  def wiki_sync_due?(scheduled_time)
+    project.wiki_enabled? && (never_synced_wiki? || wiki_sync_needed?(scheduled_time))
   end
 
-  def repository_synced_since?(timestamp)
-    last_repository_synced_at && last_repository_synced_at > timestamp
+  private
+
+  def never_synced_repository?
+    last_repository_successful_sync_at.nil?
+  end
+
+  def never_synced_wiki?
+    last_wiki_successful_sync_at.nil?
+  end
+
+  def repository_sync_needed?(timestamp)
+    resync_repository? && (last_repository_synced_at.nil? || timestamp > last_repository_synced_at)
   end
 
-  def wiki_synced_since?(timestamp)
-    last_wiki_synced_at && last_wiki_synced_at > timestamp
+  def wiki_sync_needed?(timestamp)
+    resync_wiki? && (last_wiki_synced_at.nil? || timestamp > last_wiki_synced_at)
   end
 end

app/services/geo/repository_created_event_store.rb

@@ -10,7 +10,7 @@ module Geo
         repository_storage_name: project.repository.storage,
         repository_storage_path: project.repository_storage_path,
         repo_path: project.disk_path,
-        wiki_path: "#{project.disk_path}.wiki",
+        wiki_path: ("#{project.disk_path}.wiki" if project.wiki_enabled?),
         project_name: project.name
       )
     end

app/workers/geo/project_sync_worker.rb

@@ -11,30 +11,16 @@ module Geo
     end
 
     def perform(project_id, scheduled_time)
-      project  = Project.find(project_id)
       registry = Geo::ProjectRegistry.find_or_initialize_by(project_id: project_id)
+      project = registry.project
 
-      Geo::RepositorySyncService.new(project).execute if sync_repository?(registry, scheduled_time)
-      Geo::WikiSyncService.new(project).execute if sync_wiki?(registry, scheduled_time)
-    rescue ActiveRecord::RecordNotFound => e
-      Gitlab::Geo::Logger.error(
-        class: self.class.name,
-        message: "Couldn't find project, skipping syncing",
-        project_id: project_id,
-        error: e
-      )
-    end
-
-    private
-
-    def sync_repository?(registry, scheduled_time)
-      !registry.repository_synced_since?(scheduled_time) &&
-        registry.resync_repository?
-    end
+      if project.nil?
+        Gitlab::Geo::Logger.error(class: self.class.name, message: "Couldn't find project, skipping syncing", project_id: project_id)
+        return
+      end
 
-    def sync_wiki?(registry, scheduled_time)
-      !registry.wiki_synced_since?(scheduled_time) &&
-        registry.resync_wiki?
+      Geo::RepositorySyncService.new(project).execute if registry.repository_sync_due?(scheduled_time)
+      Geo::WikiSyncService.new(project).execute if registry.wiki_sync_due?(scheduled_time)
     end
   end
 end

changelogs/unreleased-ee/3569-geo-backfill-unconditional-wiki.yml

+---
+title: 'Geo: Don''t sync disabled project wikis'
+merge_request: 3109
+author:
+type: fixed

lib/gitlab/geo/log_cursor/daemon.rb

@@ -19,9 +19,7 @@ module Gitlab
           full_scan! if options[:full_scan]
 
           until exit?
-            Events.fetch_in_batches do |batch|
-              handle_events(batch)
-            end
+            run_once!
 
             return if exit?
 
@@ -30,6 +28,10 @@ module Gitlab
           end
         end
 
+        def run_once!
+          Events.fetch_in_batches { |batch| handle_events(batch) }
+        end
+
         # Execute routines to verify the required initial data is available
         # and mark non-replicated data as requiring replication.
         def full_scan!

spec/factories/geo/event_log.rb

@@ -24,7 +24,7 @@ FactoryGirl.define do
     repository_storage_path { project.repository_storage_path }
     add_attribute(:repo_path) { project.disk_path }
     project_name { project.name }
-    wiki_path { "{project.disk_path}.wiki" }
+    wiki_path { "#{project.disk_path}.wiki" }
   end
 
   factory :geo_repository_updated_event, class: Geo::RepositoryUpdatedEvent do

spec/lib/gitlab/geo/log_cursor/daemon_spec.rb

@@ -3,48 +3,70 @@ require 'spec_helper'
 describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
   include ::EE::GeoHelpers
 
+  set(:primary) { create(:geo_node, :primary) }
+  set(:secondary) { create(:geo_node) }
+
+  let(:options) { {} }
+  subject(:daemon) { described_class.new(options) }
+
+  around do |example|
+    Sidekiq::Testing.fake! { example.run }
+  end
+
+  before do
+    stub_current_geo_node(secondary)
+    stub_env("::#{described_class}::POOL_WAIT", 0.1)
+
+    allow(daemon).to receive(:trap_signals)
+  end
+
   describe '#run!' do
-    set(:geo_node) { create(:geo_node, :primary) }
+    it 'traps signals' do
+      is_expected.to receive(:exit?).and_return(true)
+      is_expected.to receive(:trap_signals)
 
-    before do
-      stub_current_geo_node(geo_node)
+      daemon.run!
     end
 
-    it 'traps signals' do
-      allow(subject).to receive(:exit?) { true }
-      expect(subject).to receive(:trap_signals)
+    it 'does not perform a full scan by default' do
+      is_expected.to receive(:exit?).and_return(true)
+      is_expected.not_to receive(:full_scan!)
 
-      subject.run!
+      daemon.run!
     end
 
-    context 'when the command-line defines full_scan: true' do
-      subject { described_class.new(full_scan: true) }
+    context 'the command-line defines full_scan: true' do
+      let(:options) { { full_scan: true } }
 
       it 'executes a full-scan' do
-        allow(subject).to receive(:exit?) { true }
+        is_expected.to receive(:exit?).and_return(true)
+        is_expected.to receive(:full_scan!)
 
-        expect(subject).to receive(:full_scan!)
-
-        subject.run!
+        daemon.run!
       end
     end
 
+    it 'delegates to #run_once! in a loop' do
+      is_expected.to receive(:exit?).and_return(false, false, false, true)
+      is_expected.to receive(:run_once!).twice
+
+      daemon.run!
+    end
+  end
+
+  describe '#run_once!' do
     context 'when replaying a repository created event' do
       let(:project) { create(:project) }
       let(:repository_created_event) { create(:geo_repository_created_event, project: project) }
       let(:event_log) { create(:geo_event_log, repository_created_event: repository_created_event) }
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
 
-      before do
-        allow(subject).to receive(:exit?).and_return(false, true)
-      end
-
       it 'creates a new project registry' do
-        expect { subject.run! }.to change(Geo::ProjectRegistry, :count).by(1)
+        expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
       end
 
       it 'sets resync attributes to true' do
-        subject.run!
+        daemon.run_once!
 
         registry = Geo::ProjectRegistry.last
 
@@ -52,9 +74,9 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
       end
 
       it 'sets resync_wiki to false if wiki_path is nil' do
-        repository_created_event.update_attribute(:wiki_path, nil)
+        repository_created_event.update!(wiki_path: nil)
 
-        subject.run!
+        daemon.run_once!
 
         registry = Geo::ProjectRegistry.last
 
@@ -65,7 +87,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
         expect(Geo::ProjectSyncWorker).to receive(:perform_async)
           .with(project.id, anything).once
 
-        subject.run!
+        daemon.run_once!
       end
     end
 
@@ -75,28 +97,24 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
       let(:event_log) { create(:geo_event_log, repository_updated_event: repository_updated_event) }
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
 
-      before do
-        allow(subject).to receive(:exit?).and_return(false, true)
-      end
-
       it 'creates a new project registry if it does not exist' do
-        expect { subject.run! }.to change(Geo::ProjectRegistry, :count).by(1)
+        expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
       end
 
       it 'sets resync_repository to true if event source is repository' do
-        repository_updated_event.update_attribute(:source, Geo::RepositoryUpdatedEvent::REPOSITORY)
+        repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::REPOSITORY)
         registry = create(:geo_project_registry, :synced, project: repository_updated_event.project)
 
-        subject.run!
+        daemon.run_once!
 
         expect(registry.reload.resync_repository).to be true
       end
 
       it 'sets resync_wiki to true if event source is wiki' do
-        repository_updated_event.update_attribute(:source, Geo::RepositoryUpdatedEvent::WIKI)
+        repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::WIKI)
         registry = create(:geo_project_registry, :synced, project: repository_updated_event.project)
 
-        subject.run!
+        daemon.run_once!
 
         expect(registry.reload.resync_wiki).to be true
       end
@@ -105,7 +123,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
         expect(Geo::ProjectSyncWorker).to receive(:perform_async)
           .with(project.id, anything).once
 
-        subject.run!
+        daemon.run_once!
       end
     end
 
@@ -115,12 +133,8 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
       let(:repository_deleted_event) { event_log.repository_deleted_event }
 
-      before do
-        allow(subject).to receive(:exit?).and_return(false, true)
-      end
-
       it 'does not create a new project registry' do
-        expect { subject.run! }.not_to change(Geo::ProjectRegistry, :count)
+        expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
       end
 
       it 'schedules a GeoRepositoryDestroyWorker' do
@@ -132,24 +146,19 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
         expect(::GeoRepositoryDestroyWorker).to receive(:perform_async)
           .with(project_id, project_name, full_path, project.repository_storage)
 
-        subject.run!
+        daemon.run_once!
       end
     end
 
     context 'when replaying a repositories changed event' do
-      let(:geo_node) { create(:geo_node) }
-      let(:repositories_changed_event) { create(:geo_repositories_changed_event, geo_node: geo_node) }
+      let(:repositories_changed_event) { create(:geo_repositories_changed_event, geo_node: secondary) }
       let(:event_log) { create(:geo_event_log, repositories_changed_event: repositories_changed_event) }
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
 
-      before do
-        allow(subject).to receive(:exit?).and_return(false, true)
-      end
-
       it 'schedules a GeoRepositoryDestroyWorker when event node is the current node' do
-        expect(Geo::RepositoriesCleanUpWorker).to receive(:perform_in).with(within(5.minutes).of(1.hour), geo_node.id)
+        expect(Geo::RepositoriesCleanUpWorker).to receive(:perform_in).with(within(5.minutes).of(1.hour), secondary.id)
 
-        subject.run!
+        daemon.run_once!
       end
 
       it 'does not schedule a GeoRepositoryDestroyWorker when event node is not the current node' do
@@ -157,7 +166,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
 
         expect(Geo::RepositoriesCleanUpWorker).not_to receive(:perform_in)
 
-        subject.run!
+        daemon.run_once!
       end
     end
 
@@ -170,36 +179,19 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
 
       before do
-        allow(subject).to receive(:exit?).and_return(false, true)
         allow(Geo::ProjectSyncWorker).to receive(:perform_async)
       end
 
       it 'replays events for projects that belong to selected namespaces to replicate' do
-        geo_node.update_attribute(:namespaces, [group_1])
+        secondary.update!(namespaces: [group_1])
 
-        expect { subject.run! }.to change(Geo::ProjectRegistry, :count).by(1)
+        expect { daemon.run_once! }.to change(Geo::ProjectRegistry, :count).by(1)
       end
 
       it 'does not replay events for projects that do not belong to selected namespaces to replicate' do
-        geo_node.update_attribute(:namespaces, [group_2])
+        secondary.update!(namespaces: [group_2])
 
-        expect { subject.run! }.not_to change(Geo::ProjectRegistry, :count)
-      end
-
-      context 'when performing a full scan' do
-        subject { described_class.new(full_scan: true) }
-
-        it 'creates registries for missing projects that belong to selected namespaces' do
-          geo_node.update_attribute(:namespaces, [group_1])
-
-          expect { subject.run! }.to change(Geo::ProjectRegistry, :count).by(1)
-        end
-
-        it 'does not create registries for missing projects that do not belong to selected namespaces' do
-          geo_node.update_attribute(:namespaces, [group_2])
-
-          expect { subject.run! }.not_to change(Geo::ProjectRegistry, :count)
-        end
+        expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
       end
     end
 
@@ -209,12 +201,8 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
       let!(:event_log_state) { create(:geo_event_log_state, event_id: event_log.id - 1) }
       let(:repository_rename_event) { event_log.repository_renamed_event }
 
-      before do
-        allow(subject).to receive(:exit?).and_return(false, true)
-      end
-
       it 'does not create a new project registry' do
-        expect { subject.run! }.not_to change(Geo::ProjectRegistry, :count)
+        expect { daemon.run_once! }.not_to change(Geo::ProjectRegistry, :count)
       end
 
       it 'schedules a GeoRepositoryDestroyWorker' do
@@ -225,7 +213,25 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql do
         expect(::GeoRepositoryMoveWorker).to receive(:perform_async)
           .with(project_id, '', old_path_with_namespace, new_path_with_namespace)
 
-        subject.run!
+        daemon.run_once!
+      end
+    end
+  end
+
+  describe '#full_scan!' do
+    let(:project) { create(:project) }
+
+    context 'with selective sync enabled' do
+      it 'creates registries for missing projects that belong to selected namespaces' do
+        secondary.update!(namespaces: [project.namespace])
+
+        expect { daemon.full_scan! }.to change(Geo::ProjectRegistry, :count).by(1)
+      end
+
+      it 'does not create registries for missing projects that do not belong to selected namespaces' do
+        secondary.update!(namespaces: [create(:group)])
+
+        expect { daemon.full_scan! }.not_to change(Geo::ProjectRegistry, :count)
       end
     end
   end

spec/models/geo/project_registry_spec.rb

 require 'spec_helper'
 
 describe Geo::ProjectRegistry do
-  subject { create(:geo_project_registry) }
+  using RSpec::Parameterized::TableSyntax
+
+  set(:project) { create(:project) }
+  set(:registry) { create(:geo_project_registry, project_id: project.id) }
+
+  subject { registry }
 
   describe 'relationships' do
     it { is_expected.to belong_to(:project) }
@@ -33,85 +38,79 @@ describe Geo::ProjectRegistry do
     end
   end
 
-  describe '#resync_repository?' do
-    it 'returns true when resync_repository is true' do
-      subject.resync_repository = true
-
-      expect(subject.resync_repository).to be true
-    end
-
-    it 'returns true when last_repository_successful_sync_at is nil' do
-      subject.last_repository_successful_sync_at = nil
-
-      expect(subject.resync_repository).to be true
-    end
-
-    it 'returns false when resync_repository is false and last_repository_successful_sync_at is present' do
-      subject.resync_repository = false
-      subject.last_repository_successful_sync_at = Time.now
-
-      expect(subject.resync_repository).to be false
-    end
-  end
-
-  describe '#resync_wiki?' do
-    it 'returns true when resync_wiki is true' do
-      subject.resync_wiki = true
-
-      expect(subject.resync_wiki).to be true
+  describe '#repository_sync_due?' do
+    where(:resync_repository, :last_successful_sync, :last_sync, :expected) do
+      now = Time.now
+      past = now - 1.year
+      future = now + 1.year
+
+      true  | nil | nil | true
+      true  | now | nil | true
+      false | nil | nil | true
+      false | now | nil | false
+
+      true  | nil | past | true
+      true  | now | past | true
+      false | nil | past | true
+      false | now | past | false
+
+      true  | nil | future | true
+      true  | now | future | false
+      false | nil | future | true
+      false | now | future | false
     end
 
-    it 'returns true when last_wiki_successful_sync_at is nil' do
-      subject.last_wiki_successful_sync_at = nil
+    with_them do
+      before do
+        registry.update!(resync_repository: resync_repository, last_repository_successful_sync_at: last_successful_sync, last_repository_synced_at: last_sync)
+      end
 
-      expect(subject.resync_wiki).to be true
-    end
-
-    it 'returns false when resync_wiki is false and last_wiki_successful_sync_at is present' do
-      subject.resync_wiki = false
-      subject.last_wiki_successful_sync_at = Time.now
+      subject { registry.repository_sync_due?(Time.now) }
 
-      expect(subject.resync_wiki).to be false
+      it { is_expected.to eq(expected) }
     end
   end
 
-  describe '#repository_synced_since?' do
-    it 'returns false when last_repository_synced_at is nil' do
-      subject.last_repository_synced_at = nil
-
-      expect(subject.repository_synced_since?(Time.now)).to be_nil
+  describe '#wiki_sync_due?' do
+    where(:resync_wiki, :last_successful_sync, :last_sync, :expected) do
+      now = Time.now
+      past = now - 1.year
+      future = now + 1.year
+
+      true  | nil | nil | true
+      true  | now | nil | true
+      false | nil | nil | true
+      false | now | nil | false
+
+      true  | nil | past | true
+      true  | now | past | true
+      false | nil | past | true
+      false | now | past | false
+
+      true  | nil | future | true
+      true  | now | future | false
+      false | nil | future | true
+      false | now | future | false
     end
 
-    it 'returns false when last_repository_synced_at before timestamp' do
-      subject.last_repository_synced_at = Time.now - 2.hours
+    with_them do
+      before do
+        registry.update!(resync_wiki: resync_wiki, last_wiki_successful_sync_at: last_successful_sync, last_wiki_synced_at: last_sync)
+      end
 
-      expect(subject.repository_synced_since?(Time.now)).to be false
-    end
+      subject { registry.wiki_sync_due?(Time.now) }
 
-    it 'returns true when last_repository_synced_at after timestamp' do
-      subject.last_repository_synced_at = Time.now + 2.hours
-
-      expect(subject.repository_synced_since?(Time.now)).to be true
-    end
-  end
-
-  describe '#wiki_synced_since?' do
-    it 'returns false when last_wiki_synced_at is nil' do
-      subject.last_wiki_synced_at = nil
-
-      expect(subject.wiki_synced_since?(Time.now)).to be_nil
-    end
-
-    it 'returns false when last_wiki_synced_at before timestamp' do
-      subject.last_wiki_synced_at = Time.now - 2.hours
-
-      expect(subject.wiki_synced_since?(Time.now)).to be false
-    end
+      context 'wiki enabled' do
+        it { is_expected.to eq(expected) }
+      end
 
-    it 'returns true when last_wiki_synced_at after timestamp' do
-      subject.last_wiki_synced_at = Time.now + 2.hours
+      context 'wiki disabled' do
+        before do
+          project.update!(wiki_enabled: false)
+        end
 
-      expect(subject.wiki_synced_since?(Time.now)).to be true
+        it { is_expected.to be_falsy }
+      end
     end
   end
 end

spec/services/geo/repository_created_event_store_spec.rb

 require 'spec_helper'
 
 describe Geo::RepositoryCreatedEventStore do
-  let(:project) { create(:project) }
+  set(:project) { create(:project) }
 
-  subject(:event) { described_class.new(project) }
+  subject(:create!) { described_class.new(project).create }
 
   describe '#create' do
     it 'does not create an event when not running on a primary node' do
       allow(Gitlab::Geo).to receive(:primary?) { false }
 
-      expect { event.create }.not_to change(Geo::RepositoryCreatedEvent, :count)
+      expect { create! }.not_to change(Geo::RepositoryCreatedEvent, :count)
     end
 
-    context 'when running on a primary node' do
+    context 'running on a primary node' do
       before do
         allow(Gitlab::Geo).to receive(:primary?) { true }
       end
 
       it 'creates a created event' do
-        expect { event.create }.to change(Geo::RepositoryCreatedEvent, :count).by(1)
+        expect { create! }.to change(Geo::RepositoryCreatedEvent, :count).by(1)
       end
 
       it 'tracks information for the created project' do
-        event.create
+        create!
 
         event = Geo::RepositoryCreatedEvent.last
 
@@ -35,6 +35,15 @@ describe Geo::RepositoryCreatedEventStore do
           repository_storage_path: project.repository_storage_path
         )
       end
+
+      it 'does not set a wiki path if the wiki is disabled' do
+        project.update!(wiki_enabled: false)
+
+        create!
+
+        event = Geo::RepositoryCreatedEvent.last
+        expect(event.wiki_path).to be_nil
+      end
     end
   end
 end

spec/workers/geo/project_sync_worker_spec.rb

@@ -102,6 +102,23 @@ RSpec.describe Geo::ProjectSyncWorker do
       end
     end
 
+    context 'wiki is not enabled for project' do
+      let!(:registry) { create(:geo_project_registry, resync_repository: true, resync_wiki: true, project: project) }
+
+      before do
+        project.update!(wiki_enabled: false)
+        subject.perform(project.id, Time.now)
+      end
+
+      it 'syncs the project repository' do
+        expect(repository_sync_service).to have_received(:execute)
+      end
+
+      it 'does not sync the project wiki' do
+        expect(wiki_sync_service).not_to have_received(:execute)
+      end
+    end
+
     context 'when project repository was synced after the time the job was scheduled in' do
       it 'does not perform Geo::RepositorySyncService for the given project' do
         create(:geo_project_registry, :synced, :repository_dirty, project: project, last_repository_synced_at: Time.now)