Merge branch '4050-log-cursor-full-scan' into 'master'

Remove the full-scan option from the Geo log cursor

Closes #4050

See merge request gitlab-org/gitlab-ee!3412

bin/geo_log_cursor

@@ -6,14 +6,13 @@ require 'optparse'
 
 class GeoLogCursorOptionParser
   def self.parse(argv)
-    options = { full_scan: false }
+    options = {}
 
     op = OptionParser.new
     op.banner = 'GitLab Geo: Log Cursor'
     op.separator ''
     op.separator 'Usage: ./geo_log_cursor [options]'
     op.separator ''
-    op.on('-f', '--full-scan', 'Performs full-scan to lookup for un-replicated data') { options[:full_scan] = true }
     op.on('-d', '--debug', 'Enable detailed logging with extra debug information') { options[:debug] = true }
 
     op.separator 'Common options:'

changelogs/unreleased-ee/4050-log-cursor-full-scan.yml

+---
+title: Remove the full-scan option from the Geo log cursor
+merge_request: 3412
+author:
+type: removed

lib/gitlab/geo/log_cursor/daemon.rb

@@ -16,8 +16,6 @@ module Gitlab
         def run!
           trap_signals
 
-          full_scan! if options[:full_scan]
-
           until exit?
             lease = Lease.try_obtain_with_ttl { run_once! }
 
@@ -32,35 +30,6 @@ module Gitlab
           LogCursor::Events.fetch_in_batches { |batch| handle_events(batch) }
         end
 
-        # Execute routines to verify the required initial data is available
-        # and mark non-replicated data as requiring replication.
-        def full_scan!
-          # This is slow and can be improved in the future by using PostgreSQL FDW
-          # so we can query with a LEFT JOIN and have a list of
-          # Projects without corresponding ProjectRegistry in the DR database
-          # See: https://robots.thoughtbot.com/postgres-foreign-data-wrapper (requires PG 9.6)
-          $stdout.print 'Searching for non replicated projects...'
-
-          Gitlab::Geo.current_node.projects.select(:id).find_in_batches(batch_size: BATCH_SIZE) do |batch|
-            $stdout.print '.'
-
-            project_ids = batch.map(&:id)
-            existing = ::Geo::ProjectRegistry.where(project_id: project_ids).pluck(:project_id)
-            missing_projects = project_ids - existing
-
-            logger.info(
-              "Missing projects",
-              projects: missing_projects,
-              project_count: missing_projects.count)
-
-            missing_projects.each do |id|
-              ::Geo::ProjectRegistry.create(project_id: id)
-            end
-          end
-          $stdout.puts 'Done!'
-          puts
-        end
-
         def handle_events(batch)
           batch.each do |event_log|
             next unless can_replay?(event_log)

spec/bin/geo_log_cursor_spec.rb

-require 'spec_helper'
-
-load File.expand_path('../../bin/geo_log_cursor', __dir__)
-
-describe 'scripts/geo_log_cursor' do
-  describe GeoLogCursorOptionParser do
-    it 'parses -f and --full-scan' do
-      %w[-f --full-scan].each do |flag|
-        options = described_class.parse(%W[foo #{flag} bar])
-
-        expect(options[:full_scan]).to eq true
-      end
-    end
-  end
-end

spec/lib/gitlab/geo/log_cursor/daemon_spec.rb

@@ -28,24 +28,6 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
       daemon.run!
     end
 
-    it 'does not perform a full scan by default' do
-      is_expected.to receive(:exit?).and_return(true)
-      is_expected.not_to receive(:full_scan!)
-
-      daemon.run!
-    end
-
-    context 'the command-line defines full_scan: true' do
-      let(:options) { { full_scan: true } }
-
-      it 'executes a full-scan' do
-        is_expected.to receive(:exit?).and_return(true)
-        is_expected.to receive(:full_scan!)
-
-        daemon.run!
-      end
-    end
-
     it 'delegates to #run_once! in a loop' do
       is_expected.to receive(:exit?).and_return(false, false, false, true)
       is_expected.to receive(:run_once!).twice
@@ -246,22 +228,4 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
       end
     end
   end
-
-  describe '#full_scan!' do
-    let(:project) { create(:project) }
-
-    context 'with selective sync enabled' do
-      it 'creates registries for missing projects that belong to selected namespaces' do
-        secondary.update!(namespaces: [project.namespace])
-
-        expect { daemon.full_scan! }.to change(Geo::ProjectRegistry, :count).by(1)
-      end
-
-      it 'does not create registries for missing projects that do not belong to selected namespaces' do
-        secondary.update!(namespaces: [create(:group)])
-
-        expect { daemon.full_scan! }.not_to change(Geo::ProjectRegistry, :count)
-      end
-    end
-  end
 end