Use FDW to find unsynced projects when feature flag is enabled

parent dc726031
...@@ -72,18 +72,11 @@ module Geo ...@@ -72,18 +72,11 @@ module Geo
.execute .execute
end end
# rubocop: disable CodeReuse/ActiveRecord def find_unsynced_projects(shard_name:, batch_size:)
def find_unsynced_projects(batch_size:) finder_klass_for_unsynced_projects
relation = .new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
if use_legacy_queries? .execute
legacy_find_unsynced_projects
else
fdw_find_unsynced_projects
end
relation.limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_projects_updated_recently(batch_size:) def find_projects_updated_recently(batch_size:)
...@@ -100,18 +93,6 @@ module Geo ...@@ -100,18 +93,6 @@ module Geo
protected protected
#
# FDW accessors
#
# @return [ActiveRecord::Relation<Geo::Fdw::Project>]
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_unsynced_projects
Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_project_table.name}.id")
.where(project_registry: { project_id: nil })
end
# rubocop: enable CodeReuse/ActiveRecord
# @return [ActiveRecord::Relation<Geo::Fdw::Project>] # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def fdw_find_projects_updated_recently def fdw_find_projects_updated_recently
...@@ -121,21 +102,6 @@ module Geo ...@@ -121,21 +102,6 @@ module Geo
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
#
# Legacy accessors (non FDW)
#
# @return [ActiveRecord::Relation<Project>] list of unsynced projects
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_unsynced_projects
legacy_left_outer_join_registry_ids(
current_node.projects,
Geo::ProjectRegistry.pluck(:project_id),
Project
)
end
# rubocop: enable CodeReuse/ActiveRecord
# @return [ActiveRecord::Relation<Project>] list of projects updated recently # @return [ActiveRecord::Relation<Project>] list of projects updated recently
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def legacy_find_projects_updated_recently def legacy_find_projects_updated_recently
...@@ -171,6 +137,14 @@ module Geo ...@@ -171,6 +137,14 @@ module Geo
fdw_disabled? || selective_sync? && !Gitlab::Geo::Fdw.enabled_for_selective_sync? fdw_disabled? || selective_sync? && !Gitlab::Geo::Fdw.enabled_for_selective_sync?
end end
def finder_klass_for_unsynced_projects
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectUnsyncedFinder
else
Geo::ProjectUnsyncedFinder
end
end
def finder_klass_for_synced_registries def finder_klass_for_synced_registries
if use_legacy_queries_for_selective_sync? if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistrySyncedFinder Geo::LegacyProjectRegistrySyncedFinder
......
...@@ -77,19 +77,19 @@ module Geo ...@@ -77,19 +77,19 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_not_synced(batch_size:) def find_project_ids_not_synced(batch_size:)
shard_restriction(finder.find_unsynced_projects(batch_size: batch_size)) finder.find_unsynced_projects(shard_name: shard_name, batch_size: batch_size)
.where.not(id: scheduled_project_ids) .id_not_in(scheduled_project_ids)
.reorder(last_repository_updated_at: :desc) .reorder(last_repository_updated_at: :desc)
.pluck(:id) .pluck_primary_key
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(batch_size:) def find_project_ids_updated_recently(batch_size:)
shard_restriction(finder.find_projects_updated_recently(batch_size: batch_size)) shard_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
.where.not(id: scheduled_project_ids) .id_not_in(scheduled_project_ids)
.order('project_registry.last_repository_synced_at ASC NULLS FIRST, projects.last_repository_updated_at ASC') .order('project_registry.last_repository_synced_at ASC NULLS FIRST, projects.last_repository_updated_at ASC')
.pluck(:id) .pluck_primary_key
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
......
...@@ -380,43 +380,6 @@ describe Geo::ProjectRegistryFinder, :geo do ...@@ -380,43 +380,6 @@ describe Geo::ProjectRegistryFinder, :geo do
end end
shared_examples 'finds all the things' do |method_prefix| shared_examples 'finds all the things' do |method_prefix|
describe '#find_unsynced_projects' do
it 'delegates to the correct method' do
expect(subject).to receive("#{method_prefix}_find_unsynced_projects".to_sym).and_call_original
subject.find_unsynced_projects(batch_size: 10)
end
it 'returns projects without an entry on the tracking database' do
project_not_synced = create(:project)
create(:geo_project_registry, :synced, :repository_dirty, project: project_1_in_synced_group)
projects = subject.find_unsynced_projects(batch_size: 10)
expect(projects).to match_ids(project_not_synced)
end
context 'with selective sync' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_unsynced_projects' do
expect(subject).to receive(:legacy_find_unsynced_projects).and_call_original
subject.find_unsynced_projects(batch_size: 10)
end
it 'returns untracked projects in the synced group' do
create(:geo_project_registry, :sync_failed, project: project_1_in_synced_group)
projects = subject.find_unsynced_projects(batch_size: 10)
expect(projects).to match_ids(project_2_in_synced_group)
end
end
end
describe '#find_projects_updated_recently' do describe '#find_projects_updated_recently' do
it 'delegates to the correct method' do it 'delegates to the correct method' do
expect(subject).to receive("#{method_prefix}_find_projects_updated_recently".to_sym).and_call_original expect(subject).to receive("#{method_prefix}_find_projects_updated_recently".to_sym).and_call_original
...@@ -522,6 +485,13 @@ describe Geo::ProjectRegistryFinder, :geo do ...@@ -522,6 +485,13 @@ describe Geo::ProjectRegistryFinder, :geo do
include_examples 'finds all the things', 'legacy' include_examples 'finds all the things', 'legacy'
end end
describe '#find_unsynced_projects', :delete do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectUnsyncedFinder,
Geo::ProjectUnsyncedFinder,
:find_unsynced_projects, [shard_name: 'default', batch_size: 100]
end
describe '#find_failed_project_registries', :delete do describe '#find_failed_project_registries', :delete do
include_examples 'delegates to the proper finder', include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistrySyncFailedFinder, Geo::LegacyProjectRegistrySyncFailedFinder,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment