Use FDW to find projects updated recently when feature flag is enabled

parent 72d068a1
......@@ -78,53 +78,10 @@ module Geo
.execute
end
# rubocop: disable CodeReuse/ActiveRecord
def find_projects_updated_recently(batch_size:)
relation =
if use_legacy_queries?
legacy_find_projects_updated_recently
else
fdw_find_projects_updated_recently
end
relation.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
protected
# @return [ActiveRecord::Relation<Geo::Fdw::Project>]
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_projects_updated_recently
Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_project_table.name}.id")
.merge(Geo::ProjectRegistry.dirty)
.merge(Geo::ProjectRegistry.retry_due)
end
# rubocop: enable CodeReuse/ActiveRecord
# @return [ActiveRecord::Relation<Project>] list of projects updated recently
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_projects_updated_recently
registries = Geo::ProjectRegistry.dirty.retry_due.pluck(:project_id, :last_repository_synced_at)
return Project.none if registries.empty?
id_and_last_sync_values = registries.map do |id, last_repository_synced_at|
"(#{id}, #{quote_value(last_repository_synced_at)})"
end
joined_relation = current_node.projects.joins(<<~SQL)
INNER JOIN
(VALUES #{id_and_last_sync_values.join(',')})
project_registry(id, last_repository_synced_at)
ON #{Project.table_name}.id = project_registry.id
SQL
joined_relation
end
# rubocop: enable CodeReuse/ActiveRecord
def fdw_project_table
Geo::Fdw::Project.arel_table
def find_projects_updated_recently(shard_name:, batch_size:)
finder_klass_for_projects_updated_recently
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
private
......@@ -145,6 +102,14 @@ module Geo
end
end
def finder_klass_for_projects_updated_recently
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectUpdatedRecentlyFinder
else
Geo::ProjectUpdatedRecentlyFinder
end
end
def finder_klass_for_synced_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistrySyncedFinder
......
......@@ -86,17 +86,11 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(batch_size:)
shard_restriction(finder.find_projects_updated_recently(batch_size: batch_size))
finder.find_projects_updated_recently(shard_name: shard_name, batch_size: batch_size)
.id_not_in(scheduled_project_ids)
.order('project_registry.last_repository_synced_at ASC NULLS FIRST, projects.last_repository_updated_at ASC')
.pluck_primary_key
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def shard_restriction(relation)
relation.where(repository_storage: shard_name)
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
......@@ -379,48 +379,6 @@ describe Geo::ProjectRegistryFinder, :geo do
end
end
shared_examples 'finds all the things' do |method_prefix|
describe '#find_projects_updated_recently' do
it 'delegates to the correct method' do
expect(subject).to receive("#{method_prefix}_find_projects_updated_recently".to_sym).and_call_original
subject.find_projects_updated_recently(batch_size: 10)
end
it 'returns projects with a dirty entry on the tracking database' do
create(:geo_project_registry, :synced, :repository_dirty, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_2_in_synced_group)
projects = subject.find_projects_updated_recently(batch_size: 10)
expect(projects).to match_ids([project_1_in_synced_group, project_2_in_synced_group])
end
context 'with selective sync' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_projects_updated_recently' do
expect(subject).to receive(:legacy_find_projects_updated_recently).and_call_original
subject.find_projects_updated_recently(batch_size: 10)
end
it 'returns dirty projects in the synced group' do
create(:project, group: synced_group)
create(:geo_project_registry, :synced, :repository_dirty, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_2_in_synced_group)
create(:geo_project_registry, :synced, project: project_3_in_synced_group)
projects = subject.find_projects_updated_recently(batch_size: 10)
expect(projects).to match_ids(project_1_in_synced_group, project_2_in_synced_group)
end
end
end
end
shared_examples 'delegates to the proper finder' do |legacy_finder_klass, finder_klass, method, args|
where(:selective_sync, :fdw_enabled, :fdw_for_selective_sync, :finder) do
false | false | false | legacy_finder_klass
......@@ -463,7 +421,6 @@ describe Geo::ProjectRegistryFinder, :geo do
end
include_examples 'counts all the things', 'fdw'
include_examples 'finds all the things', 'fdw'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
......@@ -472,7 +429,6 @@ describe Geo::ProjectRegistryFinder, :geo do
end
include_examples 'counts all the things', 'fdw'
include_examples 'finds all the things', 'fdw'
end
end
......@@ -482,7 +438,6 @@ describe Geo::ProjectRegistryFinder, :geo do
end
include_examples 'counts all the things', 'legacy'
include_examples 'finds all the things', 'legacy'
end
describe '#find_unsynced_projects', :delete do
......@@ -492,6 +447,13 @@ describe Geo::ProjectRegistryFinder, :geo do
:find_unsynced_projects, [shard_name: 'default', batch_size: 100]
end
describe '#find_projects_updated_recently', :delete do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectUpdatedRecentlyFinder,
Geo::ProjectUpdatedRecentlyFinder,
:find_projects_updated_recently, [shard_name: 'default', batch_size: 100]
end
describe '#find_failed_project_registries', :delete do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistrySyncFailedFinder,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment