Commit e0622d3f authored by Stan Hu's avatar Stan Hu

Make FDW Geo download code DRY

parent 4f418a1c
# This module is intended to centralize all database access to the secondary # This module is intended to centralize all database access to the secondary
# tracking database for Geo. # tracking database for Geo.
module Geo module Geo
class TrackingBase < ActiveRecord::Base class TrackingBase < ActiveRecord::Base
self.abstract_class = true self.abstract_class = true
......
...@@ -33,24 +33,40 @@ module Geo ...@@ -33,24 +33,40 @@ module Geo
.pluck(:file_id, :file_type) .pluck(:file_id, :file_type)
end end
def selective_sync
current_node.restricted_project_ids
end
def find_lfs_object_ids def find_lfs_object_ids
# Selective project replication adds a wrinkle to FDW queries, so # Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now. # we fallback to the legacy version for now.
if Gitlab::Geo.fdw? && !current_node.restricted_project_ids relation =
fdw_find_lfs_object_ids if Gitlab::Geo.fdw? && !selective_sync
else fdw_find_lfs_object_ids
legacy_find_lfs_object_ids else
end legacy_find_lfs_object_ids
end
relation
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end end
def find_upload_object_ids def find_upload_object_ids
# Selective project replication adds a wrinkle to FDW queries, so # Selective project replication adds a wrinkle to FDW queries, so
# we fallback to the legacy version for now. # we fallback to the legacy version for now.
if Gitlab::Geo.fdw? && !current_node.restricted_project_ids relation =
fdw_find_upload_object_ids if Gitlab::Geo.fdw? && !selective_sync
else fdw_find_upload_object_ids
legacy_find_upload_object_ids else
end legacy_find_upload_object_ids
end
relation
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end end
def fdw_find_lfs_object_ids def fdw_find_lfs_object_ids
...@@ -60,10 +76,6 @@ module Geo ...@@ -60,10 +76,6 @@ module Geo
Geo::Fdw::LfsObject.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type = 'lfs'") Geo::Fdw::LfsObject.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type = 'lfs'")
.where("#{fdw_table}.file_store IS NULL OR #{fdw_table}.file_store = #{LfsObjectUploader::LOCAL_STORE}") .where("#{fdw_table}.file_store IS NULL OR #{fdw_table}.file_store = #{LfsObjectUploader::LOCAL_STORE}")
.where('file_registry.file_id IS NULL') .where('file_registry.file_id IS NULL')
.order(created_at: :desc)
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end end
def fdw_find_upload_object_ids def fdw_find_upload_object_ids
...@@ -73,35 +85,22 @@ module Geo ...@@ -73,35 +85,22 @@ module Geo
Geo::Fdw::Upload.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type IN (#{obj_types})") Geo::Fdw::Upload.joins("LEFT OUTER JOIN file_registry ON file_registry.file_id = #{fdw_table}.id AND file_registry.file_type IN (#{obj_types})")
.where('file_registry.file_id IS NULL') .where('file_registry.file_id IS NULL')
.order(created_at: :desc) .order(created_at: :desc)
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end end
def legacy_find_upload_object_ids def legacy_find_upload_object_ids
unsynced_downloads = legacy_filter_registry_ids( legacy_filter_registry_ids(
current_node.uploads, current_node.uploads,
Geo::FileService::DEFAULT_OBJECT_TYPES, Geo::FileService::DEFAULT_OBJECT_TYPES,
Upload.table_name Upload.table_name
) )
unsynced_downloads
.limit(db_retrieve_batch_size)
.pluck(:id, :uploader)
.map { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
end end
def legacy_find_lfs_object_ids def legacy_find_lfs_object_ids
unsynced_downloads = legacy_filter_registry_ids( legacy_filter_registry_ids(
current_node.lfs_objects, current_node.lfs_objects,
[:lfs], [:lfs],
LfsObject.table_name LfsObject.table_name
) )
unsynced_downloads
.limit(db_retrieve_batch_size)
.pluck(:id)
.map { |id| [id, :lfs] }
end end
# This query requires data from two different databases, and unavoidably # This query requires data from two different databases, and unavoidably
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment