Skip to content

G
gitlab-ce
Commit eb9437ba authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Remove code duplication on file registry finders 

The file registry finders uses the tracking tables
as SSOT and share the same logic.
parent 024ce684
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 444 deletions
ee/app/finders/geo/attachment_registry_finder.rb
View file @ eb9437ba
...@@ -2,30 +2,6 @@ ...@@ -2,30 +2,6 @@
module Geo module Geo
class AttachmentRegistryFinder < FileRegistryFinder class AttachmentRegistryFinder < FileRegistryFinder
def count_registry
syncable.count
end
def count_syncable
syncable.count
end
def count_synced
syncable.synced.count
end
def count_failed
syncable.failed.count
end
def count_synced_missing_on_primary
syncable.synced.missing_on_primary.count
end
def syncable
Geo::UploadRegistry
end
# Returns untracked uploads as well as tracked uploads that are unused. # Returns untracked uploads as well as tracked uploads that are unused.
# #
# Untracked uploads is an array where each item is a tuple of [id, file_type] # Untracked uploads is an array where each item is a tuple of [id, file_type]
...@@ -49,7 +25,7 @@ module Geo ...@@ -49,7 +25,7 @@ module Geo
def find_registry_differences(range) def find_registry_differences(range)
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
source = source =
attachments replicables
.id_in(range) .id_in(range)
.pluck(::Upload.arel_table[:id], ::Upload.arel_table[:uploader]) .pluck(::Upload.arel_table[:id], ::Upload.arel_table[:uploader])
.map! { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] } .map! { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
...@@ -66,61 +42,12 @@ module Geo ...@@ -66,61 +42,12 @@ module Geo
[untracked, unused_tracked] [untracked, unused_tracked]
end end
# Returns Geo::UploadRegistry records that have never been synced. def replicables
# current_node(fdw: false).attachments
# Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other
# processes need to ensure that the table only contains records that should
# be synced.
#
# Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like
# #find_retryable_failed_registries)
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
syncable
.never
.model_id_not_in(except_ids)
.limit(batch_size)
end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: [])
syncable
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
syncable
.synced
.missing_on_primary
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
private
def attachments
local_storage_only?(fdw: false) ? all_attachments.with_files_stored_locally : all_attachments
end end
def all_attachments def syncable
current_node(fdw: false).attachments Geo::UploadRegistry
end end
end end
end end
ee/app/finders/geo/file_registry_finder.rb
View file @ eb9437ba
...@@ -2,76 +2,157 @@ ...@@ -2,76 +2,157 @@
module Geo module Geo
class FileRegistryFinder < RegistryFinder class FileRegistryFinder < RegistryFinder
# @abstract Subclass is expected to implement the declared methods
# @!method syncable
# Return an ActiveRecord::Relation of tracked resource records, filtered
# by selective sync, with files stored locally
def syncable
raise NotImplementedError
end
# @!method count_syncable # @!method count_syncable
# Return a count of tracked resource records, filtered by selective # Return a count of the registry records for the tracked file_type(s)
# sync, with files stored locally
def count_syncable def count_syncable
raise NotImplementedError syncable.count
end end
# @!method count_synced # @!method count_synced
# Return a count of tracked resource records, filtered by selective # Return a count of the registry records for the tracked file_type(s)
# sync, with files stored locally, and are synced # that are synced
def count_synced def count_synced
raise NotImplementedError syncable.synced.count
end end
# @!method count_failed # @!method count_failed
# Return a count of tracked resource records, filtered by selective # Return a count of the registry records for the tracked file_type(s)
# sync, with files stored locally, and are failed # that are sync failed
def count_failed def count_failed
raise NotImplementedError syncable.failed.count
end end
# @!method count_synced_missing_on_primary # @!method count_synced_missing_on_primary
# Return a count of tracked resource records, filtered by selective # Return a count of the registry records for the tracked file_type(s)
# sync, with files stored locally, and are synced and missing on the # that are synced and missing on the primary
# primary
def count_synced_missing_on_primary def count_synced_missing_on_primary
raise NotImplementedError syncable.synced.missing_on_primary.count
end end
# @!method count_registry # @!method count_registry
# Return a count of the registry records for the tracked file_type(s) # Return a count of the registry records for the tracked file_type(s)
def count_registry def count_registry
raise NotImplementedError syncable.count
end end
# @!method find_unsynced # @!method find_registry_differences
# Return an ActiveRecord::Relation of not-yet-tracked resource records, # Returns untracked IDs as well as tracked IDs that are unused.
# filtered by selective sync, with files stored locally, excluding #
# specified IDs, limited to batch_size # Untracked IDs are model IDs that are supposed to be synced but don't yet
def find_unsynced # have a registry entry.
raise NotImplementedError #
# Unused tracked IDs are model IDs that are not supposed to be synced but
# already have a registry entry. For example:
#
# - orphaned registries
# - records that became excluded from selective sync
# - records that are in object storage, and `sync_object_storage` became
# disabled
#
# We compute both sets in this method to reduce the number of DB queries
# performed.
#
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range)
source = local_storage_only? ? replicables.with_files_stored_locally : replicables
source_ids = source.id_in(range).pluck(replicable_primary_key) # rubocop:disable CodeReuse/ActiveRecord
tracked_ids = syncable.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids
[untracked_ids, unused_tracked_ids]
end end
# @!method find_never_synced_registries
# Return an ActiveRecord::Relation of the registry records for the
# tracked file_type(s) that have never been synced.
#
# Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other
# processes need to ensure that the table only contains records that should
# be synced.
#
# Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like
# #find_retryable_failed_registries)
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
#
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
syncable
.never
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# @!method find_retryable_failed_registries # @!method find_retryable_failed_registries
# Return an ActiveRecord::Relation of registry records marked as failed, # Return an ActiveRecord::Relation of registry records marked as failed,
# which are ready to be retried, excluding specified IDs, limited to # which are ready to be retried, excluding specified IDs, limited to
# batch_size # batch_size
def find_retryable_failed_registries #
raise NotImplementedError # @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
#
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: [])
syncable
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end end
# rubocop:enable CodeReuse/ActiveRecord
# @!method find_retryable_synced_missing_on_primary_registries # @!method find_retryable_synced_missing_on_primary_registries
# Return an ActiveRecord::Relation of registry records marked as synced # Return an ActiveRecord::Relation of registry records marked as synced
# and missing on the primary, which are ready to be retried, excluding # and missing on the primary, which are ready to be retried, excluding
# specified IDs, limited to batch_size # specified IDs, limited to batch_size
def find_retryable_synced_missing_on_primary_registries #
raise NotImplementedError # @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
#
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
syncable
.synced
.missing_on_primary
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# @!method syncable
# Return an ActiveRecord::Base class for the tracked file_type(s)
def syncable
raise NotImplementedError,
"#{self.class} does not implement #{__method__}"
end
# @!method replicables
# Return an ActiveRecord::Relation of the replicable records for the
# tracked file_type(s)
def replicables
raise NotImplementedError,
"#{self.class} does not implement #{__method__}"
end
# @!method syncable
# Return the fully qualified name of the replicable primary key for the
# tracked file_type(s)
def replicable_primary_key
syncable::MODEL_CLASS.arel_table[:id]
end end
def local_storage_only?(fdw: true) def local_storage_only?
!current_node(fdw: fdw)&.sync_object_storage !current_node(fdw: false)&.sync_object_storage
end end
end end
end end
ee/app/finders/geo/job_artifact_registry_finder.rb
View file @ eb9437ba
...@@ -2,112 +2,12 @@ ...@@ -2,112 +2,12 @@
module Geo module Geo
class JobArtifactRegistryFinder < FileRegistryFinder class JobArtifactRegistryFinder < FileRegistryFinder
def count_registry def replicables
syncable.count current_node(fdw: false).job_artifacts.not_expired
end
def count_syncable
syncable.count
end
def count_synced
syncable.synced.count
end
def count_failed
syncable.failed.count
end
def count_synced_missing_on_primary
syncable.synced.missing_on_primary.count
end end
def syncable def syncable
Geo::JobArtifactRegistry Geo::JobArtifactRegistry
end end
# Returns untracked IDs as well as tracked IDs that are unused.
#
# Untracked IDs are model IDs that are supposed to be synced but don't yet
# have a registry entry.
#
# Unused tracked IDs are model IDs that are not supposed to be synced but
# already have a registry entry. For example:
#
# - orphaned registries
# - records that became excluded from selective sync
# - records that are in object storage, and `sync_object_storage` became
# disabled
#
# We compute both sets in this method to reduce the number of DB queries
# performed.
#
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range)
source_ids = job_artifacts.not_expired.id_in(range).pluck(::Ci::JobArtifact.arel_table[:id]) # rubocop:disable CodeReuse/ActiveRecord
tracked_ids = syncable.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids
[untracked_ids, unused_tracked_ids]
end
# Returns Geo::JobArtifactRegistry records that have never been synced.
#
# Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other
# processes need to ensure that the table only contains records that should
# be synced.
#
# Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like
# #find_retryable_failed_registries)
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
syncable
.never
.model_id_not_in(except_ids)
.limit(batch_size)
end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: [])
syncable
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
syncable
.synced
.missing_on_primary
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
private
def job_artifacts
local_storage_only?(fdw: false) ? all_job_artifacts.with_files_stored_locally : all_job_artifacts
end
def all_job_artifacts
current_node(fdw: false).job_artifacts
end
end end
end end
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @ eb9437ba
...@@ -2,112 +2,12 @@ ...@@ -2,112 +2,12 @@
module Geo module Geo
class LfsObjectRegistryFinder < FileRegistryFinder class LfsObjectRegistryFinder < FileRegistryFinder
def count_registry def replicables
syncable.count current_node(fdw: false).lfs_objects
end
def count_syncable
syncable.count
end
def count_synced
syncable.synced.count
end
def count_failed
syncable.failed.count
end
def count_synced_missing_on_primary
syncable.synced.missing_on_primary.count
end end
def syncable def syncable
Geo::LfsObjectRegistry Geo::LfsObjectRegistry
end end
# Returns untracked IDs as well as tracked IDs that are unused.
#
# Untracked IDs are model IDs that are supposed to be synced but don't yet
# have a registry entry.
#
# Unused tracked IDs are model IDs that are not supposed to be synced but
# already have a registry entry. For example:
#
# - orphaned registries
# - records that became excluded from selective sync
# - records that are in object storage, and `sync_object_storage` became
# disabled
#
# We compute both sets in this method to reduce the number of DB queries
# performed.
#
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range)
source_ids = lfs_objects.id_in(range).pluck_primary_key
tracked_ids = syncable.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids
[untracked_ids, unused_tracked_ids]
end
# Returns LfsObjectRegistry records that have never been synced.
#
# Does not care about selective sync, because it considers the Registry
# table to be the single source of truth. The contract is that other
# processes need to ensure that the table only contains records that should
# be synced.
#
# Any registries that have ever been synced that currently need to be
# resynced will be handled by other find methods (like
# #find_retryable_failed_registries)
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
syncable
.never
.model_id_not_in(except_ids)
.limit(batch_size)
end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: [])
syncable
.failed
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
syncable
.synced
.missing_on_primary
.retry_due
.model_id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
private
def lfs_objects
local_storage_only?(fdw: false) ? all_lfs_objects.with_files_stored_locally : all_lfs_objects
end
def all_lfs_objects
current_node(fdw: false).lfs_objects
end
end end
end end
ee/app/workers/geo/file_download_dispatch_worker/attachment_job_finder.rb
View file @ eb9437ba
...@@ -9,12 +9,6 @@ module Geo ...@@ -9,12 +9,6 @@ module Geo
@registry_finder ||= Geo::AttachmentRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id) @registry_finder ||= Geo::AttachmentRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id)
end end
def find_unsynced_jobs(batch_size:)
convert_registry_relation_to_job_args(
registry_finder.find_never_synced_registries(find_batch_params(batch_size))
)
end
private private
# Why do we need a different `file_type` for each Uploader? Why not just use 'upload'? # Why do we need a different `file_type` for each Uploader? Why not just use 'upload'?
......
ee/app/workers/geo/file_download_dispatch_worker/job_artifact_job_finder.rb
View file @ eb9437ba
...@@ -10,12 +10,6 @@ module Geo ...@@ -10,12 +10,6 @@ module Geo
def registry_finder def registry_finder
@registry_finder ||= Geo::JobArtifactRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id) @registry_finder ||= Geo::JobArtifactRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id)
end end
def find_unsynced_jobs(batch_size:)
convert_registry_relation_to_job_args(
registry_finder.find_never_synced_registries(find_batch_params(batch_size))
)
end
end end
end end
end end
ee/app/workers/geo/file_download_dispatch_worker/job_finder.rb
View file @ eb9437ba
...@@ -22,8 +22,8 @@ module Geo ...@@ -22,8 +22,8 @@ module Geo
end end
def find_unsynced_jobs(batch_size:) def find_unsynced_jobs(batch_size:)
convert_resource_relation_to_job_args( convert_registry_relation_to_job_args(
registry_finder.find_unsynced(find_batch_params(batch_size)) registry_finder.find_never_synced_registries(find_batch_params(batch_size))
) )
end end
......
ee/app/workers/geo/file_download_dispatch_worker/lfs_object_job_finder.rb
View file @ eb9437ba
...@@ -10,12 +10,6 @@ module Geo ...@@ -10,12 +10,6 @@ module Geo
def registry_finder def registry_finder
@registry_finder ||= Geo::LfsObjectRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id) @registry_finder ||= Geo::LfsObjectRegistryFinder.new(current_node_id: Gitlab::Geo.current_node.id)
end end
def find_unsynced_jobs(batch_size:)
convert_registry_relation_to_job_args(
registry_finder.find_never_synced_registries(find_batch_params(batch_size))
)
end
end end
end end
end end
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @ eb9437ba
...@@ -168,38 +168,6 @@ RSpec.describe Geo::AttachmentRegistryFinder, :geo do ...@@ -168,38 +168,6 @@ RSpec.describe Geo::AttachmentRegistryFinder, :geo do
end end
end end
describe '#find_unsynced' do
it 'returns registries for uploads that have never been synced' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
registry_upload_3 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
registries = subject.find_unsynced(batch_size: 10)
expect(registries).to match_ids(registry_upload_3, registry_upload_8)
end
it 'excludes except_ids' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
registry_upload_3 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
registries = subject.find_unsynced(batch_size: 10, except_ids: [registry_upload_3.file_id])
expect(registries).to match_ids(registry_upload_8)
end
end
describe '#find_retryable_failed_registries' do describe '#find_retryable_failed_registries' do
it 'returns registries for job artifacts that have failed to sync' do it 'returns registries for job artifacts that have failed to sync' do
registry_upload_1 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id) registry_upload_1 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
......
ee/spec/finders/geo/file_registry_finder_spec.rb
View file @ eb9437ba
...@@ -2,20 +2,13 @@ ...@@ -2,20 +2,13 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Geo::FileRegistryFinder, :geo, :geo_fdw do RSpec.describe Geo::FileRegistryFinder, :geo do
include ::EE::GeoHelpers include ::EE::GeoHelpers
context 'with abstract methods' do context 'with abstract methods' do
%w[ %w[
replicables
syncable syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
].each do |required_method| ].each do |required_method|
it "requires subclasses to implement #{required_method}" do it "requires subclasses to implement #{required_method}" do
expect { subject.send(required_method) }.to raise_error(NotImplementedError) expect { subject.send(required_method) }.to raise_error(NotImplementedError)
......
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
View file @ eb9437ba
...@@ -353,38 +353,6 @@ RSpec.describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -353,38 +353,6 @@ RSpec.describe Geo::JobArtifactRegistryFinder, :geo do
end end
end end
describe '#find_unsynced' do
it 'returns registries for job artifacts that have never been synced' do
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_2.id, missing_on_primary: true)
registry_ci_job_artifact_3 = create(:geo_job_artifact_registry, :never_synced, artifact_id: ci_job_artifact_3.id)
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_4.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_remote_2.id, missing_on_primary: true)
registry_ci_job_artifact_remote_3 = create(:geo_job_artifact_registry, :never_synced, artifact_id: ci_job_artifact_remote_3.id)
registries = subject.find_unsynced(batch_size: 10)
expect(registries).to match_ids(registry_ci_job_artifact_3, registry_ci_job_artifact_remote_3)
end
it 'excludes except_ids' do
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_2.id, missing_on_primary: true)
create(:geo_job_artifact_registry, :never_synced, artifact_id: ci_job_artifact_3.id)
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_4.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: ci_job_artifact_remote_2.id, missing_on_primary: true)
registry_ci_job_artifact_remote_3 = create(:geo_job_artifact_registry, :never_synced, artifact_id: ci_job_artifact_remote_3.id)
registries = subject.find_unsynced(batch_size: 10, except_ids: [ci_job_artifact_3.id])
expect(registries).to match_ids(registry_ci_job_artifact_remote_3)
end
end
describe '#find_retryable_failed_registries' do describe '#find_retryable_failed_registries' do
it 'returns registries for job artifacts that have failed to sync' do it 'returns registries for job artifacts that have failed to sync' do
registry_ci_job_artifact_1 = create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_1.id) registry_ci_job_artifact_1 = create(:geo_job_artifact_registry, :failed, artifact_id: ci_job_artifact_1.id)
......
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ eb9437ba
...@@ -294,39 +294,7 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo do ...@@ -294,39 +294,7 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id) registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id]) registries = subject.find_never_synced_registries(batch_size: 10, except_ids: [lfs_object_3.id])
expect(registries).to match_ids(registry_lfs_object_remote_3)
end
end
describe '#find_unsynced' do
it 'returns registries for LFS objects that have never been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
registries = subject.find_unsynced(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
it 'excludes except_ids' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
expect(registries).to match_ids(registry_lfs_object_remote_3) expect(registries).to match_ids(registry_lfs_object_remote_3)
end end
......
ee/spec/support/shared_examples/finders/geo/file_registry_finders_shared_examples.rb
View file @ eb9437ba
...@@ -9,7 +9,6 @@ RSpec.shared_examples 'a file registry finder' do ...@@ -9,7 +9,6 @@ RSpec.shared_examples 'a file registry finder' do
count_failed count_failed
count_synced_missing_on_primary count_synced_missing_on_primary
count_registry count_registry
find_unsynced
find_retryable_failed_registries find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries find_retryable_synced_missing_on_primary_registries
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7