Skip to content

G
gitlab-ce
Commit 67ed817f authored by Michael Kozono's avatar Michael Kozono
Browse files
Options

Merge branch '205128-geo-continue-to-make-registry-table-ssot-for-lfs-objects' into 'master' 

Geo - Make registry table SSOT for LFS objects

See merge request gitlab-org/gitlab!33432
parents 81f0e592 c3e0176f
Show whitespace changes
Inline Side-by-side
Showing with 296 additions and 752 deletions
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @ 67ed817f
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
module Geo module Geo
class LfsObjectRegistryFinder < FileRegistryFinder class LfsObjectRegistryFinder < FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def count_registry def count_registry
Geo::LfsObjectRegistry.count syncable.count
end end
def count_syncable def count_syncable
...@@ -13,22 +11,19 @@ module Geo ...@@ -13,22 +11,19 @@ module Geo
end end
def count_synced def count_synced
lfs_objects.synced.count syncable.synced.count
end end
def count_failed def count_failed
lfs_objects.failed.count syncable.failed.count
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
lfs_objects.synced.missing_on_primary.count syncable.synced.missing_on_primary.count
end end
def syncable def syncable
return lfs_objects if selective_sync? Geo::LfsObjectRegistry
return LfsObject.with_files_stored_locally if local_storage_only?
LfsObject
end end
# Returns untracked IDs as well as tracked IDs that are unused. # Returns untracked IDs as well as tracked IDs that are unused.
...@@ -49,14 +44,8 @@ module Geo ...@@ -49,14 +44,8 @@ module Geo
# #
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused # @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range) def find_registry_differences(range)
source_ids = source_ids = lfs_objects.id_in(range).pluck_primary_key
lfs_objects(fdw: false) tracked_ids = syncable.pluck_model_ids_in_range(range)
.id_in(range)
.pluck_primary_key
tracked_ids =
Geo::LfsObjectRegistry
.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids unused_tracked_ids = tracked_ids - source_ids
...@@ -82,47 +71,19 @@ module Geo ...@@ -82,47 +71,19 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: []) def find_never_synced_registries(batch_size:, except_ids: [])
Geo::LfsObjectRegistry syncable
.never .never
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop:enable CodeReuse/ActiveRecord alias_method :find_unsynced, :find_never_synced_registries
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated lfs objects.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
lfs_objects
.missing_registry
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_ids: [])
all_lfs_objects
.inner_join_registry
.with_files_stored_remotely
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: []) def find_retryable_failed_registries(batch_size:, except_ids: [])
registries_for_lfs_objects syncable
.merge(Geo::LfsObjectRegistry.failed) .failed
.merge(Geo::LfsObjectRegistry.retry_due) .retry_due
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
...@@ -130,7 +91,7 @@ module Geo ...@@ -130,7 +91,7 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
registries_for_lfs_objects syncable
.synced .synced
.missing_on_primary .missing_on_primary
.retry_due .retry_due
...@@ -141,16 +102,12 @@ module Geo ...@@ -141,16 +102,12 @@ module Geo
private private
def lfs_objects(fdw: true) def lfs_objects
local_storage_only?(fdw: fdw) ? all_lfs_objects(fdw: fdw).with_files_stored_locally : all_lfs_objects(fdw: fdw) local_storage_only?(fdw: false) ? all_lfs_objects.with_files_stored_locally : all_lfs_objects
end
def all_lfs_objects(fdw: true)
current_node(fdw: fdw).lfs_objects
end end
def registries_for_lfs_objects def all_lfs_objects
current_node.lfs_object_registries current_node(fdw: false).lfs_objects
end end
end end
end end
ee/app/workers/geo/migrated_local_files_clean_up_worker.rb
View file @ 67ed817f
...@@ -37,23 +37,12 @@ module Geo ...@@ -37,23 +37,12 @@ module Geo
end end
def find_migrated_local_objects(batch_size:) def find_migrated_local_objects(batch_size:)
lfs_object_ids = find_migrated_local_lfs_objects_ids(batch_size: batch_size)
attachment_ids = find_migrated_local_attachments_ids(batch_size: batch_size) attachment_ids = find_migrated_local_attachments_ids(batch_size: batch_size)
job_artifact_ids = find_migrated_local_job_artifacts_ids(batch_size: batch_size) job_artifact_ids = find_migrated_local_job_artifacts_ids(batch_size: batch_size)
take_batch(lfs_object_ids, attachment_ids, job_artifact_ids) take_batch(attachment_ids, job_artifact_ids)
end end
# rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local_lfs_objects_ids(batch_size:)
return [] unless lfs_objects_object_store_enabled?
lfs_objects_finder.find_migrated_local(batch_size: batch_size, except_ids: scheduled_file_ids(:lfs))
.pluck(Geo::Fdw::LfsObject.arel_table[:id])
.map { |id| ['lfs', id] }
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local_attachments_ids(batch_size:) def find_migrated_local_attachments_ids(batch_size:)
return [] unless attachments_object_store_enabled? return [] unless attachments_object_store_enabled?
...@@ -85,17 +74,12 @@ module Geo ...@@ -85,17 +74,12 @@ module Geo
FileUploader.object_store_enabled? FileUploader.object_store_enabled?
end end
def lfs_objects_object_store_enabled?
LfsObjectUploader.object_store_enabled?
end
def job_artifacts_object_store_enabled? def job_artifacts_object_store_enabled?
JobArtifactUploader.object_store_enabled? JobArtifactUploader.object_store_enabled?
end end
def object_store_enabled? def object_store_enabled?
attachments_object_store_enabled? || attachments_object_store_enabled? ||
lfs_objects_object_store_enabled? ||
job_artifacts_object_store_enabled? job_artifacts_object_store_enabled?
end end
...@@ -107,10 +91,6 @@ module Geo ...@@ -107,10 +91,6 @@ module Geo
@attachments_finder ||= AttachmentRegistryFinder.new(current_node_id: current_node.id) @attachments_finder ||= AttachmentRegistryFinder.new(current_node_id: current_node.id)
end end
def lfs_objects_finder
@lfs_objects_finder ||= LfsObjectRegistryFinder.new(current_node_id: current_node.id)
end
def job_artifacts_finder def job_artifacts_finder
@job_artifacts_finder ||= JobArtifactRegistryFinder.new(current_node_id: current_node.id) @job_artifacts_finder ||= JobArtifactRegistryFinder.new(current_node_id: current_node.id)
end end
......
ee/changelogs/unreleased/205128-geo-continue-to-make-registry-table-ssot-for-lfs-objects.yml 0 → 100644
View file @ 67ed817f
---
title: Geo - Make registry table SSOT for LFS objects
merge_request: 33432
author:
type: performance
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ 67ed817f
...@@ -2,25 +2,10 @@ ...@@ -2,25 +2,10 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do RSpec.describe Geo::LfsObjectRegistryFinder, :geo do
include ::EE::GeoHelpers let_it_be(:secondary) { create(:geo_node) }
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let(:secondary) { create(:geo_node) }
let(:synced_group) { create(:group) }
let(:nested_group_1) { create(:group, parent: synced_group) }
let(:synced_project) { create(:project, group: synced_group) }
let(:synced_project_in_nested_group) { create(:project, group: nested_group_1) }
let(:unsynced_project) { create(:project) }
let(:project_broken_storage) { create(:project, :broken_storage) }
subject { described_class.new(current_node_id: secondary.id) }
before do before do
stub_current_geo_node(secondary)
stub_lfs_object_storage stub_lfs_object_storage
end end
...@@ -33,257 +18,97 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do ...@@ -33,257 +18,97 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
let!(:lfs_object_remote_2) { create(:lfs_object, :object_storage) } let!(:lfs_object_remote_2) { create(:lfs_object, :object_storage) }
let!(:lfs_object_remote_3) { create(:lfs_object, :object_storage) } let!(:lfs_object_remote_3) { create(:lfs_object, :object_storage) }
context 'counts all the things' do subject { described_class.new(current_node_id: secondary.id) }
describe '#count_syncable' do
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1) describe '#count_syncable' do
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2) it 'counts registries for LFS objects' do
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 8 expect(subject.count_syncable).to eq 8
end end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects' do
expect(subject.count_syncable).to eq 2
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects ignoring remote objects' do
expect(subject.count_syncable).to eq 5
end
end
end end
describe '#count_registry' do describe '#count_registry' do
before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
create(:geo_upload_registry, :avatar)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_1)
end
it 'counts registries for LFS objects' do it 'counts registries for LFS objects' do
expect(subject.count_registry).to eq 4 create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
end create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
context 'with selective sync by namespace' do create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
it 'does not apply the selective sync restriction' do create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
expect(subject.count_registry).to eq 4 create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 4
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts registries for LFS objects' do expect(subject.count_registry).to eq 8
expect(subject.count_registry).to eq 4
end
end end
end end
describe '#count_synced' do describe '#count_synced' do
before do it 'counts registries that has been synced' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
end
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that has been synced' do
expect(subject.count_synced).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that has been synced ignoring remote objects' do
expect(subject.count_synced).to eq 3
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that has been synced, ignoring remotes' do
expect(subject.count_synced).to eq 3 expect(subject.count_synced).to eq 3
end end
end end
end
describe '#count_failed' do describe '#count_failed' do
before do it 'counts registries that sync has failed' do
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_5.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 4
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that sync has failed' do
expect(subject.count_failed).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that sync has failed, ignoring remotes' do
expect(subject.count_failed).to eq 3 expect(subject.count_failed).to eq 3
end end
end end
end
describe '#count_synced_missing_on_primary' do describe '#count_synced_missing_on_primary' do
before do it 'counts registries that have been synced and are missing on the primary, excluding not synced ones' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, missing_on_primary: true) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id, missing_on_primary: true) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
allow(instance).to receive(:update_project_statistics).and_return(nil) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
end create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'counts LFS objects that has been synced' do
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts LFS objects that have been synced and are missing on the primary, excluding not synced ones' do expect(subject.count_synced_missing_on_primary).to eq 3
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
end end
end end
context 'finds all the things' do
describe '#find_registry_differences' do describe '#find_registry_differences' do
let_it_be(:synced_group) { create(:group) }
let_it_be(:nested_group_1) { create(:group, parent: synced_group) }
let_it_be(:synced_project) { create(:project, group: synced_group) }
let_it_be(:synced_project_in_nested_group) { create(:project, group: nested_group_1) }
let_it_be(:unsynced_project) { create(:project) }
let_it_be(:project_broken_storage) { create(:project, :broken_storage) }
context 'untracked IDs' do context 'untracked IDs' do
before do before do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
allow_any_instance_of(LfsObjectsProject).to receive(:update_project_statistics).and_return(nil)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1) create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2) create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3) create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
...@@ -444,284 +269,130 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do ...@@ -444,284 +269,130 @@ RSpec.describe Geo::LfsObjectRegistryFinder, :geo_fdw do
end end
describe '#find_never_synced_registries' do describe '#find_never_synced_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have never been synced' do it 'returns registries for LFS objects that have never been synced' do
registries = subject.find_never_synced_registries(batch_size: 10) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1) registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
end create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
describe '#find_unsynced' do create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
before do registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_5)
end
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5,
lfs_object_remote_1, lfs_object_remote_2, lfs_object_remote_3)
end
it 'excludes LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_2.id])
expect(lfs_objects).to match_ids(lfs_object_5, lfs_object_remote_1,
lfs_object_remote_2, lfs_object_remote_3)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts LFS objects that sync has failed' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_5)
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'returns LFS objects without an entry on the tracking database' do
lfs_objects = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_2, lfs_object_5)
end
end
end
describe '#find_migrated_local' do
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_1)
end
it 'excludes LFS objects stored remotely, but not synced yet' do
create(:lfs_object, :object_storage)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to be_empty
end
it 'excludes synced LFS objects that are stored locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) registries = subject.find_never_synced_registries(batch_size: 10)
expect(lfs_objects).to be_empty expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end end
it 'excludes except_ids' do it 'excludes except_ids' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10, except_ids: [lfs_object_remote_1.id]) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
expect(lfs_objects).to match_ids(lfs_object_remote_2) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
end create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_remote_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_remote_3)
end
it 'returns LFS objects remotely and successfully synced locally' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_2)
end
end
context 'with selective sync by shard' do registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do expect(registries).to match_ids(registry_lfs_object_remote_3)
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_3)
end end
it 'returns LFS objects remotely and successfully synced locally' do describe '#find_unsynced' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) it 'returns registries for LFS objects that have never been synced' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
registry_lfs_object_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) registries = subject.find_unsynced(batch_size: 10)
expect(lfs_objects).to match_ids(lfs_object_remote_2) expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_remote_3)
end
end end
context 'with object storage sync disabled' do it 'excludes except_ids' do
let(:secondary) { create(:geo_node, :local_storage_only) } create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
it 'returns LFS objects remotely and successfully synced locally' do create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
registry_lfs_object_remote_3 = create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
lfs_objects = subject.find_migrated_local(batch_size: 10) registries = subject.find_unsynced(batch_size: 10, except_ids: [lfs_object_3.id])
expect(lfs_objects).to match_ids(lfs_object_remote_1) expect(registries).to match_ids(registry_lfs_object_remote_3)
end
end end
end end
describe '#find_retryable_failed_registries' do describe '#find_retryable_failed_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_2.id) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_3.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_remote_1) { create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id) }
it 'returns registries for LFS objects that have failed to sync' do it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10) registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_2, registry_lfs_object_remote_1) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end registry_lfs_object_4 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
context 'with selective sync by namespace' do registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_4)
end
it 'returns registries for LFS objects that have failed to sync' do
registries = subject.find_retryable_failed_registries(batch_size: 10) registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_2) expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_4, registry_lfs_object_remote_1)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end end
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_1) it 'excludes except_ids' do
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2) registry_lfs_object_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
registry_lfs_object_remote_1 = create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
it 'returns registries for LFS objects that have failed to sync' do registries = subject.find_retryable_failed_registries(batch_size: 10, except_ids: [lfs_object_4.id])
registries = subject.find_retryable_failed_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1) expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_remote_1)
end
end end
end end
describe '#find_retryable_synced_missing_on_primary_registries' do describe '#find_retryable_synced_missing_on_primary_registries' do
let!(:registry_lfs_object_1) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_1.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_2) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true, retry_at: 1.day.from_now) }
let!(:registry_lfs_object_3) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_3.id, missing_on_primary: true, retry_at: 1.day.ago) }
let!(:registry_lfs_object_4) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_4.id, missing_on_primary: true, retry_at: nil) }
let!(:registry_lfs_object_5) { create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: false) }
it 'returns registries for LFS objects that have been synced and are missing on the primary' do it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10) create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
expect(registries).to match_ids(registry_lfs_object_1, registry_lfs_object_3, registry_lfs_object_4) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
registry_lfs_object_5 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
context 'with selective sync by namespace' do create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
before do
allow_next_instance_of(LfsObjectsProject) do |instance|
allow(instance).to receive(:update_project_statistics).and_return(nil)
end
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_3)
end
it 'returns registries for LFS objects that have been synced and are missing on the primary' do
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10) registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_1) expect(registries).to match_ids(registry_lfs_object_2, registry_lfs_object_5)
end
end end
context 'with selective sync by shard' do it 'excludes except_ids' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_1.id)
registry_lfs_object_2 = create(:geo_lfs_object_registry, lfs_object_id: lfs_object_2.id, missing_on_primary: true)
before do create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_3.id)
allow_next_instance_of(LfsObjectsProject) do |instance| create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_4.id)
allow(instance).to receive(:update_project_statistics).and_return(nil) create(:geo_lfs_object_registry, lfs_object_id: lfs_object_5.id, missing_on_primary: true, retry_at: 1.day.ago)
end create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_1.id)
create(:geo_lfs_object_registry, :failed, lfs_object_id: lfs_object_remote_2.id, missing_on_primary: true)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_1) create(:geo_lfs_object_registry, :never_synced, lfs_object_id: lfs_object_remote_3.id)
create(:lfs_objects_project, project: synced_project_in_nested_group, lfs_object: lfs_object_2)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_3)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_4)
end
it 'returns registries for LFS objects that have been synced and are missing on the primary' do registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10, except_ids: [lfs_object_5.id])
registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(registries).to match_ids(registry_lfs_object_3, registry_lfs_object_4) expect(registries).to match_ids(registry_lfs_object_2)
end
end
end end
end end
......
ee/spec/models/geo_node_status_spec.rb
View file @ 67ed817f
...@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -244,9 +244,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar) create(:geo_upload_registry, :avatar)
create(:geo_upload_registry, file_type: :attachment) create(:geo_upload_registry, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object) create(:geo_lfs_object_registry)
expect(subject.lfs_objects_synced_count).to eq(1) expect(subject.lfs_objects_synced_count).to eq(1)
end end
...@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -258,9 +258,9 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, missing_on_primary: true) create(:geo_upload_registry, :avatar, missing_on_primary: true)
create(:geo_upload_registry, file_type: :attachment, missing_on_primary: true) create(:geo_upload_registry, file_type: :attachment, missing_on_primary: true)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :with_lfs_object, missing_on_primary: true) create(:geo_lfs_object_registry, missing_on_primary: true)
expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1) expect(subject.lfs_objects_synced_missing_on_primary_count).to eq(1)
end end
...@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do ...@@ -272,40 +272,27 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
create(:geo_upload_registry, :failed) create(:geo_upload_registry, :failed)
create(:geo_upload_registry, :avatar, :failed) create(:geo_upload_registry, :avatar, :failed)
create(:geo_upload_registry, :failed, file_type: :attachment) create(:geo_upload_registry, :failed, file_type: :attachment)
create(:geo_lfs_object_registry, :with_lfs_object) create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :with_lfs_object, :failed) create(:geo_lfs_object_registry, :failed)
expect(subject.lfs_objects_failed_count).to eq(1) expect(subject.lfs_objects_failed_count).to eq(1)
end end
end end
describe '#lfs_objects_synced_in_percentage' do describe '#lfs_objects_synced_in_percentage' do
let(:lfs_object_project) { create(:lfs_objects_project, project: project_1) } it 'returns 0 when there are no registries' do
before do
allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
create(:lfs_objects_project, project: project_1)
create_list(:lfs_objects_project, 2, project: project_3)
end
it 'returns 0 when no objects are available' do
expect(subject.lfs_objects_synced_in_percentage).to eq(0) expect(subject.lfs_objects_synced_in_percentage).to eq(0)
end end
it 'returns the right percentage with no group restrictions' do it 'returns the right percentage' do
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id) create(:geo_lfs_object_registry)
create(:geo_lfs_object_registry, :failed)
create(:geo_lfs_object_registry, :never_synced)
create(:geo_lfs_object_registry, :never_synced)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25) expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(25)
end end
it 'returns the right percentage with group restrictions' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_project.lfs_object_id)
expect(subject.lfs_objects_synced_in_percentage).to be_within(0.0001).of(50)
end
end end
describe '#job_artifacts_synced_count' do describe '#job_artifacts_synced_count' do
......
ee/spec/workers/geo/migrated_local_files_clean_up_worker_spec.rb
View file @ 67ed817f
...@@ -35,62 +35,6 @@ RSpec.describe Geo::MigratedLocalFilesCleanUpWorker, :geo, :geo_fdw, :use_sql_qu ...@@ -35,62 +35,6 @@ RSpec.describe Geo::MigratedLocalFilesCleanUpWorker, :geo, :geo_fdw, :use_sql_qu
subject.perform subject.perform
end end
context 'with LFS objects' do
let(:lfs_object_local) { create(:lfs_object) }
let(:lfs_object_remote_1) { create(:lfs_object, :object_storage) }
let(:lfs_object_remote_2) { create(:lfs_object, :object_storage) }
before do
stub_lfs_object_storage
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_local.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, :local_storage_only, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_local)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_remote_2)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_remote_2.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, :local_storage_only, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_local)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_2)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_remote_2.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
end
end
context 'with attachments' do context 'with attachments' do
let(:avatar_upload) { create(:upload) } let(:avatar_upload) { create(:upload) }
let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) } let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7