Skip to content

G
gitlab-ce
Commit 90e1de0d authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Default FDW to false on Geo::AttachmentRegistryFinder 

The feature flag have that disable the FDW queries
has been enabled by default and will be removed.
parent 42603e9b
Show whitespace changes
Inline Side-by-side
Showing with 241 additions and 386 deletions
ee/app/finders/geo/attachment_registry_finder.rb
View file @ 90e1de0d
...@@ -2,10 +2,8 @@ ...@@ -2,10 +2,8 @@
module Geo module Geo
class AttachmentRegistryFinder < FileRegistryFinder class AttachmentRegistryFinder < FileRegistryFinder
# Counts all existing registries independent
# of any change on filters / selective sync
def count_registry def count_registry
Geo::UploadRegistry.count syncable.count
end end
def count_syncable def count_syncable
...@@ -13,25 +11,19 @@ module Geo ...@@ -13,25 +11,19 @@ module Geo
end end
def count_synced def count_synced
registries_for_attachments.merge(Geo::UploadRegistry.synced).count syncable.synced.count
end end
def count_failed def count_failed
registries_for_attachments.merge(Geo::UploadRegistry.failed).count syncable.failed.count
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
registries_for_attachments syncable.synced.missing_on_primary.count
.merge(Geo::UploadRegistry.synced)
.merge(Geo::UploadRegistry.missing_on_primary)
.count
end end
def syncable def syncable
return attachments if selective_sync? Geo::UploadRegistry
return Upload.with_files_stored_locally if local_storage_only?
Upload
end end
# Returns untracked uploads as well as tracked uploads that are unused. # Returns untracked uploads as well as tracked uploads that are unused.
...@@ -57,13 +49,13 @@ module Geo ...@@ -57,13 +49,13 @@ module Geo
def find_registry_differences(range) def find_registry_differences(range)
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
source = source =
attachments(fdw: false) attachments
.id_in(range) .id_in(range)
.pluck(::Upload.arel_table[:id], ::Upload.arel_table[:uploader]) .pluck(::Upload.arel_table[:id], ::Upload.arel_table[:uploader])
.map! { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] } .map! { |id, uploader| [id, uploader.sub(/Uploader\z/, '').underscore] }
tracked = tracked =
Geo::UploadRegistry syncable
.model_id_in(range) .model_id_in(range)
.pluck(:file_id, :file_type) .pluck(:file_id, :file_type)
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
...@@ -92,48 +84,21 @@ module Geo ...@@ -92,48 +84,21 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query # @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: []) def find_never_synced_registries(batch_size:, except_ids: [])
Geo::UploadRegistry syncable
.never .never
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
.limit(batch_size) .limit(batch_size)
end end
alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# Deprecated in favor of the process using
# #find_registry_differences and #find_never_synced_registries
#
# Find limited amount of non replicated attachments.
#
# You can pass a list with `except_ids:` so you can exclude items you
# already scheduled but haven't finished and aren't persisted to the database yet
#
# TODO: Alternative here is to use some sort of window function with a cursor instead
# of simply limiting the query and passing a list of items we don't want
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop: disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_ids: [])
attachments
.missing_registry
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_ids: []) def find_migrated_local(batch_size:, except_ids: [])
all_attachments Geo::UploadRegistry.none
.inner_join_registry
.with_files_stored_remotely
.id_not_in(except_ids)
.limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: []) def find_retryable_failed_registries(batch_size:, except_ids: [])
Geo::UploadRegistry syncable
.failed .failed
.retry_due .retry_due
.model_id_not_in(except_ids) .model_id_not_in(except_ids)
...@@ -143,7 +108,7 @@ module Geo ...@@ -143,7 +108,7 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_ids: [])
Geo::UploadRegistry syncable
.synced .synced
.missing_on_primary .missing_on_primary
.retry_due .retry_due
...@@ -154,16 +119,12 @@ module Geo ...@@ -154,16 +119,12 @@ module Geo
private private
def attachments(fdw: true) def attachments
local_storage_only?(fdw: fdw) ? all_attachments(fdw: fdw).with_files_stored_locally : all_attachments(fdw: fdw) local_storage_only?(fdw: false) ? all_attachments.with_files_stored_locally : all_attachments
end
def all_attachments(fdw: true)
current_node(fdw: fdw).attachments
end end
def registries_for_attachments def all_attachments
attachments.inner_join_registry current_node(fdw: false).attachments
end end
end end
end end
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @ 90e1de0d
...@@ -2,373 +2,267 @@ ...@@ -2,373 +2,267 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Geo::AttachmentRegistryFinder, :geo, :geo_fdw do RSpec.describe Geo::AttachmentRegistryFinder, :geo do
include ::EE::GeoHelpers include ::EE::GeoHelpers
# Using let() instead of set() because set() does not work properly let_it_be(:secondary) { create(:geo_node) }
# when using the :delete DatabaseCleaner strategy, which is required
# for FDW tests because a foreign table can't see changes inside a let_it_be(:synced_group) { create(:group) }
# transaction of a different connection. let_it_be(:synced_subgroup) { create(:group, parent: synced_group) }
let(:secondary) { create(:geo_node) } let_it_be(:unsynced_group) { create(:group) }
let_it_be(:synced_project) { create(:project, group: synced_group) }
let(:synced_group) { create(:group) } let_it_be(:synced_project_in_nested_group) { create(:project, group: synced_subgroup) }
let(:synced_subgroup) { create(:group, parent: synced_group) } let_it_be(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) }
let(:unsynced_group) { create(:group) }
let_it_be(:upload_1) { create(:upload, model: synced_group) }
let(:synced_project) { create(:project, group: synced_group) } let_it_be(:upload_2) { create(:upload, model: unsynced_group) }
let(:synced_project_in_nested_group) { create(:project, group: synced_subgroup) } let_it_be(:upload_3) { create(:upload, :issuable_upload, model: synced_project_in_nested_group) }
let(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) } let_it_be(:upload_4) { create(:upload, model: unsynced_project) }
let_it_be(:upload_5) { create(:upload, model: synced_project) }
subject { described_class.new(current_node_id: secondary.id) } let_it_be(:upload_6) { create(:upload, :personal_snippet_upload) }
let_it_be(:upload_7) { create(:upload, :object_storage, model: synced_project) }
let_it_be(:upload_8) { create(:upload, :object_storage, model: unsynced_project) }
let_it_be(:upload_9) { create(:upload, :object_storage, model: synced_group) }
before do before do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
end end
let!(:upload_synced_group) { create(:upload, model: synced_group) } subject { described_class.new(current_node_id: secondary.id) }
let!(:upload_unsynced_group) { create(:upload, model: unsynced_group) }
let!(:upload_issuable_synced_nested_project) { create(:upload, :issuable_upload, model: synced_project_in_nested_group) }
let!(:upload_unsynced_project) { create(:upload, model: unsynced_project) }
let!(:upload_synced_project) { create(:upload, model: synced_project) }
let!(:upload_personal_snippet) { create(:upload, :personal_snippet_upload) }
let!(:upload_remote_synced_project) { create(:upload, :object_storage, model: synced_project) }
let!(:upload_remote_unsynced_project) { create(:upload, :object_storage, model: unsynced_project) }
let!(:upload_remote_synced_group) { create(:upload, :object_storage, model: synced_group) }
context 'finds all the things' do
describe '#find_unsynced' do
before do
create(:geo_upload_registry, :avatar, file_id: upload_synced_group.id)
create(:geo_upload_registry, :avatar, file_id: upload_unsynced_group.id)
create(:geo_upload_registry, :avatar, file_id: upload_remote_synced_project.id)
end
context 'with object storage sync enabled' do
it 'returns attachments without an entry on the tracking database' do
attachments = subject.find_unsynced(batch_size: 10)
expect(attachments).to match_ids(upload_issuable_synced_nested_project, upload_unsynced_project,
upload_synced_project, upload_personal_snippet, upload_remote_unsynced_project,
upload_remote_synced_group)
end
it 'returns attachments without an entry on the tracking database, excluding from exception list' do
attachments = subject.find_unsynced(batch_size: 10, except_ids: [upload_issuable_synced_nested_project.id])
expect(attachments).to match_ids(upload_unsynced_project, upload_synced_project, upload_personal_snippet,
upload_remote_unsynced_project, upload_remote_synced_group)
end
end
context 'with object storage sync disabled' do
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'returns local attachments only' do
attachments = subject.find_unsynced(batch_size: 10, except_ids: [upload_synced_project.id])
expect(attachments).to match_ids(upload_issuable_synced_nested_project, upload_unsynced_project,
upload_personal_snippet)
end
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns attachments without an entry on the tracking database, excluding from exception list' do
attachments = subject.find_unsynced(batch_size: 10, except_ids: [upload_synced_project.id])
expect(attachments).to match_ids(upload_issuable_synced_nested_project, upload_personal_snippet,
upload_remote_synced_group)
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'returns attachments without an entry on the tracking database' do
attachments = subject.find_unsynced(batch_size: 10)
expect(attachments).to match_ids(upload_unsynced_project, upload_personal_snippet,
upload_remote_unsynced_project)
end
end
end
describe '#find_migrated_local' do
before do
create(:geo_upload_registry, :avatar, file_id: upload_remote_synced_project.id)
create(:geo_upload_registry, :avatar, file_id: upload_remote_unsynced_project.id)
end
it 'returns attachments stored remotely and successfully synced locally' do
attachments = subject.find_migrated_local(batch_size: 100, except_ids: [upload_remote_unsynced_project.id])
expect(attachments).to match_ids(upload_remote_synced_project)
end
it 'excludes attachments stored remotely, but not synced yet' do
attachments = subject.find_migrated_local(batch_size: 100)
expect(attachments).to match_ids(upload_remote_synced_project, upload_remote_unsynced_project)
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
it 'returns attachments stored remotely and successfully synced locally' do describe '#count_syncable' do
attachments = subject.find_migrated_local(batch_size: 10) it 'counts registries for uploads' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
expect(attachments).to match_ids(upload_remote_synced_project) expect(subject.count_syncable).to eq 8
end end
end end
context 'with selective sync by shard' do describe '#count_registry' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } it 'counts registries for uploads' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
it 'returns attachments stored remotely and successfully synced locally' do create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
attachments = subject.find_migrated_local(batch_size: 10) create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
expect(attachments).to match_ids(upload_remote_unsynced_project) expect(subject.count_registry).to eq 8
end
end
end end
end end
context 'counts all the things' do
describe '#count_synced' do describe '#count_synced' do
before do it 'counts registries that has been synced' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_synced_group.id) create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_group.id) create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_issuable_synced_nested_project.id) create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_project.id) create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_synced_project.id) create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, file_id: upload_personal_snippet.id) create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_remote_synced_project.id) create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_remote_unsynced_project.id) create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
end
context 'with object storage sync enabled' do expect(subject.count_synced).to eq 3
it 'counts attachments that have been synced' do
expect(subject.count_synced).to eq 7
end end
end end
context 'with object storage sync disabled' do describe '#count_failed' do
let(:secondary) { create(:geo_node, :local_storage_only) } it 'counts registries that sync has failed' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
it 'counts only local attachments that have been synced' do create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
expect(subject.count_synced).to eq 5 create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
end create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
context 'with selective sync by namespace' do create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
it 'counts attachments that has been synced' do expect(subject.count_failed).to eq 3
expect(subject.count_synced).to eq 4
end end
end end
context 'with selective sync by shard' do describe '#count_synced_missing_on_primary' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } it 'counts registries that have been synced and are missing on the primary, excluding not synced ones' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
it 'counts attachments that has been synced' do expect(subject.count_synced_missing_on_primary).to eq 3
expect(subject.count_synced).to eq 4
end
end
end end
describe '#count_failed' do
before do
create(:geo_upload_registry, :attachment, file_id: upload_synced_group.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_unsynced_group.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_issuable_synced_nested_project.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_unsynced_project.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_synced_project.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_personal_snippet.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_remote_synced_project.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_remote_unsynced_project.id)
end end
context 'with object storage sync enabled' do describe '#find_registry_differences' do
it 'counts attachments that sync has failed' do it 'returns untracked IDs as well as tracked IDs that are unused', :aggregate_failures do
expect(subject.count_failed).to eq 7 max_id = Upload.maximum(:id)
end create(:geo_upload_registry, :avatar, file_id: upload_1.id)
end create(:geo_upload_registry, :file, file_id: upload_3.id)
create(:geo_upload_registry, :avatar, file_id: upload_5.id)
create(:geo_upload_registry, :personal_file, file_id: upload_6.id)
create(:geo_upload_registry, :avatar, file_id: upload_7.id)
unused_registry_1 = create(:geo_upload_registry, :attachment, file_id: max_id + 1)
unused_registry_2 = create(:geo_upload_registry, :personal_file, file_id: max_id + 2)
range = 1..(max_id + 2)
context 'with object storage sync disabled' do untracked, unused = subject.find_registry_differences(range)
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts only local attachments that have failed' do expected_untracked = [
expect(subject.count_failed).to eq 5 [upload_2.id, 'avatar'],
end [upload_4.id, 'avatar'],
end [upload_8.id, 'avatar'],
[upload_9.id, 'avatar']
]
context 'with selective sync by namespace' do expected_unused = [
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } [unused_registry_1.file_id, 'attachment'],
[unused_registry_2.file_id, 'personal_file']
]
it 'counts attachments that sync has failed' do expect(untracked).to match_array(expected_untracked)
expect(subject.count_failed).to eq 4 expect(unused).to match_array(expected_unused)
end end
end end
context 'with selective sync by shard' do describe '#find_never_synced_registries' do
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) } it 'returns registries for uploads that have never been synced' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
registry_upload_3 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
it 'counts attachments that sync has failed' do registries = subject.find_never_synced_registries(batch_size: 10)
expect(subject.count_failed).to eq 4
end
end
end
describe '#count_synced_missing_on_primary' do expect(registries).to match_ids(registry_upload_3, registry_upload_8)
before do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_synced_group.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_group.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_issuable_synced_nested_project.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_project.id, missing_on_primary: false)
create(:geo_upload_registry, :attachment, file_id: upload_synced_project.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_personal_snippet.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_remote_synced_project.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_remote_unsynced_project.id, missing_on_primary: true)
end end
context 'with object storage sync enabled' do it 'excludes except_ids' do
it 'counts attachments that have been synced and are missing on the primary' do create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
expect(subject.count_synced_missing_on_primary).to eq 6 create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
end create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
context 'with object storage sync disabled' do registries = subject.find_never_synced_registries(batch_size: 10, except_ids: [upload_3.id])
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts only local attachments that have been synced and are missing on the primary' do expect(registries).to match_ids(registry_upload_8)
expect(subject.count_synced_missing_on_primary).to eq 4
end end
end end
context 'with selective sync by namespace' do describe '#find_unsynced' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } it 'returns registries for uploads that have never been synced' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
it 'counts attachments that have been synced and are missing on the primary' do create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 4 registry_upload_3 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
end create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
context 'with selective sync by shard' do registries = subject.find_unsynced(batch_size: 10)
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts attachments that have been synced, are missing on the primary' do expect(registries).to match_ids(registry_upload_3, registry_upload_8)
expect(subject.count_synced_missing_on_primary).to eq 3
end
end
end end
describe '#count_syncable' do it 'excludes except_ids' do
context 'with object storage sync enabled' do create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
it 'counts attachments' do create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
expect(subject.count_syncable).to eq 9 registry_upload_3 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
end create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
registry_upload_8 = create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
context 'with object storage sync disabled' do registries = subject.find_unsynced(batch_size: 10, except_ids: [registry_upload_3.file_id])
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'counts only local attachments' do expect(registries).to match_ids(registry_upload_8)
expect(subject.count_syncable).to eq 6
end end
end end
context 'with selective sync by namespace' do describe '#find_retryable_failed_registries' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } it 'returns registries for job artifacts that have failed to sync' do
registry_upload_1 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
it 'counts attachments' do create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
expect(subject.count_syncable).to eq 6 create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end registry_upload_4 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
end create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
registry_upload_6 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
context 'with selective sync by shard' do registries = subject.find_retryable_failed_registries(batch_size: 10)
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'counts attachments' do expect(registries).to match_ids(registry_upload_1, registry_upload_4, registry_upload_6)
expect(subject.count_syncable).to eq 4
end
end
end
describe '#count_registry' do
before do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_synced_group.id)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_group.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_issuable_synced_nested_project.id)
create(:geo_upload_registry, :attachment, file_id: upload_unsynced_project.id, missing_on_primary: false)
create(:geo_upload_registry, :attachment, file_id: upload_synced_project.id)
create(:geo_upload_registry, :attachment, file_id: upload_personal_snippet.id)
create(:geo_upload_registry, :attachment, file_id: upload_remote_synced_project.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_remote_unsynced_project.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, file_id: upload_remote_synced_group.id, missing_on_primary: true)
end end
context 'with object storage sync enabled' do it 'excludes except_ids' do
it 'counts file registries for attachments' do registry_upload_1 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
expect(subject.count_registry).to eq 9 create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
end create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
end create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
registry_upload_6 = create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
context 'with object storage sync disabled' do registries = subject.find_retryable_failed_registries(batch_size: 10, except_ids: [upload_4.id])
let(:secondary) { create(:geo_node, :local_storage_only) }
it 'does not apply local attachments only restriction' do expect(registries).to match_ids(registry_upload_1, registry_upload_6)
expect(subject.count_registry).to eq 9
end end
end end
context 'with selective sync by namespace' do describe '#find_retryable_synced_missing_on_primary_registries' do
let(:secondary) { create(:geo_node, selective_sync_type: 'namespaces', namespaces: [synced_group]) } it 'returns registries for job artifacts that have been synced and are missing on the primary' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
registry_upload_2 = create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
registry_upload_5 = create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
create(:geo_upload_registry, :attachment, :failed, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
it 'does not apply the selective sync restriction' do registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10)
expect(subject.count_registry).to eq 9
end
end
context 'with selective sync by shard' do expect(registries).to match_ids(registry_upload_2, registry_upload_5)
let(:secondary) { create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
it 'does not apply the selective sync restriction' do
expect(subject.count_registry).to eq 9
end
end end
end
describe '#find_registry_differences' do
it 'returns untracked IDs as well as tracked IDs that are unused', :aggregate_failures do
max_id = Upload.maximum(:id)
create(:geo_upload_registry, :avatar, file_id: upload_synced_group.id)
create(:geo_upload_registry, :file, file_id: upload_issuable_synced_nested_project.id)
create(:geo_upload_registry, :avatar, file_id: upload_synced_project.id)
create(:geo_upload_registry, :personal_file, file_id: upload_personal_snippet.id)
create(:geo_upload_registry, :avatar, file_id: upload_remote_synced_project.id)
unused_registry_1 = create(:geo_upload_registry, :attachment, file_id: max_id + 1)
unused_registry_2 = create(:geo_upload_registry, :personal_file, file_id: max_id + 2)
range = 1..(max_id + 2)
untracked, unused = subject.find_registry_differences(range) it 'excludes except_ids' do
create(:geo_upload_registry, :attachment, :failed, file_id: upload_1.id)
expected_untracked = [ registry_upload_2 = create(:geo_upload_registry, :attachment, file_id: upload_2.id, missing_on_primary: true)
[upload_unsynced_group.id, 'avatar'], create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_3.id)
[upload_unsynced_project.id, 'avatar'], create(:geo_upload_registry, :attachment, :failed, file_id: upload_4.id)
[upload_remote_unsynced_project.id, 'avatar'], create(:geo_upload_registry, :attachment, file_id: upload_5.id, missing_on_primary: true, retry_at: 1.day.ago)
[upload_remote_synced_group.id, 'avatar'] create(:geo_upload_registry, :attachment, :failed, file_id: upload_6.id)
] create(:geo_upload_registry, :attachment, :failed, file_id: upload_7.id, missing_on_primary: true)
create(:geo_upload_registry, :attachment, :never_synced, file_id: upload_8.id)
expected_unused = [ registries = subject.find_retryable_synced_missing_on_primary_registries(batch_size: 10, except_ids: [upload_5.id])
[unused_registry_1.file_id, 'attachment'],
[unused_registry_2.file_id, 'personal_file']
]
expect(untracked).to match_array(expected_untracked) expect(registries).to match_ids(registry_upload_2)
expect(unused).to match_array(expected_unused)
end
end end
end end
it_behaves_like 'a file registry finder'
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7