Skip to content

G
gitlab-ce
Commit ca5c28aa authored by Michael Kozono's avatar Michael Kozono
Browse files
Options

Merge branch '223253-remove-feature-flag-to-make-registry-table-ssot-for-designs' into 'master' 

Remove feature flag to make registry table SSOT for Designs

Closes #223253

See merge request gitlab-org/gitlab!37833
parents 4d68603e af1d8f4c
Hide whitespace changes
Inline Side-by-side
Showing with 107 additions and 650 deletions
ee/app/finders/geo/design_registry_finder.rb
View file @ ca5c28aa
......@@ -3,15 +3,15 @@
module Geo
class DesignRegistryFinder < RegistryFinder
def count_syncable
current_node(fdw: false).designs.count
designs.count
end
def count_synced
registries.merge(Geo::DesignRegistry.synced).count
registries.synced.count
end
def count_failed
registries.merge(Geo::DesignRegistry.failed).count
registries.failed.count
end
def count_registry
......@@ -19,8 +19,8 @@ module Geo
end
def find_registry_differences(range)
source_ids = Gitlab::Geo.current_node.designs.id_in(range).pluck_primary_key
tracked_ids = Geo::DesignRegistry.pluck_model_ids_in_range(range)
source_ids = designs.id_in(range).pluck_primary_key
tracked_ids = registries.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids
unused_tracked_ids = tracked_ids - source_ids
......@@ -46,7 +46,7 @@ module Geo
# @param [Array<Integer>] except_ids ids that will be ignored from the query
# rubocop:disable CodeReuse/ActiveRecord
def find_never_synced_registries(batch_size:, except_ids: [])
Geo::DesignRegistry
registries
.never_synced
.model_id_not_in(except_ids)
.limit(batch_size)
......@@ -55,7 +55,7 @@ module Geo
# rubocop:disable CodeReuse/ActiveRecord
def find_retryable_dirty_registries(batch_size:, except_ids: [])
Geo::DesignRegistry
registries
.updated_recently
.model_id_not_in(except_ids)
.order(Gitlab::Database.nulls_first_order(:last_synced_at))
......@@ -65,15 +65,12 @@ module Geo
private
def designs
current_node_non_fdw.designs
end
def registries
if Geo::DesignRegistry.registry_consistency_worker_enabled?
Geo::DesignRegistry.all
else
current_node(fdw: true)
.projects
.with_designs
.inner_join_design_registry
end
Geo::DesignRegistry
end
end
end
ee/app/finders/geo/design_unsynced_finder.rb deleted 100644 → 0
View file @ 4d68603e
# frozen_string_literal: true
#
# rubocop:disable CodeReuse/ActiveRecord
module Geo
# Finder for retrieving unsynced designs that belong to a specific
# shard using FDW queries.
#
# Basic usage:
#
# Geo::DesignUnsyncedFinder
# .new(shard_name: 'default', batch_size: 1000)
# .execute.
class DesignUnsyncedFinder
def initialize(scheduled_project_ids: [], shard_name:, batch_size: nil)
@current_node = Geo::Fdw::GeoNode.find(Gitlab::Geo.current_node.id)
@scheduled_project_ids = scheduled_project_ids
@shard_name = shard_name
@batch_size = batch_size
end
def execute
return Geo::Fdw::Project.none unless valid_shard?
relation = projects
.with_designs
.missing_design_registry
.within_shards(shard_name)
.id_not_in(scheduled_project_ids)
.reorder(last_repository_updated_at: :desc)
relation = relation.limit(batch_size) unless batch_size.nil?
relation.pluck_primary_key
end
private
attr_reader :scheduled_project_ids, :current_node, :shard_name, :batch_size
def projects
return Geo::Fdw::Project.all if current_node.selective_sync_by_shards?
current_node.projects
end
def valid_shard?
return true unless current_node.selective_sync_by_shards?
current_node.selective_sync_shards.include?(shard_name)
end
end
end
ee/app/finders/geo/design_updated_recently_finder.rb deleted 100644 → 0
View file @ 4d68603e
# frozen_string_literal: true
module Geo
# Finder for retrieving designs updated recently that belong to a specific
# shard using FDW queries.
#
# Basic usage:
#
# Geo::DesignUpdatedRecentlyFinder
# .new(shard_name: 'default', batch_size: 1000)
# .execute.
class DesignUpdatedRecentlyFinder
def initialize(scheduled_project_ids: [], shard_name:, batch_size: nil)
@current_node = Geo::Fdw::GeoNode.find(Gitlab::Geo.current_node.id)
@scheduled_project_ids = scheduled_project_ids
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
return Geo::Fdw::Project.none unless valid_shard?
relation = projects
.with_designs
.recently_updated_designs
.within_shards(shard_name)
.id_not_in(scheduled_project_ids)
.order('design_registry.last_synced_at ASC NULLS FIRST')
relation = relation.limit(batch_size) unless batch_size.nil?
relation.pluck_primary_key
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :scheduled_project_ids, :current_node, :shard_name, :batch_size
def projects
return Geo::Fdw::Project.all if current_node.selective_sync_by_shards?
current_node.projects
end
def valid_shard?
return true unless current_node.selective_sync_by_shards?
current_node.selective_sync_shards.include?(shard_name)
end
end
end
ee/app/models/geo/design_registry.rb
View file @ ca5c28aa
......@@ -40,10 +40,6 @@ class Geo::DesignRegistry < Geo::BaseRegistry
end
end
def self.registry_consistency_worker_enabled?
Feature.enabled?(:geo_design_registry_ssot_sync, default_enabled: true)
end
def self.delete_for_model_ids(project_ids)
# We only need to delete the registry entries here. The design
# repository deletion should happen when a project is destroyed.
......
ee/app/models/geo/fdw/design_management_design.rb deleted 100644 → 0
View file @ 4d68603e
# frozen_string_literal: true
module Geo
module Fdw
class DesignManagementDesign < ::Geo::BaseFdw
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('design_management_designs')
self.primary_key = :id
end
end
end
ee/app/models/geo/fdw/project.rb
View file @ ca5c28aa
......@@ -12,7 +12,6 @@ module Geo
has_many :container_repositories, class_name: 'Geo::Fdw::ContainerRepository'
belongs_to :namespace, class_name: 'Geo::Fdw::Namespace'
belongs_to :design_management_designs, class_name: 'Geo::Fdw::DesignManagementDesign'
scope :outside_shards, -> (shard_names) { where.not(repository_storage: Array(shard_names)) }
......@@ -78,37 +77,6 @@ module Geo
joins(join_statement.join_sources)
end
def inner_join_design_registry
join_statement =
arel_table
.join(Geo::DesignRegistry.arel_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(Geo::DesignRegistry.arel_table[:project_id]))
joins(join_statement.join_sources)
end
def missing_design_registry
left_outer_join_design_registry
.where(Geo::DesignRegistry.arel_table[:project_id].eq(nil))
end
def recently_updated_designs
inner_join_design_registry
.merge(Geo::DesignRegistry.updated_recently)
end
def with_designs
design_table = Geo::Fdw::DesignManagementDesign.arel_table
design_subquery = design_table.project(design_table[:project_id]).distinct.as('sub_design_table')
join_statement =
arel_table
.join(design_subquery, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(design_subquery[:project_id]))
joins(join_statement.join_sources)
end
private
def left_outer_join_project_registry
......@@ -119,15 +87,6 @@ module Geo
joins(join_statement.join_sources)
end
def left_outer_join_design_registry
join_statement =
arel_table
.join(Geo::DesignRegistry.arel_table, Arel::Nodes::OuterJoin)
.on(arel_table[:id].eq(Geo::DesignRegistry.arel_table[:project_id]))
joins(join_statement.join_sources)
end
end
end
end
......
ee/app/workers/geo/design_repository_shard_sync_worker.rb
View file @ ca5c28aa
......@@ -12,45 +12,25 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_not_synced(except_ids:, batch_size:)
if Geo::DesignRegistry.registry_consistency_worker_enabled?
project_ids =
find_never_synced_project_ids(batch_size: batch_size, except_ids: except_ids)
find_project_ids_within_shard(project_ids, direction: :desc)
else
Geo::DesignUnsyncedFinder
.new(scheduled_project_ids: except_ids, shard_name: shard_name, batch_size: batch_size)
.execute
end
project_ids =
registry_finder
.find_never_synced_registries(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :desc)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(except_ids:, batch_size:)
if Geo::DesignRegistry.registry_consistency_worker_enabled?
project_ids =
find_retryable_dirty_project_ids(batch_size: batch_size, except_ids: except_ids)
find_project_ids_within_shard(project_ids, direction: :asc)
else
Geo::DesignUpdatedRecentlyFinder
.new(scheduled_project_ids: except_ids, shard_name: shard_name, batch_size: batch_size)
.execute
end
end
# rubocop: enable CodeReuse/ActiveRecord
project_ids =
registry_finder
.find_retryable_dirty_registries(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key
def find_never_synced_project_ids(batch_size:, except_ids:)
registry_finder
.find_never_synced_registries(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key
end
def find_retryable_dirty_project_ids(batch_size:, except_ids:)
registry_finder
.find_retryable_dirty_registries(batch_size: batch_size, except_ids: except_ids)
.pluck_model_foreign_key
find_project_ids_within_shard(project_ids, direction: :asc)
end
# rubocop: enable CodeReuse/ActiveRecord
def registry_finder
@registry_finder ||= Geo::DesignRegistryFinder.new
......
ee/spec/finders/geo/design_registry_finder_spec.rb
View file @ ca5c28aa
......@@ -4,158 +4,63 @@ require 'spec_helper'
RSpec.describe Geo::DesignRegistryFinder, :geo do
include ::EE::GeoHelpers
let(:secondary) { create(:geo_node) }
let(:project_1) { create(:project) }
let(:project_2) { create(:project) }
let(:project_3) { create(:project) }
let(:project_4) { create(:project) }
let(:project_5) { create(:project) }
let(:project_6) { create(:project) }
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:synced_group) { create(:group) }
let_it_be(:nested_group) { create(:group, parent: synced_group) }
let_it_be(:project_1) { create(:project, group: synced_group) }
let_it_be(:project_2) { create(:project, group: nested_group) }
let_it_be(:project_3) { create(:project) }
let_it_be(:project_4) { create(:project) }
let_it_be(:project_5) { create(:project, :broken_storage) }
let_it_be(:project_6) { create(:project, :broken_storage) }
let_it_be(:project_7) { create(:project) }
subject { described_class.new(current_node_id: secondary.id) }
context 'when geo_design_registry_ssot_sync is disabled', :geo_fdw do
let!(:failed_registry) { create(:geo_design_registry, :sync_failed) }
let!(:synced_registry) { create(:geo_design_registry, :synced) }
before do
stub_feature_flags(geo_design_registry_ssot_sync: false)
end
describe '#count_syncable' do
it 'returns number of design repositories' do
# One more design for the same project to assert absence of duplicates
create(:design, project: synced_registry.project)
expect(subject.count_syncable).to eq(2)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
expect(subject.count_synced).to eq(1)
end
end
describe '#count_failed' do
it 'returns only failed registry' do
expect(subject.count_failed).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
expect(subject.count_registry).to eq(2)
end
end
context 'selective sync' do
let(:synced_group) { create(:group) }
let(:unsynced_group) { create(:group) }
let(:synced_project) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project, :broken_storage, group: unsynced_group) }
let(:unsynced_project2) { create(:project, group: unsynced_group) }
let(:synced_project2) { create(:project, group: synced_group) }
before do
create(:geo_design_registry, :synced, project: synced_project)
create(:geo_design_registry, :sync_failed, project: synced_project2)
create(:geo_design_registry, :synced, project: unsynced_project)
create(:geo_design_registry, :sync_failed, project: unsynced_project2)
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
context 'count all the things' do
describe '#count_syncable' do
it 'returns number of design repositories' do
result = subject.count_syncable
expect(result).to eq(2)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
result = subject.count_synced
expect(result).to eq(1)
end
end
describe '#count_failed' do
it 'returns only failed registry' do
result = subject.count_failed
before do
stub_current_geo_node(secondary)
end
expect(result).to eq(1)
end
end
describe '#count_syncable' do
it 'returns number of designs' do
# Two designs for the same project to assert absence of duplicates
create_list(:design, 2, project: project_1)
create(:design, project: project_2)
describe '#count_registry' do
it 'returns number of all registries' do
result = subject.count_registry
result = subject.count_syncable
expect(result).to eq(2)
end
end
end
expect(result).to eq(2)
end
end
context 'when geo_design_registry_ssot_sync is enabled' do
let!(:registry_project_1) { create(:geo_design_registry, :synced, project_id: project_1.id) }
let!(:registry_project_2) { create(:geo_design_registry, :sync_failed, project_id: project_2.id) }
let!(:registry_project_3) { create(:geo_design_registry, project_id: project_3.id, last_synced_at: nil) }
let!(:registry_project_4) { create(:geo_design_registry, project_id: project_4.id, last_synced_at: 3.days.ago, retry_at: 2.days.ago) }
let!(:registry_project_5) { create(:geo_design_registry, project_id: project_5.id, last_synced_at: 6.days.ago) }
let!(:registry_project_6) { create(:geo_design_registry, project_id: project_6.id, last_synced_at: nil) }
describe '#count_synced' do
it 'returns number of synced registries' do
create(:geo_design_registry, :synced, project_id: project_1.id)
create(:geo_design_registry, :sync_failed, project_id: project_2.id)
before do
stub_feature_flags(geo_design_registry_ssot_sync: true)
expect(subject.count_synced).to eq(1)
end
end
describe '#count_syncable' do
it 'returns number of design repositories' do
# One more design for the same project to assert absence of duplicates
create(:design, project: project_1)
result = subject.count_syncable
describe '#count_failed' do
it 'returns number of failed registries' do
create(:geo_design_registry, :synced, project_id: project_1.id)
create(:geo_design_registry, :sync_failed, project_id: project_2.id)
expect(result).to eq(6)
end
expect(subject.count_failed).to eq(1)
end
end
describe '#count_synced' do
it 'returns only synced registry' do
expect(subject.count_synced).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
create(:geo_design_registry, :synced, project_id: project_1.id)
create(:geo_design_registry, :sync_failed, project_id: project_2.id)
describe '#count_failed' do
it 'returns only failed registry' do
expect(subject.count_failed).to eq(1)
end
end
describe '#count_registry' do
it 'returns number of all registries' do
expect(subject.count_registry).to eq(6)
end
expect(subject.count_registry).to eq(2)
end
end
describe '#find_registry_differences' do
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:synced_group) { create(:group) }
let_it_be(:nested_group) { create(:group, parent: synced_group) }
let_it_be(:project_1) { create(:project, group: synced_group) }
let_it_be(:project_2) { create(:project, group: nested_group) }
let_it_be(:project_3) { create(:project) }
let_it_be(:project_4) { create(:project) }
let_it_be(:project_5) { create(:project, :broken_storage) }
let_it_be(:project_6) { create(:project, :broken_storage) }
let_it_be(:project_7) { create(:project) }
before_all do
create(:design, project: project_1)
create(:design, project: project_2)
......@@ -165,10 +70,6 @@ RSpec.describe Geo::DesignRegistryFinder, :geo do
create(:design, project: project_6)
end
before do
stub_current_geo_node(secondary)
end
context 'untracked IDs' do
before do
create(:geo_design_registry, project_id: project_1.id)
......
ee/spec/finders/geo/design_unsynced_finder_spec.rb deleted 100644 → 0
View file @ 4d68603e
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::DesignUnsyncedFinder, :geo, :geo_fdw do
include EE::GeoHelpers
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
create(:design, project: project_1)
create(:design, project: project_2)
create(:design, project: project_3)
create(:design, project: project_4)
stub_current_geo_node(node)
end
subject { described_class.new(shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns designs without an entry on the tracking database' do
create(:geo_design_registry, :synced, project: project_2)
expect(subject.execute).to match_array([project_1.id, project_3.id])
end
end
context 'with selective sync by namespace' do
it 'returns designs that belong to the namespaces without an entry on the tracking database' do
create(:geo_design_registry, :synced, project: project_4)
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to match_array([project_1.id, project_2.id])
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return designs out of synced shards' do
subject = described_class.new(shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns designs that belong to the shards without an entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:design, project: project_5)
create(:geo_design_registry, :synced, project: project_4)
subject = described_class.new(shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_array([project_5.id])
end
end
end
end
ee/spec/finders/geo/design_updated_recently_finder_spec.rb deleted 100644 → 0
View file @ 4d68603e
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::DesignUpdatedRecentlyFinder, :geo, :geo_fdw do
include EE::GeoHelpers
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
create(:geo_design_registry, project: project_1)
create(:geo_design_registry, project: project_2)
create(:geo_design_registry, :synced, project: project_3)
create(:geo_design_registry, project: project_4)
create(:design, project: project_1)
create(:design, project: project_2)
create(:design, project: project_3)
create(:design, project: project_4)
stub_current_geo_node(node)
end
subject { described_class.new(shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns desings with a dirty entry on the tracking database' do
expect(subject.execute).to match_array([project_1.id, project_2.id])
end
end
context 'with selective sync by namespace' do
it 'returns designs that belong to the namespaces with a dirty entry on the tracking database' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([project_1.id, project_2.id])
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return designs out of selected shard' do
subject = described_class.new(shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns designs that belong to the shards with a dirty entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:design, project: project_5)
create(:geo_design_registry, :synced, project: project_5)
subject = described_class.new(shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_array([project_4.id])
end
end
end
end
ee/spec/workers/geo/design_repository_shard_sync_worker_spec.rb
View file @ ca5c28aa
......@@ -2,31 +2,26 @@
require 'spec_helper'
RSpec.describe Geo::DesignRepositoryShardSyncWorker, :geo, :geo_fdw, :clean_gitlab_redis_cache do
RSpec.describe Geo::DesignRepositoryShardSyncWorker, :geo, :clean_gitlab_redis_cache do
include ::EE::GeoHelpers
include ExclusiveLeaseHelpers
let!(:primary) { create(:geo_node, :primary) }
let!(:secondary) { create(:geo_node) }
let(:shard_name) { Gitlab.config.repositories.storages.each_key.first }
before do
stub_current_geo_node(secondary)
end
describe '#perform' do
let(:restricted_group) { create(:group) }
let(:unsynced_project_in_restricted_group) { create(:project, group: restricted_group) }
let(:unsynced_project) { create(:project) }
let_it_be(:primary) { create(:geo_node, :primary) }
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:project_1) { create(:project) }
let_it_be(:project_2) { create(:project) }
let(:shard_name) { Gitlab.config.repositories.storages.each_key.first }
before do
stub_current_geo_node(secondary)
stub_exclusive_lease(renew: true)
Gitlab::ShardHealthCache.update([shard_name])
create(:design, project: unsynced_project_in_restricted_group)
create(:design, project: unsynced_project)
create(:design, project: project_1)
create(:design, project: project_2)
end
it 'does not perform Geo::DesignRepositorySyncWorker when shard becomes unhealthy' do
......@@ -65,152 +60,61 @@ RSpec.describe Geo::DesignRepositoryShardSyncWorker, :geo, :geo_fdw, :clean_gitl
subject.perform(shard_name)
end
context 'when geo_design_registry_ssot_sync is disabled' do
before do
stub_feature_flags(geo_design_registry_ssot_sync: false)
end
it 'performs Geo::DesignRepositorySyncWorker for each project' do
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
subject.perform(shard_name)
end
it 'performs Geo::DesignRepositorySyncWorker for designs where last attempt to sync failed' do
create(:geo_design_registry, :sync_failed, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, :synced, project: unsynced_project)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).once.and_return(spy)
subject.perform(shard_name)
end
it 'performs Geo::DesignRepositorySyncWorker for designs updated recently' do
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, :synced, project: unsynced_project)
create(:geo_design_registry)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
subject.perform(shard_name)
end
it 'performs Geo::DesignRepositorySyncWorker for each registry' do
create(:geo_design_registry, project: project_1)
create(:geo_design_registry, project: project_2)
it 'does not schedule a job twice for the same project' do
scheduled_jobs = [
{ job_id: 1, project_id: unsynced_project.id },
{ job_id: 2, project_id: unsynced_project_in_restricted_group.id }
]
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
is_expected.to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(:once)
is_expected.not_to receive(:schedule_job)
Sidekiq::Testing.inline! { subject.perform(shard_name) }
end
context 'multiple shards' do
it 'uses two loops to schedule jobs', :sidekiq_might_not_need_inline do
expect(subject).to receive(:schedule_jobs).twice.and_call_original
Gitlab::ShardHealthCache.update([shard_name, 'shard2', 'shard3', 'shard4', 'shard5'])
secondary.update!(repos_max_capacity: 5)
subject.perform(shard_name)
end
end
context 'when node has namespace restrictions', :request_store do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [restricted_group])
subject.perform(shard_name)
end
allow(::Gitlab::Geo).to receive(:current_node).and_call_original
Rails.cache.write(:current_node, secondary.to_json)
allow(::GeoNode).to receive(:current_node).and_return(secondary)
end
it 'performs Geo::DesignRepositorySyncWorker for designs where last attempt to sync failed' do
create(:geo_design_registry, :sync_failed, project: project_1)
create(:geo_design_registry, :synced, project: project_2)
it 'does not perform Geo::DesignRepositorySyncWorker for projects that do not belong to selected namespaces to replicate' do
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async)
.with(unsynced_project_in_restricted_group.id)
.once
.and_return(spy)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).once.and_return(spy)
subject.perform(shard_name)
end
subject.perform(shard_name)
end
it 'does not perform Geo::DesignRepositorySyncWorker for synced projects updated recently that do not belong to selected namespaces to replicate' do
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, project: unsynced_project)
it 'performs Geo::DesignRepositorySyncWorker for designs updated recently' do
create(:geo_design_registry, project: project_1)
create(:geo_design_registry, :synced, project: project_2)
create(:geo_design_registry)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async)
.with(unsynced_project_in_restricted_group.id)
.once
.and_return(spy)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
subject.perform(shard_name)
end
end
subject.perform(shard_name)
end
context 'when geo_design_registry_ssot_sync is enabled' do
before do
stub_feature_flags(geo_design_registry_ssot_sync: true)
end
it 'does not schedule a job twice for the same project' do
create(:geo_design_registry, project: project_1)
create(:geo_design_registry, project: project_2)
it 'performs Geo::DesignRepositorySyncWorker for each registry' do
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, project: unsynced_project)
scheduled_jobs = [
{ job_id: 1, project_id: project_2.id },
{ job_id: 2, project_id: project_1.id }
]
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
is_expected.to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(:once)
is_expected.not_to receive(:schedule_job)
subject.perform(shard_name)
end
Sidekiq::Testing.inline! { subject.perform(shard_name) }
end
it 'performs Geo::DesignRepositorySyncWorker for designs where last attempt to sync failed' do
create(:geo_design_registry, :sync_failed, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, :synced, project: unsynced_project)
context 'with multiple shards' do
it 'uses two loops to schedule jobs', :sidekiq_inline do
expect(subject).to receive(:schedule_jobs).twice.and_call_original
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).once.and_return(spy)
Gitlab::ShardHealthCache.update([shard_name, 'shard2', 'shard3', 'shard4', 'shard5'])
secondary.update!(repos_max_capacity: 5)
subject.perform(shard_name)
end
it 'performs Geo::DesignRepositorySyncWorker for designs updated recently' do
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, :synced, project: unsynced_project)
create(:geo_design_registry)
expect(Geo::DesignRepositorySyncWorker).to receive(:perform_async).twice.and_return(spy)
create(:geo_design_registry, project: project_1)
create(:geo_design_registry, project: project_2)
subject.perform(shard_name)
end
it 'does not schedule a job twice for the same project' do
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, project: unsynced_project)
scheduled_jobs = [
{ job_id: 1, project_id: unsynced_project.id },
{ job_id: 2, project_id: unsynced_project_in_restricted_group.id }
]
is_expected.to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(:once)
is_expected.not_to receive(:schedule_job)
Sidekiq::Testing.inline! { subject.perform(shard_name) }
end
context 'multiple shards' do
it 'uses two loops to schedule jobs', :sidekiq_might_not_need_inline do
expect(subject).to receive(:schedule_jobs).twice.and_call_original
Gitlab::ShardHealthCache.update([shard_name, 'shard2', 'shard3', 'shard4', 'shard5'])
secondary.update!(repos_max_capacity: 5)
create(:geo_design_registry, project: unsynced_project_in_restricted_group)
create(:geo_design_registry, project: unsynced_project)
subject.perform(shard_name)
end
end
end
end
end
ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb
View file @ ca5c28aa
......@@ -108,30 +108,6 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(1)
end
context 'when geo_design_registry_ssot_sync is disabled' do
before do
stub_feature_flags(geo_design_registry_ssot_sync: false)
end
it 'returns false' do
expect(subject.perform).to be_falsey
end
it 'does not execute RegistryConsistencyService for designs' do
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::JobArtifactRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::LfsObjectRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::PackageFileRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::UploadRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::ProjectRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::VulnerabilityExportRegistry, batch_size: batch_size).and_call_original
allow(Geo::RegistryConsistencyService).to receive(:new).with(Geo::ContainerRepositoryRegistry, batch_size: batch_size).and_call_original
expect(Geo::RegistryConsistencyService).not_to receive(:new).with(Geo::DesignRegistry, batch_size: batch_size)
subject.perform
end
end
context 'when geo_container_registry_ssot_sync is disabled' do
before do
stub_feature_flags(geo_container_registry_ssot_sync: false)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7