Commit 75512f06 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre Committed by Nick Thomas

Add selective sync support to find registries retrying verification

These changes introduces a new finder to make it easier to
remove the legacy queries in the future.
parent 13430c60
# frozen_string_literal: true
# Finder for retrieving project registries that are retrying verification
# scoped to a type (repository or wiki) using cross-database joins
# for selective sync.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryRetryingVerificationFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistryRetryingVerificationFinder < RegistryFinder
def initialize(current_node: nil, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
if selective_sync?
registries_retrying_verification_for_selective_sync
else
registries_retrying_verification
end
end
private
attr_reader :type
def registries_retrying_verification
Geo::ProjectRegistry.retrying_verification(type)
end
# rubocop: disable CodeReuse/ActiveRecord
def registries_retrying_verification_for_selective_sync
legacy_inner_join_registry_ids(
registries_retrying_verification,
current_node.projects.pluck(:id),
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
......@@ -55,11 +55,11 @@ module Geo
end
def count_repositories_retrying_verification
Geo::ProjectRegistry.repositories_retrying_verification.count
registries_retrying_verification(:repository).count
end
def count_wikis_retrying_verification
Geo::ProjectRegistry.wikis_retrying_verification.count
registries_retrying_verification(:wiki).count
end
def find_checksum_mismatch_project_registries(type = nil)
......@@ -359,5 +359,19 @@ module Geo
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_registries_retrying_verification
if Gitlab::Geo::Fdw.enabled_for_selective_sync?
Geo::ProjectRegistryRetryingVerificationFinder
else
Geo::LegacyProjectRegistryRetryingVerificationFinder
end
end
def registries_retrying_verification(type)
finder_klass_for_registries_retrying_verification
.new(current_node: current_node, type: type)
.execute
end
end
end
# frozen_string_literal: true
# Finder for retrieving project registries that are retrying verification
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryRetryingVerificationFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistryRetryingVerificationFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_s.to_sym
end
def execute
current_node.project_registries.retrying_verification(type)
end
private
attr_reader :current_node, :type
end
end
......@@ -127,6 +127,17 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
end
end
def self.retrying_verification(type)
case type
when :repository
repositories_retrying_verification
when :wiki
wikis_retrying_verification
else
none
end
end
def self.flag_repositories_for_resync!
update_all(
resync_repository: true,
......
---
title: Geo - Add selective sync support for queries to find registries retrying verification
merge_request: 10436
author:
type: changed
......@@ -95,6 +95,11 @@ FactoryBot.define do
last_repository_verification_failure nil
end
trait :repository_retrying_verification do
repository_verification_retry_count 1
resync_repository true
end
trait :wiki_verified do
wiki_verification_checksum_sha 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
last_wiki_verification_failure nil
......@@ -114,5 +119,10 @@ FactoryBot.define do
wiki_verification_checksum_sha nil
last_wiki_verification_failure nil
end
trait :wiki_retrying_verification do
wiki_verification_retry_count 1
resync_wiki true
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryRetryingVerificationFinder, :geo do
include EE::GeoHelpers
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_1) }
let!(:repository_retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_2) }
let!(:wiki_retrying_verification) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_3) }
let!(:wiki_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_4) }
let!(:repository_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_5) }
let!(:verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
shared_examples 'finds registries retrying verification' do
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification, repository_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(repository_retrying_verification_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification, wiki_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(wiki_retrying_verification_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by namespace' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by shard' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context 'FDW', :delete do
before do
skip('FDW is not configured') unless Gitlab::Geo::Fdw.enabled?
end
include_examples 'finds registries retrying verification'
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'finds registries retrying verification'
end
end
end
......@@ -195,6 +195,56 @@ describe Geo::ProjectRegistryFinder, :geo do
expect(subject.count_verification_failed_wikis).to eq 1
end
end
describe '#count_repositories_retrying_verification' do
before do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_synced)
create(:geo_project_registry, :repository_retrying_verification, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_retrying_verification, project: project_2_in_synced_group)
end
it 'counts registries that repository retrying verification' do
expect(subject.count_repositories_retrying_verification).to eq 2
end
context 'with selective sync' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'counts registries that repository retrying verification' do
expect(subject.count_repositories_retrying_verification).to eq 1
end
end
end
describe '#count_wikis_retrying_verification' do
before do
project_1_in_synced_group = create(:project, group: synced_group)
project_2_in_synced_group = create(:project, group: synced_group)
create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_synced)
create(:geo_project_registry, :repository_retrying_verification, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_retrying_verification, project: project_2_in_synced_group)
end
it 'counts registries that wiki retrying verification' do
expect(subject.count_wikis_retrying_verification).to eq 2
end
context 'with selective sync' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'counts registries that wiki retrying verification' do
expect(subject.count_wikis_retrying_verification).to eq 1
end
end
end
end
describe '#find_checksum_mismatch_project_registries' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Geo::ProjectRegistryRetryingVerificationFinder, :geo do
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
describe '#execute', :delete do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_1) }
let!(:repository_retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_2) }
let!(:wiki_retrying_verification) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_3) }
let!(:wiki_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_4) }
let!(:repository_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_5) }
let!(:verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
before do
skip('FDW is not configured') unless Gitlab::Geo::Fdw.enabled?
end
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification, repository_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(repository_retrying_verification_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification, wiki_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(wiki_retrying_verification_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by namespace' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by shard' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment