Commit 314bbca3 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre Committed by Nick Thomas

Resolve "Geo: Prometheus metrics for checksummed/verification is confusing"

parent e1c8d698
......@@ -1094,6 +1094,12 @@ ActiveRecord::Schema.define(version: 20180425131009) do
t.integer "lfs_objects_synced_missing_on_primary_count"
t.integer "job_artifacts_synced_missing_on_primary_count"
t.integer "attachments_synced_missing_on_primary_count"
t.integer "repositories_checksummed_count"
t.integer "repositories_checksum_failed_count"
t.integer "repositories_checksum_mismatch_count"
t.integer "wikis_checksummed_count"
t.integer "wikis_checksum_failed_count"
t.integer "wikis_checksum_mismatch_count"
end
add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree
......
......@@ -76,6 +76,16 @@ the `monitoring.sidekiq_exporter` configuration option in `gitlab.yml`.
| geo_lfs_objects_synced_missing_on_primary | Gauge | 10.7 | Number of LFS objects marked as synced due to the file missing on the primary | url
| geo_job_artifacts_synced_missing_on_primary | Gauge | 10.7 | Number of job artifacts marked as synced due to the file missing on the primary | url
| geo_attachments_synced_missing_on_primary | Gauge | 10.7 | Number of attachments marked as synced due to the file missing on the primary | url
| geo_repositories_checksummed_count | Gauge | 10.7 | Number of repositories checksummed on primary | url
| geo_repositories_checksum_failed_count | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | url
| geo_wikis_checksummed_count | Gauge | 10.7 | Number of wikis checksummed on primary | url
| geo_wikis_checksum_failed_count | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | url
| geo_repositories_verified_count | Gauge | 10.7 | Number of repositories verified on secondary | url
| geo_repositories_verification_failed_count | Gauge | 10.7 | Number of repositories failed to verify on secondary | url
| geo_repositories_checksum_mismatch_count | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | url
| geo_wikis_verified_count | Gauge | 10.7 | Number of wikis verified on secondary | url
| geo_wikis_verification_failed_count | Gauge | 10.7 | Number of wikis failed to verify on secondary | url
| geo_wikis_checksum_mismatch_count | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | url
## Metrics shared directory
......
......@@ -175,6 +175,20 @@ Example response:
"wikis_failed_count": 0,
"wikis_synced_count": 41,
"wikis_synced_in_percentage": "100.00%",
"repositories_checksummed_count": 20,
"repositories_checksum_failed_count": 5,
"repositories_checksummed_in_percentage": "48.78%",
"wikis_checksummed_count": 10,
"wikis_checksum_failed_count": 3,
"wikis_checksummed_in_percentage": "24.39%",
"repositories_verified_count": 20,
"repositories_verification_failed_count": 5,
"repositories_verified_in_percentage": "48.78%",
"repositories_checksum_mismatch_count": 3,
"wikis_verified_count": 10,
"wikis_verification_failed_count": 3,
"wikis_verified_in_percentage": "24.39%",
"wikis_checksum_mismatch_count": 1,
"last_event_id": 23,
"last_event_timestamp": 1509681166,
"cursor_last_event_id": 23,
......
......@@ -68,6 +68,14 @@ module Geo
relation.count
end
def count_repositories_checksum_mismatch
Geo::ProjectRegistry.repository_checksum_mismatch.count
end
def count_wikis_checksum_mismatch
Geo::ProjectRegistry.wiki_checksum_mismatch.count
end
def count_verification_failed_repositories
find_verification_failed_project_registries('repository').count
end
......
......@@ -26,6 +26,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
scope :verified_wikis, -> { where.not(wiki_verification_checksum_sha: nil) }
scope :verification_failed_repos, -> { where.not(last_repository_verification_failure: nil) }
scope :verification_failed_wikis, -> { where.not(last_wiki_verification_failure: nil) }
scope :repository_checksum_mismatch, -> { where(repository_checksum_mismatch: true) }
scope :wiki_checksum_mismatch, -> { where(wiki_checksum_mismatch: true) }
def self.failed
repository_sync_failed = arel_table[:repository_retry_count].gt(0)
......
......@@ -29,10 +29,16 @@ class GeoNodeStatus < ActiveRecord::Base
wikis_count: 'Total number of wikis available on primary',
wikis_synced_count: 'Number of wikis synced on secondary',
wikis_failed_count: 'Number of wikis failed to sync on secondary',
repositories_checksummed_count: 'Number of repositories checksummed on primary',
repositories_checksum_failed_count: 'Number of repositories failed to calculate the checksum on primary',
wikis_checksummed_count: 'Number of wikis checksummed on primary',
wikis_checksum_failed_count: 'Number of wikis failed to calculate the checksum on primary',
repositories_verified_count: 'Number of repositories verified on secondary',
repositories_verification_failed_count: 'Number of repositories failed to verify on secondary',
repositories_checksum_mismatch_count: 'Number of repositories that checksum mismatch on secondary',
wikis_verified_count: 'Number of wikis verified on secondary',
wikis_verification_failed_count: 'Number of wikis failed to verify on secondary',
wikis_checksum_mismatch_count: 'Number of wikis that checksum mismatch on secondary',
lfs_objects_count: 'Total number of local LFS objects available on primary',
lfs_objects_synced_count: 'Number of local LFS objects synced on secondary',
lfs_objects_failed_count: 'Number of local LFS objects failed to sync on secondary',
......@@ -168,7 +174,6 @@ class GeoNodeStatus < ActiveRecord::Base
load_primary_data
load_secondary_data
load_verification_data
self
end
......@@ -178,6 +183,13 @@ class GeoNodeStatus < ActiveRecord::Base
self.replication_slots_count = geo_node.replication_slots_count
self.replication_slots_used_count = geo_node.replication_slots_used_count
self.replication_slots_max_retained_wal_bytes = geo_node.replication_slots_max_retained_wal_bytes
if repository_verification_enabled
self.repositories_checksummed_count = repository_verification_finder.count_verified_repositories
self.repositories_checksum_failed_count = repository_verification_finder.count_verification_failed_repositories
self.wikis_checksummed_count = repository_verification_finder.count_verified_wikis
self.wikis_checksum_failed_count = repository_verification_finder.count_verification_failed_wikis
end
end
end
......@@ -202,17 +214,20 @@ class GeoNodeStatus < ActiveRecord::Base
self.attachments_failed_count = attachments_finder.count_failed_attachments
self.attachments_registry_count = attachments_finder.count_registry_attachments
self.attachments_synced_missing_on_primary_count = attachments_finder.count_synced_missing_on_primary_attachments
load_verification_data
end
end
def load_verification_data
return unless repository_verification_enabled
finder = Gitlab::Geo.primary? ? repository_verification_finder : projects_finder
self.repositories_verified_count = finder.count_verified_repositories
self.repositories_verification_failed_count = finder.count_verification_failed_repositories
self.wikis_verified_count = finder.count_verified_wikis
self.wikis_verification_failed_count = finder.count_verification_failed_wikis
if repository_verification_enabled
self.repositories_verified_count = projects_finder.count_verified_repositories
self.repositories_verification_failed_count = projects_finder.count_verification_failed_repositories
self.repositories_checksum_mismatch_count = projects_finder.count_repositories_checksum_mismatch
self.wikis_verified_count = projects_finder.count_verified_wikis
self.wikis_verification_failed_count = projects_finder.count_verification_failed_wikis
self.wikis_checksum_mismatch_count = projects_finder.count_wikis_checksum_mismatch
end
end
alias_attribute :health, :status_message
......@@ -257,6 +272,14 @@ class GeoNodeStatus < ActiveRecord::Base
calc_percentage(wikis_count, wikis_synced_count)
end
def repositories_checksummed_in_percentage
calc_percentage(repositories_count, repositories_checksummed_count)
end
def wikis_checksummed_in_percentage
calc_percentage(wikis_count, wikis_checksummed_count)
end
def repositories_verified_in_percentage
calc_percentage(repositories_count, repositories_verified_count)
end
......
......@@ -166,6 +166,7 @@ module Geo
# Indicate that repository verification needs to be done again
attrs["#{type}_verification_checksum_sha"] = nil
attrs["#{type}_checksum_mismatch"] = false
attrs["last_#{type}_verification_failure"] = nil
end
......
......@@ -43,7 +43,7 @@ module Geo
checksum = calculate_checksum
if mismatch?(checksum)
update_registry!(failure: "#{type.to_s.capitalize} checksum mismatch: #{repository.disk_path}")
update_registry!(mismatch: true, failure: "#{type.to_s.capitalize} checksum mismatch: #{repository.disk_path}")
else
update_registry!(checksum: checksum)
end
......@@ -61,9 +61,10 @@ module Geo
primary_checksum != checksum
end
def update_registry!(checksum: nil, failure: nil, exception: nil, details: {})
def update_registry!(checksum: nil, mismatch: false, failure: nil, exception: nil, details: {})
attrs = {
"#{type}_verification_checksum_sha" => checksum,
"#{type}_checksum_mismatch" => mismatch,
"last_#{type}_verification_failure" => failure
}
......
---
title: Geo - Improve metrics for the checksum/verification feature
merge_request: 5367
author:
type: changed
class AddChecksumMismatchFieldsToProjectRegistry < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :project_registry, :repository_checksum_mismatch, :boolean, null: false, default: false
add_column :project_registry, :wiki_checksum_mismatch, :boolean, null: false, default: false
end
end
class AddIndexesToChecksumMismatchFieldsOnProjectRegistry < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
REPOSITORY_INDEX_NAME = 'idx_repository_checksum_mismatch'
WIKI_INDEX_NAME = 'idx_wiki_checksum_mismatch'
disable_ddl_transaction!
def up
add_concurrent_index(
:project_registry,
:project_id,
name: REPOSITORY_INDEX_NAME,
where: 'repository_checksum_mismatch = true'
)
add_concurrent_index(
:project_registry,
:project_id,
name: WIKI_INDEX_NAME,
where: 'wiki_checksum_mismatch = true'
)
end
def down
remove_concurrent_index_by_name(:project_registry, REPOSITORY_INDEX_NAME)
remove_concurrent_index_by_name(:project_registry, WIKI_INDEX_NAME)
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180412213305) do
ActiveRecord::Schema.define(version: 20180419192603) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -72,6 +72,8 @@ ActiveRecord::Schema.define(version: 20180412213305) do
t.string "last_wiki_verification_failure"
t.binary "repository_verification_checksum_sha"
t.binary "wiki_verification_checksum_sha"
t.boolean "repository_checksum_mismatch", default: false, null: false
t.boolean "wiki_checksum_mismatch", default: false, null: false
end
add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
......@@ -80,6 +82,8 @@ ActiveRecord::Schema.define(version: 20180412213305) do
add_index "project_registry", ["project_id"], name: "idx_project_registry_on_repository_failure_partial", where: "(last_repository_verification_failure IS NOT NULL)", using: :btree
add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_checksums_and_failure_partial", where: "((wiki_verification_checksum_sha IS NULL) AND (last_wiki_verification_failure IS NULL))", using: :btree
add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_failure_partial", where: "(last_wiki_verification_failure IS NOT NULL)", using: :btree
add_index "project_registry", ["project_id"], name: "idx_repository_checksum_mismatch", where: "(repository_checksum_mismatch = true)", using: :btree
add_index "project_registry", ["project_id"], name: "idx_wiki_checksum_mismatch", where: "(wiki_checksum_mismatch = true)", using: :btree
add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
add_index "project_registry", ["repository_verification_checksum_sha"], name: "idx_project_registry_on_repository_checksum_sha_partial", where: "(repository_verification_checksum_sha IS NULL)", using: :btree
......
class AddChecksumFieldsToGeoNodeStatuses < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :geo_node_statuses, :repositories_checksummed_count, :integer
add_column :geo_node_statuses, :repositories_checksum_failed_count, :integer
add_column :geo_node_statuses, :repositories_checksum_mismatch_count, :integer
add_column :geo_node_statuses, :wikis_checksummed_count, :integer
add_column :geo_node_statuses, :wikis_checksum_failed_count, :integer
add_column :geo_node_statuses, :wikis_checksum_mismatch_count, :integer
end
end
......@@ -299,17 +299,31 @@ module EE
expose :repository_verification_enabled
expose :repositories_checksummed_count
expose :repositories_checksum_failed_count
expose :repositories_checksummed_in_percentage do |node|
number_to_percentage(node.repositories_checksummed_in_percentage, precision: 2)
end
expose :wikis_checksummed_count
expose :wikis_checksum_failed_count
expose :wikis_checksummed_in_percentage do |node|
number_to_percentage(node.wikis_checksummed_in_percentage, precision: 2)
end
expose :repositories_verification_failed_count
expose :repositories_verified_count
expose :repositories_verified_in_percentage do |node|
number_to_percentage(node.repositories_verified_in_percentage, precision: 2)
end
expose :repositories_checksum_mismatch_count
expose :wikis_verification_failed_count
expose :wikis_verified_count
expose :wikis_verified_in_percentage do |node|
number_to_percentage(node.wikis_verified_in_percentage, precision: 2)
end
expose :wikis_checksum_mismatch_count
expose :replication_slots_count
expose :replication_slots_used_count
......
......@@ -103,9 +103,13 @@ module Gitlab
end
def handle_repository_updated_event(event, created_at)
registry = find_or_initialize_registry(event.project_id,
"resync_#{event.source}" => true, "#{event.source}_verification_checksum_sha" => nil,
"last_#{event.source}_verification_failure" => nil)
registry = find_or_initialize_registry(
event.project_id,
"resync_#{event.source}" => true,
"#{event.source}_verification_checksum_sha" => nil,
"#{event.source}_checksum_mismatch" => false,
"last_#{event.source}_verification_failure" => nil
)
registry.save!
......
......@@ -83,6 +83,11 @@ FactoryBot.define do
last_repository_verification_failure 'Repository checksum did not match'
end
trait :repository_checksum_mismatch do
last_repository_verification_failure 'Repository checksum mismatch'
repository_checksum_mismatch true
end
trait :repository_verification_outdated do
repository_verification_checksum_sha nil
last_repository_verification_failure nil
......@@ -98,6 +103,11 @@ FactoryBot.define do
last_wiki_verification_failure 'Wiki checksum did not match'
end
trait :wiki_checksum_mismatch do
last_wiki_verification_failure 'Wiki checksum mismatch'
wiki_checksum_mismatch true
end
trait :wiki_verification_outdated do
wiki_verification_checksum_sha nil
last_wiki_verification_failure nil
......
......@@ -23,10 +23,16 @@ FactoryBot.define do
wikis_count 9
wikis_synced_count 4
wikis_failed_count 1
repositories_checksummed_count 600
repositories_checksum_failed_count 120
wikis_checksummed_count 585
wikis_checksum_failed_count 55
repositories_verified_count 501
repositories_verification_failed_count 100
repositories_checksum_mismatch_count 15
wikis_verified_count 499
wikis_verification_failed_count 99
wikis_checksum_mismatch_count 10
last_event_id 2
last_event_timestamp { Time.now.to_i }
cursor_last_event_id 1
......
......@@ -26,12 +26,20 @@
"wikis_failed_count",
"wikis_synced_count",
"repository_verification_enabled",
"repositories_checksummed_count",
"repositories_checksum_failed_count",
"repositories_checksummed_in_percentage",
"wikis_checksummed_count",
"wikis_checksum_failed_count",
"wikis_checksummed_in_percentage",
"repositories_verified_count",
"repositories_verification_failed_count",
"repositories_verified_in_percentage",
"repositories_checksum_mismatch_count",
"wikis_verified_count",
"wikis_verification_failed_count",
"wikis_verified_in_percentage",
"wikis_checksum_mismatch_count",
"replication_slots_count",
"replication_slots_used_count",
"replication_slots_used_in_percentage",
......@@ -71,19 +79,27 @@
"job_artifacts_synced_in_percentage": { "type": "string" },
"repositories_count": { "type": "integer" },
"repositories_failed_count": { "type": ["integer", "null"] },
"repository_verification_enabled": { "type": "boolean" },
"repositories_synced_count": { "type": ["integer", "null"] },
"repositories_synced_in_percentage": { "type": "string" },
"wikis_count": { "type": "integer" },
"wikis_failed_count": { "type": ["integer", "null"] },
"wikis_synced_count": { "type": ["integer", "null"] },
"wikis_synced_in_percentage": { "type": "string" },
"repository_verification_enabled": { "type": "boolean" },
"repositories_checksummed_count": { "type": ["integer", "null"] },
"repositories_checksum_failed_count": { "type": ["integer", "null"] },
"repositories_checksummed_in_percentage": { "type": "string" },
"wikis_checksummed_count": { "type": ["integer", "null"] },
"wikis_checksum_failed_count": { "type": ["integer", "null"] },
"wikis_checksummed_in_percentage": { "type": "string" },
"repositories_verified_count": { "type": ["integer", "null"] },
"repositories_verification_failed_count": { "type": ["integer", "null"] },
"repositories_verified_in_percentage": { "type": "string" },
"repositories_checksum_mismatch_count": { "type": ["integer", "null"] },
"wikis_verified_count": { "type": ["integer", "null"] },
"wikis_verification_failed_count": { "type": ["integer", "null"] },
"wikis_verified_in_percentage": { "type": "string" },
"wikis_checksum_mismatch_count": { "type": ["integer", "null"] },
"replication_slots_count": { "type": ["integer", "null"] },
"replication_slots_used_count": { "type": ["integer", "null"] },
"replication_slots_used_in_percentage": { "type": "string" },
......
......@@ -113,7 +113,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
end
context 'when event source is repository' do
let!(:registry) { create(:geo_project_registry, :synced, :repository_verified, project: repository_updated_event.project) }
let!(:registry) { create(:geo_project_registry, :synced, :repository_verified, :repository_checksum_mismatch, project: repository_updated_event.project) }
before do
repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::REPOSITORY)
......@@ -125,16 +125,19 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
expect(registry.reload.resync_repository).to be true
end
it 'resets the repository verification checksum and failure' do
it 'resets the repository verification fields' do
daemon.run_once!
expect(registry.reload).to have_attributes(
resync_repository: true, repository_verification_checksum_sha: nil, last_repository_verification_failure: nil)
repository_verification_checksum_sha: nil,
repository_checksum_mismatch: false,
last_repository_verification_failure: nil
)
end
end
context 'when event source is wiki' do
let!(:registry) { create(:geo_project_registry, :synced, :wiki_verified, project: repository_updated_event.project) }
let!(:registry) { create(:geo_project_registry, :synced, :wiki_verified, :wiki_checksum_mismatch, project: repository_updated_event.project) }
before do
repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::WIKI)
......@@ -146,11 +149,14 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
expect(registry.reload.resync_wiki).to be true
end
it 'resets the wiki verification checksum and failure' do
it 'resets the wiki repository verification fields' do
daemon.run_once!
expect(registry.reload).to have_attributes(
resync_wiki: true, wiki_verification_checksum_sha: nil, last_wiki_verification_failure: nil)
wiki_verification_checksum_sha: nil,
wiki_checksum_mismatch: false,
last_wiki_verification_failure: nil
)
end
end
......
......@@ -485,117 +485,199 @@ describe GeoNodeStatus, :geo do
end
end
describe '#repositories_verified_count' do
context 'on the primary' do
describe '#repositories_checksummed_count' do
before do
stub_current_geo_node(primary)
end
it 'returns the right number of verified repositories' do
it 'returns the right number of checksummed repositories' do
stub_feature_flags(geo_repository_verification: true)
create(:repository_state, :repository_verified)
create(:repository_state, :repository_verified)
expect(subject.repositories_verified_count).to eq(2)
expect(subject.repositories_checksummed_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
it 'returns existing value when feature flag is off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: primary)
expect(subject.repositories_verified_count).to eq(501)
expect(subject.repositories_checksummed_count).to eq(600)
end
end
context 'on the secondary' do
describe '#repositories_checksum_failed_count' do
before do
stub_current_geo_node(secondary)
stub_current_geo_node(primary)
end
it 'returns the right number of verified repositories' do
it 'returns the right number of failed repositories' do
stub_feature_flags(geo_repository_verification: true)
create(:geo_project_registry, :repository_verified)
create(:geo_project_registry, :repository_verified)
create(:repository_state, :repository_failed)
create(:repository_state, :repository_failed)
expect(subject.repositories_verified_count).to eq(2)
expect(subject.repositories_checksum_failed_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: secondary)
create(:geo_node_status, :healthy, geo_node: primary)
expect(subject.repositories_verified_count).to eq(501)
expect(subject.repositories_checksum_failed_count).to eq(120)
end
end
describe '#repositories_checksummed_in_percentage' do
before do
stub_current_geo_node(primary)
end
describe '#repositories_verification_failed_count' do
context 'on the primary' do
it 'returns 0 when no projects are available' do
expect(subject.repositories_checksummed_in_percentage).to eq(0)
end
it 'returns 0 when project count is unknown' do
allow(subject).to receive(:repositories_count).and_return(nil)
expect(subject.repositories_checksummed_in_percentage).to eq(0)
end
it 'returns the right percentage' do
create(:repository_state, :repository_verified, project: project_1)
expect(subject.repositories_checksummed_in_percentage).to be_within(0.0001).of(25)
end
end
describe '#wikis_checksummed_count' do
before do
stub_current_geo_node(primary)
end
it 'returns the right number of failed repositories' do
it 'returns the right number of checksummed wikis' do
stub_feature_flags(geo_repository_verification: true)
create(:repository_state, :repository_failed)
create(:repository_state, :repository_failed)
create(:repository_state, :wiki_verified)
create(:repository_state, :wiki_verified)
expect(subject.repositories_verification_failed_count).to eq(2)
expect(subject.wikis_checksummed_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: primary)
expect(subject.repositories_verification_failed_count).to eq(100)
expect(subject.wikis_checksummed_count).to eq(585)
end
end
describe '#wikis_checksum_failed_count' do
before do
stub_current_geo_node(primary)
end
it 'returns the right number of failed wikis' do
stub_feature_flags(geo_repository_verification: true)
create(:repository_state, :wiki_failed)
create(:repository_state, :wiki_failed)
expect(subject.wikis_checksum_failed_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: primary)
expect(subject.wikis_checksum_failed_count).to eq(55)
end
end
context 'on the secondary' do
describe '#wikis_checksummed_in_percentage', :delete do
before do
stub_current_geo_node(primary)
end
it 'returns 0 when no projects are available' do
expect(subject.wikis_checksummed_in_percentage).to eq(0)
end
it 'returns 0 when project count is unknown' do
allow(subject).to receive(:wikis_count).and_return(nil)
expect(subject.wikis_checksummed_in_percentage).to eq(0)
end
it 'returns the right percentage' do
create(:repository_state, :wiki_verified, project: project_1)
expect(subject.wikis_checksummed_in_percentage).to be_within(0.0001).of(25)
end
end
describe '#repositories_verified_count' do
before do
stub_current_geo_node(secondary)
end
it 'returns the right number of failed repositories' do
it 'returns the right number of verified repositories' do
stub_feature_flags(geo_repository_verification: true)
create(:geo_project_registry, :repository_verification_failed)
create(:geo_project_registry, :repository_verification_failed)
create(:geo_project_registry, :repository_verified)
create(:geo_project_registry, :repository_verified)
expect(subject.repositories_verification_failed_count).to eq(2)
expect(subject.repositories_verified_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: secondary)
expect(subject.repositories_verification_failed_count).to eq(100)
expect(subject.repositories_verified_count).to eq(501)
end
end
describe '#repositories_checksum_mismatch_count' do
before do
stub_current_geo_node(secondary)
end
describe '#wikis_verified_count' do
context 'on the primary' do
it 'returns the right number of repositories that checksum mismatch' do
stub_feature_flags(geo_repository_verification: true)
create(:geo_project_registry, :repository_checksum_mismatch)
create(:geo_project_registry, :repository_verification_failed)
create(:geo_project_registry, :repository_verified)
expect(subject.repositories_checksum_mismatch_count).to eq(1)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: secondary)
expect(subject.repositories_checksum_mismatch_count).to eq(15)
end
end
describe '#repositories_verification_failed_count' do
before do
stub_current_geo_node(primary)
stub_current_geo_node(secondary)
end
it 'returns the right number of verified wikis' do
it 'returns the right number of failed repositories' do
stub_feature_flags(geo_repository_verification: true)
create(:repository_state, :wiki_verified)
create(:repository_state, :wiki_verified)
create(:geo_project_registry, :repository_verification_failed)
create(:geo_project_registry, :repository_verification_failed)
expect(subject.wikis_verified_count).to eq(2)
expect(subject.repositories_verification_failed_count).to eq(2)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: primary)
create(:geo_node_status, :healthy, geo_node: secondary)
expect(subject.wikis_verified_count).to eq(499)
expect(subject.repositories_verification_failed_count).to eq(100)
end
end
context 'on the secondary' do
describe '#wikis_verified_count' do
before do
stub_current_geo_node(secondary)
end
......@@ -615,31 +697,30 @@ describe GeoNodeStatus, :geo do
expect(subject.wikis_verified_count).to eq(499)
end
end
end
describe '#wikis_verification_failed_count' do
context 'on the primary' do
describe '#wikis_checksum_mismatch_count' do
before do
stub_current_geo_node(primary)
stub_current_geo_node(secondary)
end
it 'returns the right number of failed wikis' do
it 'returns the right number of wikis that checksum mismatch' do
stub_feature_flags(geo_repository_verification: true)
create(:repository_state, :wiki_failed)
create(:repository_state, :wiki_failed)
create(:geo_project_registry, :wiki_checksum_mismatch)
create(:geo_project_registry, :wiki_verification_failed)
create(:geo_project_registry, :wiki_verified)
expect(subject.wikis_verification_failed_count).to eq(2)
expect(subject.wikis_checksum_mismatch_count).to eq(1)
end
it 'returns existing value when feature flag if off' do
stub_feature_flags(geo_repository_verification: false)
create(:geo_node_status, :healthy, geo_node: primary)
create(:geo_node_status, :healthy, geo_node: secondary)
expect(subject.wikis_verification_failed_count).to eq(99)
expect(subject.wikis_checksum_mismatch_count).to eq(10)
end
end
context 'on the secondary' do
describe '#wikis_verification_failed_count' do
before do
stub_current_geo_node(secondary)
end
......@@ -659,7 +740,6 @@ describe GeoNodeStatus, :geo do
expect(subject.wikis_verification_failed_count).to eq(99)
end
end
end
describe '#last_event_id and #last_event_date' do
it 'returns nil when no events are available' do
......
......@@ -171,6 +171,12 @@ describe Geo::RepositorySyncService do
expect(registry.last_repository_verification_failure).to be_nil
end
it 'resets the repository_checksum_mismatch' do
subject.execute
expect(registry.repository_checksum_mismatch).to eq false
end
it 'logs success with timings' do
allow(Gitlab::Geo::Logger).to receive(:info).and_call_original
expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :update_delay_s, :download_time_s)).and_call_original
......
......@@ -50,8 +50,13 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
it 'sets checksum when the checksum matches' do
expect(repository).to receive(:checksum).and_return('my_checksum')
expect { service.execute }.to change(registry, "#{type}_verification_checksum_sha")
.from(nil).to('my_checksum')
service.execute
expect(registry).to have_attributes(
"#{type}_verification_checksum_sha" => 'my_checksum',
"#{type}_checksum_mismatch" => false,
"last_#{type}_verification_failure" => nil
)
end
it 'does not mark the verification as failed when there is no repo' do
......@@ -61,8 +66,9 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
service.execute
expect(registry.reload).to have_attributes(
expect(registry).to have_attributes(
"#{type}_verification_checksum_sha" => '0000000000000000000000000000000000000000',
"#{type}_checksum_mismatch" => false,
"last_#{type}_verification_failure" => nil
)
end
......@@ -70,8 +76,13 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
it 'keeps track of failure when the checksum mismatch' do
expect(repository).to receive(:checksum).and_return('other_checksum')
expect { service.execute }.to change(registry, "last_#{type}_verification_failure")
.from(nil).to(/#{Regexp.quote(type.to_s.capitalize)} checksum mismatch/)
service.execute
expect(registry).to have_attributes(
"#{type}_verification_checksum_sha" => nil,
"#{type}_checksum_mismatch" => true,
"last_#{type}_verification_failure" => /#{Regexp.quote(type.to_s.capitalize)} checksum mismatch/
)
end
def find_repository(type)
......
......@@ -150,6 +150,12 @@ RSpec.describe Geo::WikiSyncService do
expect(registry.last_wiki_verification_failure).to be_nil
end
it 'resets the wiki_checksum_mismatch' do
subject.execute
expect(registry.wiki_checksum_mismatch).to eq false
end
it 'logs success with timings' do
allow(Gitlab::Geo::Logger).to receive(:info).and_call_original
expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :update_delay_s, :download_time_s)).and_call_original
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment