Resolve "Geo: Prometheus metrics for checksummed/verification is confusing"

db/schema.rb

@@ -1094,6 +1094,12 @@ ActiveRecord::Schema.define(version: 20180425131009) do
     t.integer "lfs_objects_synced_missing_on_primary_count"
     t.integer "job_artifacts_synced_missing_on_primary_count"
     t.integer "attachments_synced_missing_on_primary_count"
+    t.integer "repositories_checksummed_count"
+    t.integer "repositories_checksum_failed_count"
+    t.integer "repositories_checksum_mismatch_count"
+    t.integer "wikis_checksummed_count"
+    t.integer "wikis_checksum_failed_count"
+    t.integer "wikis_checksum_mismatch_count"
   end
 
   add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -76,6 +76,16 @@ the `monitoring.sidekiq_exporter` configuration option in `gitlab.yml`.
 | geo_lfs_objects_synced_missing_on_primary   | Gauge   | 10.7  | Number of LFS objects marked as synced due to the file missing on the primary | url
 | geo_job_artifacts_synced_missing_on_primary | Gauge   | 10.7  | Number of job artifacts marked as synced due to the file missing on the primary | url
 | geo_attachments_synced_missing_on_primary   | Gauge   | 10.7  | Number of attachments marked as synced due to the file missing on the primary | url
+| geo_repositories_checksummed_count          | Gauge   | 10.7  | Number of repositories checksummed on primary | url
+| geo_repositories_checksum_failed_count      | Gauge   | 10.7  | Number of repositories failed to calculate the checksum on primary | url
+| geo_wikis_checksummed_count                 | Gauge   | 10.7  | Number of wikis checksummed on primary | url
+| geo_wikis_checksum_failed_count             | Gauge   | 10.7  | Number of wikis failed to calculate the checksum on primary | url
+| geo_repositories_verified_count             | Gauge   | 10.7  | Number of repositories verified on secondary | url
+| geo_repositories_verification_failed_count  | Gauge   | 10.7  | Number of repositories failed to verify on secondary | url
+| geo_repositories_checksum_mismatch_count    | Gauge   | 10.7  | Number of repositories that checksum mismatch on secondary | url
+| geo_wikis_verified_count                    | Gauge   | 10.7  | Number of wikis verified on secondary | url
+| geo_wikis_verification_failed_count         | Gauge   | 10.7  | Number of wikis failed to verify on secondary | url
+| geo_wikis_checksum_mismatch_count           | Gauge   | 10.7  | Number of wikis that checksum mismatch on secondary | url
 
 ## Metrics shared directory
 

doc/api/geo_nodes.md

@@ -175,6 +175,20 @@ Example response:
     "wikis_failed_count": 0,
     "wikis_synced_count": 41,
     "wikis_synced_in_percentage": "100.00%",
+    "repositories_checksummed_count": 20,
+    "repositories_checksum_failed_count": 5,
+    "repositories_checksummed_in_percentage": "48.78%",
+    "wikis_checksummed_count": 10,
+    "wikis_checksum_failed_count": 3,
+    "wikis_checksummed_in_percentage": "24.39%",
+    "repositories_verified_count": 20,
+    "repositories_verification_failed_count": 5,
+    "repositories_verified_in_percentage": "48.78%",
+    "repositories_checksum_mismatch_count": 3,
+    "wikis_verified_count": 10,
+    "wikis_verification_failed_count": 3,
+    "wikis_verified_in_percentage": "24.39%",
+    "wikis_checksum_mismatch_count": 1,
     "last_event_id": 23,
     "last_event_timestamp": 1509681166,
     "cursor_last_event_id": 23,

ee/app/finders/geo/project_registry_finder.rb

@@ -68,6 +68,14 @@ module Geo
       relation.count
     end
 
+    def count_repositories_checksum_mismatch
+      Geo::ProjectRegistry.repository_checksum_mismatch.count
+    end
+
+    def count_wikis_checksum_mismatch
+      Geo::ProjectRegistry.wiki_checksum_mismatch.count
+    end
+
     def count_verification_failed_repositories
       find_verification_failed_project_registries('repository').count
     end

ee/app/models/geo/project_registry.rb

@@ -26,6 +26,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   scope :verified_wikis, -> { where.not(wiki_verification_checksum_sha: nil) }
   scope :verification_failed_repos, -> { where.not(last_repository_verification_failure: nil) }
   scope :verification_failed_wikis, -> { where.not(last_wiki_verification_failure: nil) }
+  scope :repository_checksum_mismatch, -> { where(repository_checksum_mismatch: true) }
+  scope :wiki_checksum_mismatch, -> { where(wiki_checksum_mismatch: true) }
 
   def self.failed
     repository_sync_failed = arel_table[:repository_retry_count].gt(0)

ee/app/models/geo_node_status.rb

@@ -29,10 +29,16 @@ class GeoNodeStatus < ActiveRecord::Base
     wikis_count: 'Total number of wikis available on primary',
     wikis_synced_count: 'Number of wikis synced on secondary',
     wikis_failed_count: 'Number of wikis failed to sync on secondary',
+    repositories_checksummed_count: 'Number of repositories checksummed on primary',
+    repositories_checksum_failed_count: 'Number of repositories failed to calculate the checksum on primary',
+    wikis_checksummed_count: 'Number of wikis checksummed on primary',
+    wikis_checksum_failed_count: 'Number of wikis failed to calculate the checksum on primary',
     repositories_verified_count: 'Number of repositories verified on secondary',
     repositories_verification_failed_count: 'Number of repositories failed to verify on secondary',
+    repositories_checksum_mismatch_count: 'Number of repositories that checksum mismatch on secondary',
     wikis_verified_count: 'Number of wikis verified on secondary',
     wikis_verification_failed_count: 'Number of wikis failed to verify on secondary',
+    wikis_checksum_mismatch_count: 'Number of wikis that checksum mismatch on secondary',
     lfs_objects_count: 'Total number of local LFS objects available on primary',
     lfs_objects_synced_count: 'Number of local LFS objects synced on secondary',
     lfs_objects_failed_count: 'Number of local LFS objects failed to sync on secondary',
@@ -168,7 +174,6 @@ class GeoNodeStatus < ActiveRecord::Base
 
     load_primary_data
     load_secondary_data
-    load_verification_data
 
     self
   end
@@ -178,6 +183,13 @@ class GeoNodeStatus < ActiveRecord::Base
       self.replication_slots_count = geo_node.replication_slots_count
       self.replication_slots_used_count = geo_node.replication_slots_used_count
       self.replication_slots_max_retained_wal_bytes = geo_node.replication_slots_max_retained_wal_bytes
+
+      if repository_verification_enabled
+        self.repositories_checksummed_count = repository_verification_finder.count_verified_repositories
+        self.repositories_checksum_failed_count = repository_verification_finder.count_verification_failed_repositories
+        self.wikis_checksummed_count = repository_verification_finder.count_verified_wikis
+        self.wikis_checksum_failed_count = repository_verification_finder.count_verification_failed_wikis
+      end
     end
   end
 
@@ -202,17 +214,20 @@ class GeoNodeStatus < ActiveRecord::Base
       self.attachments_failed_count = attachments_finder.count_failed_attachments
       self.attachments_registry_count = attachments_finder.count_registry_attachments
       self.attachments_synced_missing_on_primary_count = attachments_finder.count_synced_missing_on_primary_attachments
+
+      load_verification_data
     end
   end
 
   def load_verification_data
-    return unless repository_verification_enabled
-
-    finder = Gitlab::Geo.primary? ? repository_verification_finder : projects_finder
-    self.repositories_verified_count = finder.count_verified_repositories
-    self.repositories_verification_failed_count = finder.count_verification_failed_repositories
-    self.wikis_verified_count = finder.count_verified_wikis
-    self.wikis_verification_failed_count = finder.count_verification_failed_wikis
+    if repository_verification_enabled
+      self.repositories_verified_count = projects_finder.count_verified_repositories
+      self.repositories_verification_failed_count = projects_finder.count_verification_failed_repositories
+      self.repositories_checksum_mismatch_count = projects_finder.count_repositories_checksum_mismatch
+      self.wikis_verified_count = projects_finder.count_verified_wikis
+      self.wikis_verification_failed_count = projects_finder.count_verification_failed_wikis
+      self.wikis_checksum_mismatch_count = projects_finder.count_wikis_checksum_mismatch
+    end
   end
 
   alias_attribute :health, :status_message
@@ -257,6 +272,14 @@ class GeoNodeStatus < ActiveRecord::Base
     calc_percentage(wikis_count, wikis_synced_count)
   end
 
+  def repositories_checksummed_in_percentage
+    calc_percentage(repositories_count, repositories_checksummed_count)
+  end
+
+  def wikis_checksummed_in_percentage
+    calc_percentage(wikis_count, wikis_checksummed_count)
+  end
+
   def repositories_verified_in_percentage
     calc_percentage(repositories_count, repositories_verified_count)
   end

ee/app/services/geo/base_sync_service.rb

@@ -166,6 +166,7 @@ module Geo
 
         # Indicate that repository verification needs to be done again
         attrs["#{type}_verification_checksum_sha"] = nil
+        attrs["#{type}_checksum_mismatch"] = false
         attrs["last_#{type}_verification_failure"] = nil
       end
 

ee/app/services/geo/repository_verify_secondary_service.rb

@@ -43,7 +43,7 @@ module Geo
       checksum = calculate_checksum
 
       if mismatch?(checksum)
-        update_registry!(failure: "#{type.to_s.capitalize} checksum mismatch: #{repository.disk_path}")
+        update_registry!(mismatch: true, failure: "#{type.to_s.capitalize} checksum mismatch: #{repository.disk_path}")
       else
         update_registry!(checksum: checksum)
       end
@@ -61,9 +61,10 @@ module Geo
       primary_checksum != checksum
     end
 
-    def update_registry!(checksum: nil, failure: nil, exception: nil, details: {})
+    def update_registry!(checksum: nil, mismatch: false, failure: nil, exception: nil, details: {})
       attrs = {
         "#{type}_verification_checksum_sha" => checksum,
+        "#{type}_checksum_mismatch" => mismatch,
         "last_#{type}_verification_failure" => failure
       }
 

ee/changelogs/unreleased/5466-geo-prometheus-metrics-for-checksummed-verification-is-confusing.yml

+---
+title: Geo - Improve metrics for the checksum/verification feature
+merge_request: 5367
+author:
+type: changed

ee/db/geo/migrate/20180419174834_add_checksum_mismatch_fields_to_project_registry.rb

+class AddChecksumMismatchFieldsToProjectRegistry < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :project_registry, :repository_checksum_mismatch, :boolean, null: false, default: false
+    add_column :project_registry, :wiki_checksum_mismatch, :boolean, null: false, default: false
+  end
+end

ee/db/geo/migrate/20180419192603_add_indexes_to_checksum_mismatch_fields_on_project_registry.rb

+class AddIndexesToChecksumMismatchFieldsOnProjectRegistry < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  REPOSITORY_INDEX_NAME = 'idx_repository_checksum_mismatch'
+  WIKI_INDEX_NAME = 'idx_wiki_checksum_mismatch'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(
+      :project_registry,
+      :project_id,
+      name: REPOSITORY_INDEX_NAME,
+      where: 'repository_checksum_mismatch = true'
+    )
+
+    add_concurrent_index(
+      :project_registry,
+      :project_id,
+      name: WIKI_INDEX_NAME,
+      where: 'wiki_checksum_mismatch = true'
+    )
+  end
+
+  def down
+    remove_concurrent_index_by_name(:project_registry, REPOSITORY_INDEX_NAME)
+    remove_concurrent_index_by_name(:project_registry, WIKI_INDEX_NAME)
+  end
+end

ee/db/geo/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180412213305) do
+ActiveRecord::Schema.define(version: 20180419192603) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -72,6 +72,8 @@ ActiveRecord::Schema.define(version: 20180412213305) do
     t.string "last_wiki_verification_failure"
     t.binary "repository_verification_checksum_sha"
     t.binary "wiki_verification_checksum_sha"
+    t.boolean "repository_checksum_mismatch", default: false, null: false
+    t.boolean "wiki_checksum_mismatch", default: false, null: false
   end
 
   add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree
@@ -80,6 +82,8 @@ ActiveRecord::Schema.define(version: 20180412213305) do
   add_index "project_registry", ["project_id"], name: "idx_project_registry_on_repository_failure_partial", where: "(last_repository_verification_failure IS NOT NULL)", using: :btree
   add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_checksums_and_failure_partial", where: "((wiki_verification_checksum_sha IS NULL) AND (last_wiki_verification_failure IS NULL))", using: :btree
   add_index "project_registry", ["project_id"], name: "idx_project_registry_on_wiki_failure_partial", where: "(last_wiki_verification_failure IS NOT NULL)", using: :btree
+  add_index "project_registry", ["project_id"], name: "idx_repository_checksum_mismatch", where: "(repository_checksum_mismatch = true)", using: :btree
+  add_index "project_registry", ["project_id"], name: "idx_wiki_checksum_mismatch", where: "(wiki_checksum_mismatch = true)", using: :btree
   add_index "project_registry", ["project_id"], name: "index_project_registry_on_project_id", unique: true, using: :btree
   add_index "project_registry", ["repository_retry_at"], name: "index_project_registry_on_repository_retry_at", using: :btree
   add_index "project_registry", ["repository_verification_checksum_sha"], name: "idx_project_registry_on_repository_checksum_sha_partial", where: "(repository_verification_checksum_sha IS NULL)", using: :btree

ee/db/migrate/20180416205949_add_checksum_fields_to_geo_node_statuses.rb

+class AddChecksumFieldsToGeoNodeStatuses < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :geo_node_statuses, :repositories_checksummed_count, :integer
+    add_column :geo_node_statuses, :repositories_checksum_failed_count, :integer
+    add_column :geo_node_statuses, :repositories_checksum_mismatch_count, :integer
+    add_column :geo_node_statuses, :wikis_checksummed_count, :integer
+    add_column :geo_node_statuses, :wikis_checksum_failed_count, :integer
+    add_column :geo_node_statuses, :wikis_checksum_mismatch_count, :integer
+  end
+end

ee/lib/ee/api/entities.rb

@@ -299,17 +299,31 @@ module EE
 
         expose :repository_verification_enabled
 
+        expose :repositories_checksummed_count
+        expose :repositories_checksum_failed_count
+        expose :repositories_checksummed_in_percentage do |node|
+          number_to_percentage(node.repositories_checksummed_in_percentage, precision: 2)
+        end
+
+        expose :wikis_checksummed_count
+        expose :wikis_checksum_failed_count
+        expose :wikis_checksummed_in_percentage do |node|
+          number_to_percentage(node.wikis_checksummed_in_percentage, precision: 2)
+        end
+
         expose :repositories_verification_failed_count
         expose :repositories_verified_count
         expose :repositories_verified_in_percentage do |node|
           number_to_percentage(node.repositories_verified_in_percentage, precision: 2)
         end
+        expose :repositories_checksum_mismatch_count
 
         expose :wikis_verification_failed_count
         expose :wikis_verified_count
         expose :wikis_verified_in_percentage do |node|
           number_to_percentage(node.wikis_verified_in_percentage, precision: 2)
         end
+        expose :wikis_checksum_mismatch_count
 
         expose :replication_slots_count
         expose :replication_slots_used_count

ee/lib/gitlab/geo/log_cursor/daemon.rb

@@ -103,9 +103,13 @@ module Gitlab
         end
 
         def handle_repository_updated_event(event, created_at)
-          registry = find_or_initialize_registry(event.project_id,
-            "resync_#{event.source}" => true, "#{event.source}_verification_checksum_sha" => nil,
-            "last_#{event.source}_verification_failure" => nil)
+          registry = find_or_initialize_registry(
+            event.project_id,
+            "resync_#{event.source}" => true,
+            "#{event.source}_verification_checksum_sha" => nil,
+            "#{event.source}_checksum_mismatch" => false,
+            "last_#{event.source}_verification_failure" => nil
+          )
 
           registry.save!
 

ee/spec/factories/geo/project_registry.rb

@@ -83,6 +83,11 @@ FactoryBot.define do
       last_repository_verification_failure 'Repository checksum did not match'
     end
 
+    trait :repository_checksum_mismatch do
+      last_repository_verification_failure 'Repository checksum mismatch'
+      repository_checksum_mismatch true
+    end
+
     trait :repository_verification_outdated do
       repository_verification_checksum_sha nil
       last_repository_verification_failure nil
@@ -98,6 +103,11 @@ FactoryBot.define do
       last_wiki_verification_failure 'Wiki checksum did not match'
     end
 
+    trait :wiki_checksum_mismatch do
+      last_wiki_verification_failure 'Wiki checksum mismatch'
+      wiki_checksum_mismatch true
+    end
+
     trait :wiki_verification_outdated do
       wiki_verification_checksum_sha nil
       last_wiki_verification_failure nil

ee/spec/factories/geo_node_statuses.rb

@@ -23,10 +23,16 @@ FactoryBot.define do
       wikis_count 9
       wikis_synced_count 4
       wikis_failed_count 1
+      repositories_checksummed_count 600
+      repositories_checksum_failed_count 120
+      wikis_checksummed_count 585
+      wikis_checksum_failed_count 55
       repositories_verified_count 501
       repositories_verification_failed_count 100
+      repositories_checksum_mismatch_count 15
       wikis_verified_count 499
       wikis_verification_failed_count 99
+      wikis_checksum_mismatch_count 10
       last_event_id 2
       last_event_timestamp { Time.now.to_i }
       cursor_last_event_id 1

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -26,12 +26,20 @@
     "wikis_failed_count",
     "wikis_synced_count",
     "repository_verification_enabled",
+    "repositories_checksummed_count",
+    "repositories_checksum_failed_count",
+    "repositories_checksummed_in_percentage",
+    "wikis_checksummed_count",
+    "wikis_checksum_failed_count",
+    "wikis_checksummed_in_percentage",
     "repositories_verified_count",
     "repositories_verification_failed_count",
     "repositories_verified_in_percentage",
+    "repositories_checksum_mismatch_count",
     "wikis_verified_count",
     "wikis_verification_failed_count",
     "wikis_verified_in_percentage",
+    "wikis_checksum_mismatch_count",
     "replication_slots_count",
     "replication_slots_used_count",
     "replication_slots_used_in_percentage",
@@ -71,19 +79,27 @@
     "job_artifacts_synced_in_percentage": { "type": "string" },
     "repositories_count": { "type": "integer" },
     "repositories_failed_count": { "type": ["integer", "null"] },
+    "repository_verification_enabled": { "type": "boolean" },
     "repositories_synced_count": { "type": ["integer", "null"] },
     "repositories_synced_in_percentage": { "type": "string" },
     "wikis_count": { "type": "integer" },
     "wikis_failed_count": { "type": ["integer", "null"] },
     "wikis_synced_count": { "type": ["integer", "null"] },
     "wikis_synced_in_percentage": { "type": "string" },
-    "repository_verification_enabled": { "type": "boolean" },
+    "repositories_checksummed_count": { "type": ["integer", "null"] },
+    "repositories_checksum_failed_count": { "type": ["integer", "null"] },
+    "repositories_checksummed_in_percentage": { "type": "string" },
+    "wikis_checksummed_count": { "type": ["integer", "null"] },
+    "wikis_checksum_failed_count": { "type": ["integer", "null"] },
+    "wikis_checksummed_in_percentage": { "type": "string" },
     "repositories_verified_count": { "type": ["integer", "null"] },
     "repositories_verification_failed_count": { "type": ["integer", "null"] },
     "repositories_verified_in_percentage": { "type": "string" },
+    "repositories_checksum_mismatch_count": { "type": ["integer", "null"] },
     "wikis_verified_count": { "type": ["integer", "null"] },
     "wikis_verification_failed_count": { "type": ["integer", "null"] },
     "wikis_verified_in_percentage": { "type": "string" },
+    "wikis_checksum_mismatch_count": { "type": ["integer", "null"] },
     "replication_slots_count": { "type": ["integer", "null"] },
     "replication_slots_used_count": { "type": ["integer", "null"] },
     "replication_slots_used_in_percentage": { "type": "string" },

ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb

@@ -113,7 +113,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
       end
 
       context 'when event source is repository' do
-        let!(:registry) { create(:geo_project_registry, :synced, :repository_verified, project: repository_updated_event.project) }
+        let!(:registry) { create(:geo_project_registry, :synced, :repository_verified, :repository_checksum_mismatch, project: repository_updated_event.project) }
 
         before do
           repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::REPOSITORY)
@@ -125,16 +125,19 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
           expect(registry.reload.resync_repository).to be true
         end
 
-        it 'resets the repository verification checksum and failure' do
+        it 'resets the repository verification fields' do
           daemon.run_once!
 
           expect(registry.reload).to have_attributes(
-            resync_repository: true, repository_verification_checksum_sha: nil, last_repository_verification_failure: nil)
+            repository_verification_checksum_sha: nil,
+            repository_checksum_mismatch: false,
+            last_repository_verification_failure: nil
+          )
         end
       end
 
       context 'when event source is wiki' do
-        let!(:registry) { create(:geo_project_registry, :synced, :wiki_verified, project: repository_updated_event.project) }
+        let!(:registry) { create(:geo_project_registry, :synced, :wiki_verified, :wiki_checksum_mismatch, project: repository_updated_event.project) }
 
         before do
           repository_updated_event.update!(source: Geo::RepositoryUpdatedEvent::WIKI)
@@ -146,11 +149,14 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
           expect(registry.reload.resync_wiki).to be true
         end
 
-        it 'resets the wiki verification checksum and failure' do
+        it 'resets the wiki repository verification fields' do
           daemon.run_once!
 
           expect(registry.reload).to have_attributes(
-            resync_wiki: true, wiki_verification_checksum_sha: nil, last_wiki_verification_failure: nil)
+            wiki_verification_checksum_sha: nil,
+            wiki_checksum_mismatch: false,
+            last_wiki_verification_failure: nil
+          )
         end
       end
 

ee/spec/models/geo_node_status_spec.rb

@@ -485,179 +485,259 @@ describe GeoNodeStatus, :geo do
     end
   end
 
-  describe '#repositories_verified_count' do
-    context 'on the primary' do
-      before do
-        stub_current_geo_node(primary)
-      end
+  describe '#repositories_checksummed_count' do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of verified repositories' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:repository_state, :repository_verified)
-        create(:repository_state, :repository_verified)
+    it 'returns the right number of checksummed repositories' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:repository_state, :repository_verified)
+      create(:repository_state, :repository_verified)
 
-        expect(subject.repositories_verified_count).to eq(2)
-      end
+      expect(subject.repositories_checksummed_count).to eq(2)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: primary)
+    it 'returns existing value when feature flag is off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: primary)
 
-        expect(subject.repositories_verified_count).to eq(501)
-      end
+      expect(subject.repositories_checksummed_count).to eq(600)
     end
+  end
 
-    context 'on the secondary' do
-      before do
-        stub_current_geo_node(secondary)
-      end
+  describe '#repositories_checksum_failed_count' do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of verified repositories' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:geo_project_registry, :repository_verified)
-        create(:geo_project_registry, :repository_verified)
+    it 'returns the right number of failed repositories' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:repository_state, :repository_failed)
+      create(:repository_state, :repository_failed)
 
-        expect(subject.repositories_verified_count).to eq(2)
-      end
+      expect(subject.repositories_checksum_failed_count).to eq(2)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: secondary)
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: primary)
 
-        expect(subject.repositories_verified_count).to eq(501)
-      end
+      expect(subject.repositories_checksum_failed_count).to eq(120)
     end
   end
 
-  describe '#repositories_verification_failed_count' do
-    context 'on the primary' do
-      before do
-        stub_current_geo_node(primary)
-      end
+  describe '#repositories_checksummed_in_percentage' do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of failed repositories' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:repository_state, :repository_failed)
-        create(:repository_state, :repository_failed)
+    it 'returns 0 when no projects are available' do
+      expect(subject.repositories_checksummed_in_percentage).to eq(0)
+    end
 
-        expect(subject.repositories_verification_failed_count).to eq(2)
-      end
+    it 'returns 0 when project count is unknown' do
+      allow(subject).to receive(:repositories_count).and_return(nil)
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: primary)
+      expect(subject.repositories_checksummed_in_percentage).to eq(0)
+    end
 
-        expect(subject.repositories_verification_failed_count).to eq(100)
-      end
+    it 'returns the right percentage' do
+      create(:repository_state, :repository_verified, project: project_1)
+
+      expect(subject.repositories_checksummed_in_percentage).to be_within(0.0001).of(25)
     end
+  end
 
-    context 'on the secondary' do
-      before do
-        stub_current_geo_node(secondary)
-      end
+  describe '#wikis_checksummed_count' do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of failed repositories' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:geo_project_registry, :repository_verification_failed)
-        create(:geo_project_registry, :repository_verification_failed)
+    it 'returns the right number of checksummed wikis' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:repository_state, :wiki_verified)
+      create(:repository_state, :wiki_verified)
 
-        expect(subject.repositories_verification_failed_count).to eq(2)
-      end
+      expect(subject.wikis_checksummed_count).to eq(2)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: secondary)
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: primary)
 
-        expect(subject.repositories_verification_failed_count).to eq(100)
-      end
+      expect(subject.wikis_checksummed_count).to eq(585)
     end
   end
 
-  describe '#wikis_verified_count' do
-    context 'on the primary' do
-      before do
-        stub_current_geo_node(primary)
-      end
+  describe '#wikis_checksum_failed_count' do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of verified wikis' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:repository_state, :wiki_verified)
-        create(:repository_state, :wiki_verified)
+    it 'returns the right number of failed wikis' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:repository_state, :wiki_failed)
+      create(:repository_state, :wiki_failed)
 
-        expect(subject.wikis_verified_count).to eq(2)
-      end
+      expect(subject.wikis_checksum_failed_count).to eq(2)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: primary)
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: primary)
 
-        expect(subject.wikis_verified_count).to eq(499)
-      end
+      expect(subject.wikis_checksum_failed_count).to eq(55)
     end
+  end
 
-    context 'on the secondary' do
-      before do
-        stub_current_geo_node(secondary)
-      end
+  describe '#wikis_checksummed_in_percentage', :delete do
+    before do
+      stub_current_geo_node(primary)
+    end
 
-      it 'returns the right number of verified wikis' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:geo_project_registry, :wiki_verified)
-        create(:geo_project_registry, :wiki_verified)
+    it 'returns 0 when no projects are available' do
+      expect(subject.wikis_checksummed_in_percentage).to eq(0)
+    end
 
-        expect(subject.wikis_verified_count).to eq(2)
-      end
+    it 'returns 0 when project count is unknown' do
+      allow(subject).to receive(:wikis_count).and_return(nil)
+
+      expect(subject.wikis_checksummed_in_percentage).to eq(0)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: secondary)
+    it 'returns the right percentage' do
+      create(:repository_state, :wiki_verified, project: project_1)
 
-        expect(subject.wikis_verified_count).to eq(499)
-      end
+      expect(subject.wikis_checksummed_in_percentage).to be_within(0.0001).of(25)
     end
   end
 
-  describe '#wikis_verification_failed_count' do
-    context 'on the primary' do
-      before do
-        stub_current_geo_node(primary)
-      end
+  describe '#repositories_verified_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
 
-      it 'returns the right number of failed wikis' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:repository_state, :wiki_failed)
-        create(:repository_state, :wiki_failed)
+    it 'returns the right number of verified repositories' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :repository_verified)
 
-        expect(subject.wikis_verification_failed_count).to eq(2)
-      end
+      expect(subject.repositories_verified_count).to eq(2)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: primary)
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
 
-        expect(subject.wikis_verification_failed_count).to eq(99)
-      end
+      expect(subject.repositories_verified_count).to eq(501)
     end
+  end
 
-    context 'on the secondary' do
-      before do
-        stub_current_geo_node(secondary)
-      end
+  describe '#repositories_checksum_mismatch_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
 
-      it 'returns the right number of failed wikis' do
-        stub_feature_flags(geo_repository_verification: true)
-        create(:geo_project_registry, :wiki_verification_failed)
-        create(:geo_project_registry, :wiki_verification_failed)
+    it 'returns the right number of repositories that checksum mismatch' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :repository_checksum_mismatch)
+      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :repository_verified)
 
-        expect(subject.wikis_verification_failed_count).to eq(2)
-      end
+      expect(subject.repositories_checksum_mismatch_count).to eq(1)
+    end
 
-      it 'returns existing value when feature flag if off' do
-        stub_feature_flags(geo_repository_verification: false)
-        create(:geo_node_status, :healthy, geo_node: secondary)
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
 
-        expect(subject.wikis_verification_failed_count).to eq(99)
-      end
+      expect(subject.repositories_checksum_mismatch_count).to eq(15)
+    end
+  end
+
+  describe '#repositories_verification_failed_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
+
+    it 'returns the right number of failed repositories' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :repository_verification_failed)
+
+      expect(subject.repositories_verification_failed_count).to eq(2)
+    end
+
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
+
+      expect(subject.repositories_verification_failed_count).to eq(100)
+    end
+  end
+
+  describe '#wikis_verified_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
+
+    it 'returns the right number of verified wikis' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :wiki_verified)
+
+      expect(subject.wikis_verified_count).to eq(2)
+    end
+
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
+
+      expect(subject.wikis_verified_count).to eq(499)
+    end
+  end
+
+  describe '#wikis_checksum_mismatch_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
+
+    it 'returns the right number of wikis that checksum mismatch' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :wiki_checksum_mismatch)
+      create(:geo_project_registry, :wiki_verification_failed)
+      create(:geo_project_registry, :wiki_verified)
+
+      expect(subject.wikis_checksum_mismatch_count).to eq(1)
+    end
+
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
+
+      expect(subject.wikis_checksum_mismatch_count).to eq(10)
+    end
+  end
+
+  describe '#wikis_verification_failed_count' do
+    before do
+      stub_current_geo_node(secondary)
+    end
+
+    it 'returns the right number of failed wikis' do
+      stub_feature_flags(geo_repository_verification: true)
+      create(:geo_project_registry, :wiki_verification_failed)
+      create(:geo_project_registry, :wiki_verification_failed)
+
+      expect(subject.wikis_verification_failed_count).to eq(2)
+    end
+
+    it 'returns existing value when feature flag if off' do
+      stub_feature_flags(geo_repository_verification: false)
+      create(:geo_node_status, :healthy, geo_node: secondary)
+
+      expect(subject.wikis_verification_failed_count).to eq(99)
     end
   end
 

ee/spec/services/geo/repository_sync_service_spec.rb

@@ -171,6 +171,12 @@ describe Geo::RepositorySyncService do
           expect(registry.last_repository_verification_failure).to be_nil
         end
 
+        it 'resets the repository_checksum_mismatch' do
+          subject.execute
+
+          expect(registry.repository_checksum_mismatch).to eq false
+        end
+
         it 'logs success with timings' do
           allow(Gitlab::Geo::Logger).to receive(:info).and_call_original
           expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :update_delay_s, :download_time_s)).and_call_original

ee/spec/services/geo/repository_verify_secondary_service_spec.rb

@@ -50,8 +50,13 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
     it 'sets checksum when the checksum matches' do
       expect(repository).to receive(:checksum).and_return('my_checksum')
 
-      expect { service.execute }.to change(registry, "#{type}_verification_checksum_sha")
-        .from(nil).to('my_checksum')
+      service.execute
+
+      expect(registry).to have_attributes(
+        "#{type}_verification_checksum_sha" => 'my_checksum',
+        "#{type}_checksum_mismatch" => false,
+        "last_#{type}_verification_failure" => nil
+      )
     end
 
     it 'does not mark the verification as failed when there is no repo' do
@@ -61,8 +66,9 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
 
       service.execute
 
-      expect(registry.reload).to have_attributes(
+      expect(registry).to have_attributes(
         "#{type}_verification_checksum_sha" => '0000000000000000000000000000000000000000',
+        "#{type}_checksum_mismatch" => false,
         "last_#{type}_verification_failure" => nil
       )
     end
@@ -70,8 +76,13 @@ describe Geo::RepositoryVerifySecondaryService, :geo do
     it 'keeps track of failure when the checksum mismatch' do
       expect(repository).to receive(:checksum).and_return('other_checksum')
 
-      expect { service.execute }.to change(registry, "last_#{type}_verification_failure")
-        .from(nil).to(/#{Regexp.quote(type.to_s.capitalize)} checksum mismatch/)
+      service.execute
+
+      expect(registry).to have_attributes(
+        "#{type}_verification_checksum_sha" => nil,
+        "#{type}_checksum_mismatch" => true,
+        "last_#{type}_verification_failure" => /#{Regexp.quote(type.to_s.capitalize)} checksum mismatch/
+      )
     end
 
     def find_repository(type)

ee/spec/services/geo/wiki_sync_service_spec.rb

@@ -150,6 +150,12 @@ RSpec.describe Geo::WikiSyncService do
           expect(registry.last_wiki_verification_failure).to be_nil
         end
 
+        it 'resets the wiki_checksum_mismatch' do
+          subject.execute
+
+          expect(registry.wiki_checksum_mismatch).to eq false
+        end
+
         it 'logs success with timings' do
           allow(Gitlab::Geo::Logger).to receive(:info).and_call_original
           expect(Gitlab::Geo::Logger).to receive(:info).with(hash_including(:message, :update_delay_s, :download_time_s)).and_call_original