Merge branch...

Merge branch '229089-geo-use-registry-tale-for-project-wikis-as-ssot-while-getting-node-status' into 'master'

Fix Geo node status when treating registry SSOT for projects

Closes #229089

See merge request gitlab-org/gitlab!36916

ee/app/models/geo_node_status.rb

@@ -431,11 +431,8 @@ class GeoNodeStatus < ApplicationRecord
     self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.new.db_replication_lag_seconds
     self.cursor_last_event_id = current_cursor_last_event_id
     self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
-    self.repositories_synced_count = registries_for_synced_projects(:repository).count
-    self.repositories_failed_count = registries_for_failed_projects(:repository).count
-    self.wikis_synced_count = registries_for_synced_projects(:wiki).count
-    self.wikis_failed_count = registries_for_failed_projects(:wiki).count
 
+    load_repositories_data
     load_lfs_objects_data
     load_job_artifacts_data
     load_attachments_data
@@ -444,6 +441,20 @@ class GeoNodeStatus < ApplicationRecord
     load_package_files_data
   end
 
+  def load_repositories_data
+    if Geo::ProjectRegistry.registry_consistency_worker_enabled?
+      self.repositories_synced_count = Geo::ProjectRegistry.synced(:repository).count
+      self.repositories_failed_count = Geo::ProjectRegistry.sync_failed(:repository).count
+      self.wikis_synced_count = Geo::ProjectRegistry.synced(:wiki).count
+      self.wikis_failed_count = Geo::ProjectRegistry.sync_failed(:wiki).count
+    else
+      self.repositories_synced_count = registries_for_synced_projects(:repository).count
+      self.repositories_failed_count = registries_for_failed_projects(:repository).count
+      self.wikis_synced_count = registries_for_synced_projects(:wiki).count
+      self.wikis_failed_count = registries_for_failed_projects(:wiki).count
+    end
+  end
+
   def load_lfs_objects_data
     return unless lfs_objects_replication_enabled
 
@@ -514,13 +525,32 @@ class GeoNodeStatus < ApplicationRecord
     return unless repository_verification_enabled
 
     if Gitlab::Geo.primary?
+      load_primary_verification_data
+    elsif Gitlab::Geo.secondary?
+      load_secondary_verification_data
+    end
+  end
+
+  def load_primary_verification_data
     self.repositories_checksummed_count = repository_verification_finder.count_verified_repositories
     self.repositories_checksum_failed_count = repository_verification_finder.count_verification_failed_repositories
     self.wikis_checksummed_count = repository_verification_finder.count_verified_wikis
     self.wikis_checksum_failed_count = repository_verification_finder.count_verification_failed_wikis
     self.package_files_checksummed_count = Geo::PackageFileReplicator.checksummed_count
     self.package_files_checksum_failed_count = Geo::PackageFileReplicator.checksum_failed_count
-    elsif Gitlab::Geo.secondary?
+  end
+
+  def load_secondary_verification_data
+    if Geo::ProjectRegistry.registry_consistency_worker_enabled?
+      self.repositories_verified_count = Geo::ProjectRegistry.verified(:repository).count
+      self.repositories_verification_failed_count = Geo::ProjectRegistry.verification_failed(:repository).count
+      self.repositories_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:repository).count
+      self.wikis_verified_count = Geo::ProjectRegistry.verified(:wiki).count
+      self.wikis_verification_failed_count = Geo::ProjectRegistry.verification_failed(:wiki).count
+      self.wikis_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:wiki).count
+      self.repositories_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:repository).count
+      self.wikis_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:wiki).count
+    else
       self.repositories_verified_count = registries_for_verified_projects(:repository).count
       self.repositories_verification_failed_count = registries_for_verification_failed_projects(:repository).count
       self.repositories_checksum_mismatch_count = registries_for_mismatch_projects(:repository).count

ee/spec/models/geo_node_status_spec.rb

@@ -319,6 +319,76 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
     end
   end
 
+  describe '#repositories_synced_count' do
+    before do
+      create(:geo_project_registry, :synced, project: project_1)
+      create(:geo_project_registry, :synced, project: project_3)
+      create(:geo_project_registry, :repository_syncing, project: project_4)
+      create(:geo_project_registry, :wiki_syncing)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of synced registries' do
+        expect(subject.repositories_synced_count).to eq(3)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of synced repositories with no group restrictions' do
+        expect(subject.repositories_synced_count).to eq(3)
+      end
+
+      it 'returns the right number of synced repositories with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.repositories_synced_count).to eq(1)
+      end
+    end
+  end
+
+  describe '#wikis_synced_count' do
+    before do
+      create(:geo_project_registry, :synced, project: project_1)
+      create(:geo_project_registry, :synced, project: project_3)
+      create(:geo_project_registry, :repository_syncing, project: project_4)
+      create(:geo_project_registry, :wiki_syncing)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of synced registries' do
+        expect(subject.wikis_synced_count).to eq(3)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of synced repositories with no group restrictions' do
+        expect(subject.wikis_synced_count).to eq(3)
+      end
+
+      it 'returns the right number of synced repositories with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.wikis_synced_count).to eq(1)
+      end
+    end
+  end
+
   describe '#repositories_failed_count' do
     before do
       create(:geo_project_registry, :sync_failed, project: project_1)
@@ -327,6 +397,21 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
       create(:geo_project_registry, :wiki_syncing)
     end
 
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of failed registries' do
+        expect(subject.repositories_failed_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
       it 'returns the right number of failed repos with no group restrictions' do
         expect(subject.repositories_failed_count).to eq(2)
       end
@@ -337,6 +422,7 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
         expect(subject.repositories_failed_count).to eq(1)
       end
     end
+  end
 
   describe '#wikis_failed_count' do
     before do
@@ -346,16 +432,32 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
       create(:geo_project_registry, :wiki_syncing)
     end
 
-    it 'returns the right number of failed repos with no group restrictions' do
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of failed registries' do
         expect(subject.wikis_failed_count).to eq(2)
       end
+    end
 
-    it 'returns the right number of failed repos with group restrictions' do
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of failed repositoriess with no group restrictions' do
+        expect(subject.wikis_failed_count).to eq(2)
+      end
+
+      it 'returns the right number of failed repositoriess with group restrictions' do
         secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
 
         expect(subject.wikis_failed_count).to eq(1)
       end
     end
+  end
 
   describe '#repositories_synced_in_percentage' do
     it 'returns 0 when no projects are available' do
@@ -780,15 +882,39 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#repositories_verified_count' do
     before do
       stub_current_geo_node(secondary)
+
+      create(:geo_project_registry, :repository_verified, project: project_1)
+      create(:geo_project_registry, :repository_verified, :repository_checksum_mismatch, project: project_3)
+      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :wiki_verified, project: project_4)
     end
 
-    it 'returns the right number of verified repositories' do
-      create(:geo_project_registry, :repository_verified)
-      create(:geo_project_registry, :repository_verified)
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of verified registries' do
+        expect(subject.repositories_verified_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
 
+      it 'returns the right number of verified repositories with no group restrictions' do
         expect(subject.repositories_verified_count).to eq(2)
       end
 
+      it 'returns the right number of verified repositories with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.repositories_verified_count).to eq(1)
+      end
+    end
+
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
       create(:geo_node_status, :healthy, geo_node: secondary)
@@ -800,15 +926,38 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#repositories_checksum_mismatch_count' do
     before do
       stub_current_geo_node(secondary)
-    end
 
-    it 'returns the right number of repositories that checksum mismatch' do
-      create(:geo_project_registry, :repository_checksum_mismatch)
-      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :repository_checksum_mismatch, project: project_1)
+      create(:geo_project_registry, :repository_checksum_mismatch, project: project_3)
       create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :wiki_checksum_mismatch, project: project_4)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries that checksum mismatch' do
+        expect(subject.repositories_checksum_mismatch_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of repositories that checksum mismatch with no group restrictions' do
+        expect(subject.repositories_checksum_mismatch_count).to eq(2)
+      end
+
+      it 'returns the right number of repositories that checksum mismatch with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
 
         expect(subject.repositories_checksum_mismatch_count).to eq(1)
       end
+    end
 
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
@@ -821,15 +970,39 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#repositories_verification_failed_count' do
     before do
       stub_current_geo_node(secondary)
+
+      create(:geo_project_registry, :repository_verification_failed, project: project_1)
+      create(:geo_project_registry, :repository_verification_failed, project: project_3)
+      create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :wiki_verification_failed, project: project_4)
     end
 
-    it 'returns the right number of failed repositories' do
-      create(:geo_project_registry, :repository_verification_failed)
-      create(:geo_project_registry, :repository_verification_failed)
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries that verification failed' do
+        expect(subject.repositories_verification_failed_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
 
+      it 'returns the right number of repositories that verification failed with no group restrictions' do
         expect(subject.repositories_verification_failed_count).to eq(2)
       end
 
+      it 'returns the right number of repositories that verification failed with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.repositories_verification_failed_count).to eq(1)
+      end
+    end
+
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
       create(:geo_node_status, :healthy, geo_node: secondary)
@@ -841,15 +1014,38 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#repositories_retrying_verification_count' do
     before do
       stub_current_geo_node(secondary)
-    end
 
-    it 'returns the right number of repositories retrying verification' do
-      create(:geo_project_registry, :repository_verification_failed, repository_verification_retry_count: 1)
-      create(:geo_project_registry, :repository_verification_failed, repository_verification_retry_count: nil)
+      create(:geo_project_registry, :repository_verification_failed, repository_verification_retry_count: 1, project: project_1)
+      create(:geo_project_registry, :repository_verification_failed, repository_verification_retry_count: nil, project: project_3)
       create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :repository_verification_failed, repository_verification_retry_count: 1, project: project_4)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries retrying verification' do
+        expect(subject.repositories_retrying_verification_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of repositories retrying verification with no group restrictions' do
+        expect(subject.repositories_retrying_verification_count).to eq(2)
+      end
+
+      it 'returns the right number of repositories retrying verification with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
 
         expect(subject.repositories_retrying_verification_count).to eq(1)
       end
+    end
 
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
@@ -862,14 +1058,38 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#wikis_verified_count' do
     before do
       stub_current_geo_node(secondary)
+
+      create(:geo_project_registry, :wiki_verified, project: project_1)
+      create(:geo_project_registry, :wiki_verified, :wiki_checksum_mismatch, project: project_3)
+      create(:geo_project_registry, :wiki_verification_failed)
+      create(:geo_project_registry, :repository_verified, project: project_4)
     end
 
-    it 'returns the right number of verified wikis' do
-      create(:geo_project_registry, :wiki_verified)
-      create(:geo_project_registry, :wiki_verified)
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
 
+      it 'returns the right number of verified registries' do
         expect(subject.wikis_verified_count).to eq(2)
       end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of verified repositories with no group restrictions' do
+        expect(subject.wikis_verified_count).to eq(2)
+      end
+
+      it 'returns the right number of verified repositories with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.wikis_verified_count).to eq(1)
+      end
+    end
 
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
@@ -882,15 +1102,38 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#wikis_checksum_mismatch_count' do
     before do
       stub_current_geo_node(secondary)
-    end
 
-    it 'returns the right number of wikis that checksum mismatch' do
-      create(:geo_project_registry, :wiki_checksum_mismatch)
-      create(:geo_project_registry, :wiki_verification_failed)
+      create(:geo_project_registry, :wiki_checksum_mismatch, project: project_1)
+      create(:geo_project_registry, :wiki_checksum_mismatch, project: project_3)
       create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :repository_checksum_mismatch, project: project_4)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries that checksum mismatch' do
+        expect(subject.wikis_checksum_mismatch_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of repositories that checksum mismatch with no group restrictions' do
+        expect(subject.wikis_checksum_mismatch_count).to eq(2)
+      end
+
+      it 'returns the right number of repositories that checksum mismatch with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
 
         expect(subject.wikis_checksum_mismatch_count).to eq(1)
       end
+    end
 
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
@@ -903,15 +1146,39 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#wikis_verification_failed_count' do
     before do
       stub_current_geo_node(secondary)
+
+      create(:geo_project_registry, :wiki_verification_failed, project: project_1)
+      create(:geo_project_registry, :wiki_verification_failed, project: project_3)
+      create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :repository_verification_failed, project: project_4)
     end
 
-    it 'returns the right number of failed wikis' do
-      create(:geo_project_registry, :wiki_verification_failed)
-      create(:geo_project_registry, :wiki_verification_failed)
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries that verification failed' do
+        expect(subject.wikis_verification_failed_count).to eq(2)
+      end
+    end
 
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of repositories that verification failed with no group restrictions' do
         expect(subject.wikis_verification_failed_count).to eq(2)
       end
 
+      it 'returns the right number of repositories that verification failed with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
+
+        expect(subject.wikis_verification_failed_count).to eq(1)
+      end
+    end
+
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)
       create(:geo_node_status, :healthy, geo_node: secondary)
@@ -923,15 +1190,38 @@ RSpec.describe GeoNodeStatus, :geo, :geo_fdw do
   describe '#wikis_retrying_verification_count' do
     before do
       stub_current_geo_node(secondary)
-    end
 
-    it 'returns the right number of wikis retrying verification' do
-      create(:geo_project_registry, :wiki_verification_failed, wiki_verification_retry_count: 1)
-      create(:geo_project_registry, :wiki_verification_failed, wiki_verification_retry_count: nil)
+      create(:geo_project_registry, :wiki_verification_failed, wiki_verification_retry_count: 1, project: project_1)
+      create(:geo_project_registry, :wiki_verification_failed, wiki_verification_retry_count: nil, project: project_3)
       create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :wiki_verification_failed, wiki_verification_retry_count: 1, project: project_4)
+    end
+
+    context 'when geo_project_registry_ssot_sync is enabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: true)
+      end
+
+      it 'returns the right number of registries retrying verification' do
+        expect(subject.wikis_retrying_verification_count).to eq(2)
+      end
+    end
+
+    context 'when geo_project_registry_ssot_sync is disabled' do
+      before do
+        stub_feature_flags(geo_project_registry_ssot_sync: false)
+      end
+
+      it 'returns the right number of repositories retrying verification with no group restrictions' do
+        expect(subject.wikis_retrying_verification_count).to eq(2)
+      end
+
+      it 'returns the right number of repositories retrying verification with group restrictions' do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [group])
 
         expect(subject.wikis_retrying_verification_count).to eq(1)
       end
+    end
 
     it 'returns existing value when feature flag if off' do
       allow(Gitlab::Geo).to receive(:repository_verification_enabled?).and_return(false)