Merge branch 'tc-mr-diffs-selective-sync' into 'master'

Geo: Refactor to enable not all records are replicable

See merge request gitlab-org/gitlab!42006

doc/development/geo/framework.md

@@ -207,6 +207,11 @@ For example, to add support for files referenced by a `Widget` model with a
    end
    ```
 
+   NOTE: **Note:**
+
+   If there is a common constraint for records to be available for replication,
+   make sure to also overwrite the `available_replicables` scope.
+
 1. Create `ee/app/replicators/geo/widget_replicator.rb`. Implement the
    `#carrierwave_uploader` method which should return a `CarrierWave::Uploader`.
    And implement the class method `.model` to return the `Widget` class.
@@ -560,6 +565,10 @@ Metrics are gathered by `Geo::MetricsUpdateWorker`, persisted in
    end
    ```
 
+1. Make sure the factory also allows setting a `project` attribute. If the model
+   does not have a direct relation to a project, you can use a `transient`
+   attribute. Check out `spec/factories/merge_request_diffs.rb` for an example.
+
 Widget replication and verification metrics should now be available in the API,
 the Admin Area UI, and Prometheus!
 

ee/app/models/ee/merge_request_diff.rb

@@ -24,8 +24,9 @@ module EE
       scope :has_external_diffs, -> { with_files.where(stored_externally: true) }
       scope :project_id_in, ->(ids) { where(merge_request_id: ::MergeRequest.where(target_project_id: ids)) }
 
-      scope :checksummed, -> { has_external_diffs.where(merge_request_diff_detail: ::MergeRequestDiffDetail.checksummed) }
-      scope :checksum_failed, -> { has_external_diffs.where(merge_request_diff_detail: ::MergeRequestDiffDetail.checksum_failed) }
+      scope :checksummed, -> { where(merge_request_diff_detail: ::MergeRequestDiffDetail.checksummed) }
+      scope :checksum_failed, -> { where(merge_request_diff_detail: ::MergeRequestDiffDetail.checksum_failed) }
+      scope :available_replicables, -> { has_external_diffs }
     end
 
     class_methods do
@@ -34,9 +35,9 @@ module EE
       def replicables_for_current_secondary(primary_key_in)
         node = ::Gitlab::Geo.current_node
 
-        has_external_diffs.primary_key_in(primary_key_in)
-                          .merge(selective_sync_scope(node))
-                          .merge(object_storage_scope(node))
+        available_replicables.primary_key_in(primary_key_in)
+                             .merge(selective_sync_scope(node))
+                             .merge(object_storage_scope(node))
       end
 
       private

ee/lib/gitlab/geo/replicable_model.rb

@@ -14,6 +14,7 @@ module Gitlab
 
         scope :checksummed, -> { where.not(verification_checksum: nil) }
         scope :checksum_failed, -> { where.not(verification_failure: nil) }
+        scope :available_replicables, -> { all }
 
         sha_attribute :verification_checksum
       end

ee/lib/gitlab/geo/replicator.rb

@@ -135,19 +135,19 @@ module Gitlab
       end
 
       def self.checksummed
-        model.checksummed
+        model.available_replicables.checksummed
       end
 
       def self.checksummed_count
-        model.checksummed.count
+        model.available_replicables.checksummed.count
       end
 
       def self.checksum_failed_count
-        model.checksum_failed.count
+        model.available_replicables.checksum_failed.count
       end
 
       def self.primary_total_count
-        model.count
+        model.available_replicables.count
       end
 
       def self.registry_count