Merge branch '4491-geo-fdw-optimise-queries' into 'master'

Improve various Geo FDW queries that need aggregate pushdown to be fast

Closes #4491

See merge request gitlab-org/gitlab-ee!4024

changelogs/unreleased-ee/4491-geo-fdw-optimise-queries.yml

+---
+title: Use the fastest available method for various Geo status counts
+merge_request: 4024
+author:
+type: fixed

ee/app/finders/geo/attachment_registry_finder.rb

@@ -13,33 +13,35 @@ module Geo
     end
 
     def count_synced_attachments
-      find_synced_attachments.count
+      if aggregate_pushdown_supported?
+        find_synced_attachments.count
+      else
+        legacy_find_synced_attachments.count
+      end
     end
 
     def count_failed_attachments
-      find_failed_attachments.count
+      if aggregate_pushdown_supported?
+        find_failed_attachments.count
+      else
+        legacy_find_failed_attachments.count
+      end
     end
 
     def find_synced_attachments
-      relation =
-        if use_legacy_queries?
-          legacy_find_synced_attachments
-        else
-          fdw_find_synced_attachments
-        end
-
-      relation
+      if use_legacy_queries?
+        legacy_find_synced_attachments
+      else
+        fdw_find_synced_attachments
+      end
     end
 
     def find_failed_attachments
-      relation =
-        if use_legacy_queries?
-          legacy_find_failed_attachments
-        else
-          fdw_find_failed_attachments
-        end
-
-      relation
+      if use_legacy_queries?
+        legacy_find_failed_attachments
+      else
+        fdw_find_failed_attachments
+      end
     end
 
     # Find limited amount of non replicated attachments.

ee/app/finders/geo/project_registry_finder.rb

@@ -102,13 +102,21 @@ module Geo
         .where(project_registry: { project_id: nil })
     end
 
-    # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
     def fdw_find_enabled_wikis
-      feature_fdw_table = Geo::Fdw::ProjectFeature.table_name
-
-      Geo::ProjectRegistry.synced_wikis
-          .joins("INNER JOIN #{feature_fdw_table} ON #{feature_fdw_table}.project_id = project_registry.project_id")
-          .where("#{feature_fdw_table}.wiki_access_level > ?", ::ProjectFeature::DISABLED)
+      project_id_matcher =
+        Geo::Fdw::ProjectFeature.arel_table[:project_id]
+          .eq(Geo::ProjectRegistry.arel_table[:project_id])
+
+      # Only read the IDs of projects with disabled wikis from the remote database
+      not_exist = Geo::Fdw::ProjectFeature
+        .where(wiki_access_level: [::ProjectFeature::DISABLED, nil])
+        .where(project_id_matcher)
+        .select('1')
+        .exists
+        .not
+
+      Geo::ProjectRegistry.synced_wikis.where(not_exist)
     end
 
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]

ee/app/finders/geo/registry_finder.rb

@@ -10,6 +10,14 @@ module Geo
 
     protected
 
+    # When this feature isn't present, FDW queries pull every row from the
+    # remote database and perform aggregates locally, leading to surprisingly
+    # slow COUNT queries on large tables. For more details, see this link:
+    # https://www.enterprisedb.com/blog/postgresql-aggregate-push-down-postgresfdw
+    def aggregate_pushdown_supported?
+      Gitlab::Geo.fdw? && Gitlab::Database.version.to_f >= 10.0
+    end
+
     def use_legacy_queries?
       # Selective project replication adds a wrinkle to FDW
       # queries, so we fallback to the legacy version for now.