Merge branch 'sh-use-cte-for-projects-finder' into 'master'

Use CTE optimization fence for loading projects in dashboard

Closes #198440

See merge request gitlab-org/gitlab!23754

app/controllers/dashboard/projects_controller.rb

@@ -66,6 +66,8 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController
     @total_user_projects_count = ProjectsFinder.new(params: { non_public: true }, current_user: current_user).execute
     @total_starred_projects_count = ProjectsFinder.new(params: { starred: true }, current_user: current_user).execute
 
+    finder_params[:use_cte] = Feature.enabled?(:use_cte_for_projects_finder, default_enabled: true)
+
     projects = ProjectsFinder
                 .new(params: finder_params, current_user: current_user)
                 .execute

app/finders/projects_finder.rb

@@ -44,6 +44,8 @@ class ProjectsFinder < UnionFinder
         init_collection
       end
 
+    use_cte = params.delete(:use_cte)
+    collection = Project.wrap_authorized_projects_with_cte(collection) if use_cte
     collection = filter_projects(collection)
     sort(collection)
   end
@@ -177,7 +179,7 @@ class ProjectsFinder < UnionFinder
   end
 
   def sort(items)
-    params[:sort].present? ? items.sort_by_attribute(params[:sort]) : items.order_id_desc
+    params[:sort].present? ? items.sort_by_attribute(params[:sort]) : items.projects_order_id_desc
   end
 
   def by_archived(projects)

app/models/project.rb

@@ -397,6 +397,8 @@ class Project < ApplicationRecord
   scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) }
   scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) }
   scope :sorted_by_name_asc_limited, ->(limit) { reorder(name: :asc).limit(limit) }
+  # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name
+  scope :projects_order_id_desc, -> { reorder("#{table_name}.id DESC") }
 
   scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
@@ -543,6 +545,11 @@ class Project < ApplicationRecord
       )
   end
 
+  def self.wrap_authorized_projects_with_cte(collection)
+    cte = Gitlab::SQL::CTE.new(:authorized_projects, collection)
+    Project.with(cte.to_arel).from(cte.alias_to(Project.arel_table))
+  end
+
   scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') }
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 

changelogs/unreleased/sh-use-cte-for-projects-finder.yml

+---
+title: Use CTE optimization fence for loading projects in dashboard
+merge_request: 23754
+author:
+type: performance

spec/models/project_spec.rb

@@ -3780,6 +3780,25 @@ describe Project do
     end
   end
 
+  describe '.wrap_authorized_projects_with_cte' do
+    let!(:user) { create(:user) }
+
+    let!(:private_project) do
+      create(:project, :private, creator: user, namespace: user.namespace)
+    end
+
+    let!(:public_project) { create(:project, :public) }
+
+    let(:projects) { described_class.all.public_or_visible_to_user(user) }
+
+    subject { described_class.wrap_authorized_projects_with_cte(projects) }
+
+    it 'wrapped query matches original' do
+      expect(subject.to_sql).to match(/^WITH "authorized_projects" AS/)
+      expect(subject).to match_array(projects)
+    end
+  end
+
   describe '#pages_available?' do
     let(:project) { create(:project, group: group) }