Merge branch 'sh-use-cte-for-projects-finder' into 'master'

Use CTE optimization fence for loading projects in dashboard

Closes #198440

See merge request gitlab-org/gitlab!23754

app/controllers/dashboard/projects_controller.rb

@@ -66,6 +66,8 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController
     @total_user_projects_count = ProjectsFinder.new(params: { non_public: true }, current_user: current_user).execute
     @total_starred_projects_count = ProjectsFinder.new(params: { starred: true }, current_user: current_user).execute
 
+    finder_params[:use_cte] = Feature.enabled?(:use_cte_for_projects_finder, default_enabled: true)
+
     projects = ProjectsFinder
                 .new(params: finder_params, current_user: current_user)
                 .execute

app/finders/projects_finder.rb

@@ -44,6 +44,8 @@ class ProjectsFinder < UnionFinder
         init_collection
       end
 
+    use_cte = params.delete(:use_cte)
+    collection = Project.wrap_authorized_projects_with_cte(collection) if use_cte
     collection = filter_projects(collection)
     sort(collection)
   end
@@ -177,7 +179,7 @@ class ProjectsFinder < UnionFinder
   end
 
   def sort(items)
-    params[:sort].present? ? items.sort_by_attribute(params[:sort]) : items.order_id_desc
+    params[:sort].present? ? items.sort_by_attribute(params[:sort]) : items.projects_order_id_desc
   end
 
   def by_archived(projects)

app/models/project.rb

@@ -397,6 +397,8 @@ class Project < ApplicationRecord
   scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) }
   scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) }
   scope :sorted_by_name_asc_limited, ->(limit) { reorder(name: :asc).limit(limit) }
+  # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name
+  scope :projects_order_id_desc, -> { reorder("#{table_name}.id DESC") }
 
   scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
@@ -543,6 +545,11 @@ class Project < ApplicationRecord
       )
   end
 
+  def self.wrap_authorized_projects_with_cte(collection)
+    cte = Gitlab::SQL::CTE.new(:authorized_projects, collection)
+    Project.with(cte.to_arel).from(cte.alias_to(Project.arel_table))
+  end
+
   scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') }
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 

changelogs/unreleased/sh-use-cte-for-projects-finder.yml

+---
+title: Use CTE optimization fence for loading projects in dashboard
+merge_request: 23754
+author:
+type: performance

spec/finders/projects_finder_spec.rb

@@ -28,10 +28,12 @@ describe ProjectsFinder, :do_not_mock_admin_mode do
     let(:params) { {} }
     let(:current_user) { user }
     let(:project_ids_relation) { nil }
-    let(:finder) { described_class.new(params: params, current_user: current_user, project_ids_relation: project_ids_relation) }
+    let(:use_cte) { true }
+    let(:finder) { described_class.new(params: params.merge(use_cte: use_cte), current_user: current_user, project_ids_relation: project_ids_relation) }
 
     subject { finder.execute }
 
+    shared_examples 'ProjectFinder#execute examples' do
       describe 'without a user' do
         let(:current_user) { nil }
 
@@ -202,6 +204,8 @@ describe ProjectsFinder, :do_not_mock_admin_mode do
           current_user.toggle_star(private_project)
 
           is_expected.to eq([public_project])
+          expect(subject.count).to eq(1)
+          expect(subject.limit(1000).count).to eq(1)
         end
       end
 
@@ -234,4 +238,17 @@ describe ProjectsFinder, :do_not_mock_admin_mode do
         end
       end
     end
+
+    describe 'without CTE flag enabled' do
+      let(:use_cte) { false }
+
+      it_behaves_like 'ProjectFinder#execute examples'
+    end
+
+    describe 'with CTE flag enabled' do
+      let(:use_cte) { true }
+
+      it_behaves_like 'ProjectFinder#execute examples'
+    end
+  end
 end

spec/models/project_spec.rb

@@ -3780,6 +3780,25 @@ describe Project do
     end
   end
 
+  describe '.wrap_authorized_projects_with_cte' do
+    let!(:user) { create(:user) }
+
+    let!(:private_project) do
+      create(:project, :private, creator: user, namespace: user.namespace)
+    end
+
+    let!(:public_project) { create(:project, :public) }
+
+    let(:projects) { described_class.all.public_or_visible_to_user(user) }
+
+    subject { described_class.wrap_authorized_projects_with_cte(projects) }
+
+    it 'wrapped query matches original' do
+      expect(subject.to_sql).to match(/^WITH "authorized_projects" AS/)
+      expect(subject).to match_array(projects)
+    end
+  end
+
   describe '#pages_available?' do
     let(:project) { create(:project, group: group) }