Fix performance of GroupsWithTemplatesFinder query

- Introduce OptimizedGroupsWithTemplatesFinder that queries group
templates more efficiently.
- Introduce a feature flag to enable the optimization:
`optimized_groups_with_templates_finder`

ee/app/finders/groups_with_templates_finder.rb

@@ -21,15 +21,30 @@ class GroupsWithTemplatesFinder
 
   attr_reader :group_id
 
+  # Cleanup issue: https://gitlab.com/gitlab-org/gitlab/issues/35733
   def extended_group_search
+    if ::Feature.enabled?(:optimized_groups_with_templates_finder)
+      groups = Group.with_project_templates_optimized
+      groups_with_plan = Gitlab::ObjectHierarchy
+        .new(groups)
+        .base_and_ancestors
+        .with_feature_available_in_plan(:group_project_templates)
+
+      Gitlab::ObjectHierarchy.new(groups_with_plan).base_and_descendants
+    else
       groups = Group.with_feature_available_in_plan(:group_project_templates)
       Gitlab::ObjectHierarchy.new(groups).base_and_descendants
     end
+  end
 
   def simple_group_search(groups)
     groups = group_id ? groups.find_by(id: group_id)&.self_and_ancestors : groups # rubocop: disable CodeReuse/ActiveRecord
     return Group.none unless groups
 
+    if ::Feature.enabled?(:optimized_groups_with_templates_finder)
+      groups.with_project_templates_optimized
+    else
       groups.with_project_templates
     end
+  end
 end

ee/app/models/ee/group.rb

@@ -62,6 +62,8 @@ module EE
           .distinct
       end
 
+      scope :with_project_templates_optimized, -> { where.not(custom_project_templates_group_id: nil) }
+
       scope :with_custom_file_templates, -> do
         preload(
           file_template_project: :route,

ee/app/models/ee/user.rb

@@ -207,15 +207,25 @@ module EE
     end
 
     def available_subgroups_with_custom_project_templates(group_id = nil)
-      groups = GroupsWithTemplatesFinder.new(group_id).execute
+      found_groups = GroupsWithTemplatesFinder.new(group_id).execute
 
+      if ::Feature.enabled?(:optimized_groups_with_templates_finder)
         GroupsFinder.new(self, min_access_level: ::Gitlab::Access::DEVELOPER)
           .execute
-                  .where(id: groups.select(:custom_project_templates_group_id))
+          .where(id: found_groups.select(:custom_project_templates_group_id))
+          .preload(:projects)
+          .joins(:projects)
+          .reorder(nil)
+          .distinct
+      else
+        GroupsFinder.new(self, min_access_level: ::Gitlab::Access::DEVELOPER)
+          .execute
+          .where(id: found_groups.select(:custom_project_templates_group_id))
           .includes(:projects)
           .reorder(nil)
           .distinct
       end
+    end
 
     def roadmap_layout
       super || DEFAULT_ROADMAP_LAYOUT

ee/changelogs/unreleased/maybe-fix-projects-newperformance.yml

+---
+title: Fix new project page load performance
+merge_request: 18180
+author:
+type: performance

ee/spec/features/projects/new_project_spec.rb

@@ -269,7 +269,7 @@ describe 'New project' do
         end
       end
 
-      context 'when custom project group template is set' do
+      shared_context 'when custom project group template is set' do
         let(:group1) { create(:group) }
         let(:group2) { create(:group) }
         let(:group3) { create(:group) }
@@ -429,6 +429,23 @@ describe 'New project' do
         end
       end
 
+      # Cleanup issue: https://gitlab.com/gitlab-org/gitlab/issues/35733
+      context 'when `optimized_groups_with_templates_finder` feature flag is enabled' do
+        before do
+          stub_feature_flags(optimized_groups_with_templates_finder: true)
+        end
+
+        include_context 'when custom project group template is set'
+      end
+
+      context 'when `optimized_groups_with_templates_finder` feature flag is disabled' do
+        before do
+          stub_feature_flags(optimized_groups_with_templates_finder: false)
+        end
+
+        include_context 'when custom project group template is set'
+      end
+
       context 'when group template is not set' do
         it_behaves_like 'group templates displayed' do
           let(:template_number) { 0 }

ee/spec/finders/groups_with_templates_finder_spec.rb

@@ -26,6 +26,7 @@ describe GroupsWithTemplatesFinder do
     create(:gitlab_subscription, :silver, namespace: group_2)
   end
 
+  shared_examples 'GroupsWithTemplatesFinder examples' do
     describe 'without group id' do
       it 'returns all groups' do
         expect(described_class.new.execute).to contain_exactly(group_1, group_2, group_3)
@@ -116,4 +117,21 @@ describe GroupsWithTemplatesFinder do
         end
       end
     end
+  end
+
+  context 'when `optimized_groups_with_templates_finder` feature flag is enabled' do
+    before do
+      stub_feature_flags(optimized_groups_with_templates_finder: true)
+    end
+
+    include_examples 'GroupsWithTemplatesFinder examples'
+  end
+
+  context 'when `optimized_groups_with_templates_finder` feature flag is disabled' do
+    before do
+      stub_feature_flags(optimized_groups_with_templates_finder: false)
+    end
+
+    include_examples 'GroupsWithTemplatesFinder examples'
+  end
 end