Merge branch 'ee-search-100' into 'master'

[EE] Use limit for search count queries

See merge request gitlab-org/gitlab-ee!4158

app/finders/issues_finder.rb

@@ -15,6 +15,7 @@
 #     label_name: string
 #     sort: string
 #     my_reaction_emoji: string
+#     public_only: boolean
 #
 class IssuesFinder < IssuableFinder
   CONFIDENTIAL_ACCESS_LEVEL = Gitlab::Access::REPORTER
@@ -40,7 +41,15 @@ class IssuesFinder < IssuableFinder
   private
 
   def init_collection
-    with_confidentiality_access_check
+    if public_only?
+      Issue.public_only
+    else
+      with_confidentiality_access_check
+    end
+  end
+
+  def public_only?
+    params.fetch(:public_only, false)
   end
 
   def user_can_see_all_confidential_issues?

app/helpers/search_helper.rb

@@ -180,4 +180,8 @@ module SearchHelper
     # Truncato's filtered_tags and filtered_attributes are not quite the same
     sanitize(html, tags: %w(a p ol ul li pre code))
   end
+
+  def limited_count(count, limit = 1000)
+    count > limit ? "#{limit}+" : count
+  end
 end

app/views/search/_category.html.haml

@@ -63,35 +63,35 @@
         = link_to search_filter_path(scope: 'projects') do
           Projects
           %span.badge
-            = @search_results.projects_count
+            = limited_count(@search_results.limited_projects_count)
       %li{ class: active_when(@scope == 'issues') }
         = link_to search_filter_path(scope: 'issues') do
           Issues
           %span.badge
-            = @search_results.issues_count
+            = limited_count(@search_results.limited_issues_count)
       %li{ class: active_when(@scope == 'merge_requests') }
         = link_to search_filter_path(scope: 'merge_requests') do
           Merge requests
           %span.badge
-            = @search_results.merge_requests_count
+            = limited_count(@search_results.limited_merge_requests_count)
       %li{ class: active_when(@scope == 'milestones') }
         = link_to search_filter_path(scope: 'milestones') do
           Milestones
           %span.badge
-            = @search_results.milestones_count
+            = limited_count(@search_results.limited_milestones_count)
       - if current_application_settings.elasticsearch_search?
         %li{ class: active_when(@scope == 'blobs') }
           = link_to search_filter_path(scope: 'blobs') do
             Code
             %span.badge
-              = @search_results.blobs_count
+              = limited_count(@search_results.blobs_count)
         %li{ class: active_when(@scope == 'commits') }
           = link_to search_filter_path(scope: 'commits') do
             Commits
             %span.badge
-              = @search_results.commits_count
+              = limited_count(@search_results.commits_count)
         %li{ class: active_when(@scope == 'wiki_blobs') }
           = link_to search_filter_path(scope: 'wiki_blobs') do
             Wiki
             %span.badge
-              = @search_results.wiki_blobs_count
+              = limited_count(@search_results.wiki_blobs_count)

app/views/search/_results.html.haml

@@ -3,7 +3,8 @@
   = render 'shared/promotions/promote_advanced_search'
 - else
   .row-content-block
-    = search_entries_info(@search_objects, @scope, @search_term)
+    - unless @search_objects.is_a?(Kaminari::PaginatableWithoutCount)
+      = search_entries_info(@search_objects, @scope, @search_term)
     - unless @show_snippets
       - if @project
         in project #{link_to @project.name_with_namespace, [@project.namespace.becomes(Namespace), @project]}
@@ -23,4 +24,4 @@
           = render partial: "search/results/#{@scope.singularize}", collection: @search_objects
 
     - if @scope != 'projects'
-      = paginate(@search_objects, theme: 'gitlab')
+      = paginate_collection(@search_objects)

changelogs/unreleased/40540-use-limit-for-global-search.yml

+---
+title: Optimize search queries on the search page by setting a limit for matching records.
+merge_request:
+author:
+type: performance

db/migrate/20180115201419_add_index_updated_at_to_issues.rb

+class AddIndexUpdatedAtToIssues < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :issues, :updated_at
+  end
+
+  def down
+    remove_concurrent_index :issues, :updated_at
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180113220114) do
+ActiveRecord::Schema.define(version: 20180115201419) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1224,6 +1224,7 @@ ActiveRecord::Schema.define(version: 20180113220114) do
   add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree
   add_index "issues", ["state"], name: "index_issues_on_state", using: :btree
   add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
+  add_index "issues", ["updated_at"], name: "index_issues_on_updated_at", using: :btree
   add_index "issues", ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)", using: :btree
 
   create_table "keys", force: :cascade do |t|

lib/api/v3/projects.rb

@@ -179,7 +179,7 @@ module API
         end
         get "/search/:query", requirements: { query: /[^\/]+/ } do
           search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
-          projects = search_service.objects('projects', params[:page])
+          projects = search_service.objects('projects', params[:page], false)
           projects = projects.reorder(params[:order_by] => params[:sort])
 
           present paginate(projects), with: ::API::V3::Entities::Project

lib/gitlab/elastic/search_results.rb

@@ -38,6 +38,7 @@ module Gitlab
       def projects_count
         @projects_count ||= projects.total_count
       end
+      alias_method :limited_projects_count, :projects_count
 
       def blobs_count
         @blobs_count ||= blobs.total_count
@@ -54,14 +55,17 @@ module Gitlab
       def issues_count
         @issues_count ||= issues.total_count
       end
+      alias_method :limited_issues_count, :issues_count
 
       def merge_requests_count
         @merge_requests_count ||= merge_requests.total_count
       end
+      alias_method :limited_merge_requests_count, :merge_requests_count
 
       def milestones_count
         @milestones_count ||= milestones.total_count
       end
+      alias_method :limited_milestones_count, :milestones_count
 
       def single_commit_result?
         false

lib/gitlab/project_search_results.rb

@@ -20,7 +20,7 @@ module Gitlab
       when 'commits'
         Kaminari.paginate_array(commits).page(page).per(per_page)
       else
-        super
+        super(scope, page, false)
       end
     end
 

lib/gitlab/search_results.rb

@@ -40,19 +40,21 @@ module Gitlab
       @default_project_filter = default_project_filter
     end
 
-    def objects(scope, page = nil)
-      case scope
-      when 'projects'
-        projects.page(page).per(per_page)
-      when 'issues'
-        issues.page(page).per(per_page)
-      when 'merge_requests'
-        merge_requests.page(page).per(per_page)
-      when 'milestones'
-        milestones.page(page).per(per_page)
-      else
-        Kaminari.paginate_array([]).page(page).per(per_page)
-      end
+    def objects(scope, page = nil, without_count = true)
+      collection = case scope
+                   when 'projects'
+                     projects.page(page).per(per_page)
+                   when 'issues'
+                     issues.page(page).per(per_page)
+                   when 'merge_requests'
+                     merge_requests.page(page).per(per_page)
+                   when 'milestones'
+                     milestones.page(page).per(per_page)
+                   else
+                     Kaminari.paginate_array([]).page(page).per(per_page)
+                   end
+
+      without_count ? collection.without_count : collection
     end
 
     def projects_count
@@ -71,18 +73,46 @@ module Gitlab
       @milestones_count ||= milestones.count
     end
 
+    def limited_projects_count
+      @limited_projects_count ||= projects.limit(count_limit).count
+    end
+
+    def limited_issues_count
+      return @limited_issues_count if @limited_issues_count
+
+      # By default getting limited count (e.g. 1000+) is fast on issuable
+      # collections except for issues, where filtering both not confidential
+      # and confidential issues user has access to, is too complex.
+      # It's faster to try to fetch all public issues first, then only
+      # if necessary try to fetch all issues.
+      sum = issues(public_only: true).limit(count_limit).count
+      @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum
+    end
+
+    def limited_merge_requests_count
+      @limited_merge_requests_count ||= merge_requests.limit(count_limit).count
+    end
+
+    def limited_milestones_count
+      @limited_milestones_count ||= milestones.limit(count_limit).count
+    end
+
     def single_commit_result?
       false
     end
 
+    def count_limit
+      1001
+    end
+
     private
 
     def projects
       limit_projects.search(query)
     end
 
-    def issues
-      issues = IssuesFinder.new(current_user).execute
+    def issues(finder_params = {})
+      issues = IssuesFinder.new(current_user, finder_params).execute
       unless default_project_filter
         issues = issues.where(project_id: project_ids_relation)
       end
@@ -94,13 +124,13 @@ module Gitlab
           issues.full_search(query)
         end
 
-      issues.order('updated_at DESC')
+      issues.reorder('updated_at DESC')
     end
 
     def milestones
       milestones = Milestone.where(project_id: project_ids_relation)
       milestones = milestones.search(query)
-      milestones.order('updated_at DESC')
+      milestones.reorder('updated_at DESC')
     end
 
     def merge_requests
@@ -116,7 +146,7 @@ module Gitlab
           merge_requests.full_search(query)
         end
 
-      merge_requests.order('updated_at DESC')
+      merge_requests.reorder('updated_at DESC')
     end
 
     def default_scope

lib/gitlab/snippet_search_results.rb

@@ -16,7 +16,7 @@ module Gitlab
       when 'snippet_blobs'
         snippet_blobs.page(page).per(per_page)
       else
-        super
+        super(scope, nil, false)
       end
     end
 

spec/features/global_search_spec.rb

@@ -22,7 +22,7 @@ feature 'Global search' do
       click_button "Go"
 
       select_filter("Issues")
-      expect(page).to have_selector('.gl-pagination .page', count: 2)
+      expect(page).to have_selector('.gl-pagination .next')
     end
   end
 end

spec/lib/gitlab/search_results_spec.rb

@@ -19,6 +19,12 @@ describe Gitlab::SearchResults do
       project.add_developer(user)
     end
 
+    describe '#objects' do
+      it 'returns without_page collection by default' do
+        expect(results.objects('projects')).to be_kind_of(Kaminari::PaginatableWithoutCount)
+      end
+    end
+
     describe '#projects_count' do
       it 'returns the total amount of projects' do
         expect(results.projects_count).to eq(1)
@@ -43,6 +49,58 @@ describe Gitlab::SearchResults do
       end
     end
 
+    context "when count_limit is lower than total amount" do
+      before do
+        allow(results).to receive(:count_limit).and_return(1)
+      end
+
+      describe '#limited_projects_count' do
+        it 'returns the limited amount of projects' do
+          create(:project, name: 'foo2')
+
+          expect(results.limited_projects_count).to eq(1)
+        end
+      end
+
+      describe '#limited_merge_requests_count' do
+        it 'returns the limited amount of merge requests' do
+          create(:merge_request, :simple, source_project: project, title: 'foo2')
+
+          expect(results.limited_merge_requests_count).to eq(1)
+        end
+      end
+
+      describe '#limited_milestones_count' do
+        it 'returns the limited amount of milestones' do
+          create(:milestone, project: project, title: 'foo2')
+
+          expect(results.limited_milestones_count).to eq(1)
+        end
+      end
+
+      describe '#limited_issues_count' do
+        it 'runs single SQL query to get the limited amount of issues' do
+          create(:milestone, project: project, title: 'foo2')
+
+          expect(results).to receive(:issues).with(public_only: true).and_call_original
+          expect(results).not_to receive(:issues).with(no_args).and_call_original
+
+          expect(results.limited_issues_count).to eq(1)
+        end
+      end
+    end
+
+    context "when count_limit is higher than total amount" do
+      describe '#limited_issues_count' do
+        it 'runs multiple queries to get the limited amount of issues' do
+          expect(results).to receive(:issues).with(public_only: true).and_call_original
+          expect(results).to receive(:issues).with(no_args).and_call_original
+
+          expect(results.limited_issues_count).to eq(1)
+        end
+      end
+    end
+
     it 'includes merge requests from source and target projects' do
       forked_project = fork_project(project, user)
       merge_request_2 = create(:merge_request, target_project: project, source_project: forked_project, title: 'foo')