Merge branch '2245-search-api' into 'master'

Search API

Closes #2245

See merge request gitlab-org/gitlab-ee!4403

changelogs/unreleased/41763-search-api.yml

+---
+title: Add search support into the API
+merge_request: 16878
+author:
+type: added

lib/api/api.rb

@@ -164,6 +164,7 @@ module API
     mount ::API::Repositories
     mount ::API::Runner
     mount ::API::Runners
+    mount ::API::Search
     mount ::API::Services
     mount ::API::Settings
     mount ::API::SidekiqMetrics

lib/api/entities.rb

@@ -343,24 +343,20 @@ module API
       end
     end
 
-    class ProjectSnippet < Grape::Entity
+    class Snippet < Grape::Entity
       expose :id, :title, :file_name, :description
       expose :author, using: Entities::UserBasic
       expose :updated_at, :created_at
-
-      expose :web_url do |snippet, options|
+      expose :project_id
+      expose :web_url do |snippet|
         Gitlab::UrlBuilder.build(snippet)
       end
     end
 
-    class PersonalSnippet < Grape::Entity
-      expose :id, :title, :file_name, :description
-      expose :author, using: Entities::UserBasic
-      expose :updated_at, :created_at
+    class ProjectSnippet < Snippet
+    end
 
-      expose :web_url do |snippet|
-        Gitlab::UrlBuilder.build(snippet)
-      end
+    class PersonalSnippet < Snippet
       expose :raw_url do |snippet|
         Gitlab::UrlBuilder.build(snippet) + "/raw"
       end
@@ -1446,5 +1442,14 @@ module API
     class ApplicationWithSecret < Application
       expose :secret
     end
+
+    class Blob < Grape::Entity
+      expose :basename
+      expose :data
+      expose :filename
+      expose :id
+      expose :ref
+      expose :startline
+    end
   end
 end

lib/api/helpers/pagination.rb

@@ -12,13 +12,16 @@ module API
       private
 
       def add_pagination_headers(paginated_data)
-        header 'X-Total',       paginated_data.total_count.to_s
-        header 'X-Total-Pages', total_pages(paginated_data).to_s
         header 'X-Per-Page',    paginated_data.limit_value.to_s
         header 'X-Page',        paginated_data.current_page.to_s
         header 'X-Next-Page',   paginated_data.next_page.to_s
         header 'X-Prev-Page',   paginated_data.prev_page.to_s
         header 'Link',          pagination_links(paginated_data)
+
+        return if data_without_counts?(paginated_data)
+
+        header 'X-Total',       paginated_data.total_count.to_s
+        header 'X-Total-Pages', total_pages(paginated_data).to_s
       end
 
       def pagination_links(paginated_data)
@@ -37,8 +40,10 @@ module API
         request_params[:page] = 1
         links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
 
-        request_params[:page] = total_pages(paginated_data)
-        links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+        unless data_without_counts?(paginated_data)
+          request_params[:page] = total_pages(paginated_data)
+          links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
+        end
 
         links.join(', ')
       end
@@ -55,6 +60,10 @@ module API
 
         relation
       end
+
+      def data_without_counts?(paginated_data)
+        paginated_data.is_a?(Kaminari::PaginatableWithoutCount)
+      end
     end
   end
 end

lib/api/search.rb

+module API
+  class Search < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    helpers do
+      SCOPE_ENTITY = {
+        merge_requests: Entities::MergeRequestBasic,
+        issues: Entities::IssueBasic,
+        projects: Entities::BasicProjectDetails,
+        milestones: Entities::Milestone,
+        notes: Entities::Note,
+        commits: Entities::Commit,
+        blobs: Entities::Blob,
+        wiki_blobs: Entities::Blob,
+        snippet_titles: Entities::Snippet,
+        snippet_blobs: Entities::Snippet
+      }.freeze
+
+      ELASTICSEARCH_SCOPES = %w(wiki_blobs blobs commits).freeze
+
+      def search(additional_params = {})
+        search_params = {
+          scope: params[:scope],
+          search: params[:search],
+          snippets: snippets?,
+          page: params[:page],
+          per_page: params[:per_page]
+        }.merge(additional_params)
+
+        results = SearchService.new(current_user, search_params).search_objects
+
+        process_results(results)
+      end
+
+      def process_results(results)
+        return [] if results.empty?
+
+        if results.is_a?(Elasticsearch::Model::Response::Response)
+          return paginate(results).map { |blob| Gitlab::Elastic::SearchResults.parse_search_result(blob) }
+        end
+
+        case params[:scope]
+        when 'wiki_blobs'
+          paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) }
+        when 'blobs'
+          paginate(results).map { |blob| blob[1] }
+        else
+          paginate(results)
+        end
+      end
+
+      def snippets?
+        %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s
+      end
+
+      def entity
+        SCOPE_ENTITY[params[:scope].to_sym]
+      end
+
+      def check_elasticsearch_scope!
+        if ELASTICSEARCH_SCOPES.include?(params[:scope]) && !elasticsearch?
+          render_api_error!({ error: 'Scope not supported without Elasticsearch!' }, 400)
+        end
+      end
+
+      def elasticsearch?
+        Gitlab::CurrentSettings.elasticsearch_search?
+      end
+    end
+
+    resource :search do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs,
+            if Elasticsearch enabled: wiki_blobs, blobs, commits',
+          values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs
+                     wiki_blobs blobs commits)
+        use :pagination
+      end
+      get do
+        check_elasticsearch_scope!
+
+        present search, with: entity
+      end
+    end
+
+    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            projects, issues, merge_requests, milestones,
+            if Elasticsearch enabled: wiki_blobs, blobs, commits',
+          values: %w(projects issues merge_requests milestones wiki_blobs blobs commits)
+        use :pagination
+      end
+      get ':id/-/search' do
+        check_elasticsearch_scope!
+        find_group!(params[:id])
+
+        present search(group_id: params[:id]), with: entity
+      end
+    end
+
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs',
+          values: %w(issues merge_requests milestones notes wiki_blobs commits blobs)
+        use :pagination
+      end
+      get ':id/-/search' do
+        find_project!(params[:id])
+
+        present search(project_id: params[:id]), with: entity
+      end
+    end
+  end
+end

lib/api/v3/projects.rb

@@ -178,7 +178,7 @@ module API
           use :pagination
         end
         get "/search/:query", requirements: { query: %r{[^/]+} } do
-          search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
+          search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute
           projects = search_service.objects('projects', params[:page], false)
           projects = projects.reorder(params[:order_by] => params[:sort])
 

lib/gitlab/project_search_results.rb

@@ -2,11 +2,12 @@ module Gitlab
   class ProjectSearchResults < SearchResults
     attr_reader :project, :repository_ref
 
-    def initialize(current_user, project, query, repository_ref = nil)
+    def initialize(current_user, project, query, repository_ref = nil, per_page: 20)
       @current_user = current_user
       @project = project
       @repository_ref = repository_ref.presence || project.default_branch
       @query = query
+      @per_page = per_page
     end
 
     def objects(scope, page = nil)

lib/gitlab/search_results.rb

@@ -10,6 +10,7 @@ module Gitlab
         @ref = opts.fetch(:ref, nil)
         @startline = opts.fetch(:startline, nil)
         @data = opts.fetch(:data, nil)
+        @per_page = opts.fetch(:per_page, 20)
       end
 
       def path
@@ -21,7 +22,7 @@ module Gitlab
       end
     end
 
-    attr_reader :current_user, :query
+    attr_reader :current_user, :query, :per_page
 
     # Limit search results by passed projects
     # It allows us to search only for projects user has access to
@@ -33,11 +34,12 @@ module Gitlab
     # query
     attr_reader :default_project_filter
 
-    def initialize(current_user, limit_projects, query, default_project_filter: false)
+    def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20)
       @current_user = current_user
       @limit_projects = limit_projects || Project.all
       @query = query
       @default_project_filter = default_project_filter
+      @per_page = per_page
     end
 
     def objects(scope, page = nil, without_count = true)
@@ -153,10 +155,6 @@ module Gitlab
       'projects'
     end
 
-    def per_page
-      20
-    end
-
     def project_ids_relation
       limit_projects.select(:id).reorder(nil)
     end

spec/fixtures/api/schemas/public_api/v4/blobs.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "basename": { "type": "string" },
+      "data": { "type": "string" },
+      "filename": { "type": ["string"] },
+      "id": { "type": ["string", "null"] },
+      "ref": { "type": "string" },
+      "startline": { "type": "integer" }
+    },
+    "required": [
+      "basename", "data", "filename", "id", "ref", "startline"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/merge_requests.json

@@ -28,7 +28,7 @@
         "additionalProperties": false
       },
       "assignee": {
-        "type": "object",
+        "type": ["object", "null"],
         "properties": {
           "name": { "type": "string" },
           "username": { "type": "string" },

spec/fixtures/api/schemas/public_api/v4/milestones.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "iid": { "type": "integer" },
+      "project_id": { "type": ["integer", "null"] },
+      "group_id": { "type": ["integer", "null"] },
+      "title": { "type": "string" },
+      "description": { "type": ["string", "null"] },
+      "state": { "type": "string" },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "start_date": { "type": "date" },
+      "due_date": { "type": "date" }
+    },
+    "required": [
+      "id", "iid", "title", "description", "state",
+      "state", "created_at", "updated_at", "start_date", "due_date"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/notes.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "body": { "type": "string" },
+      "attachment": { "type": ["string", "null"] },
+      "author": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "username": { "type": "string" },
+          "id": { "type": "integer" },
+          "state": { "type": "string" },
+          "avatar_url": { "type": "uri" },
+          "web_url": { "type": "uri" }
+        },
+        "additionalProperties": false
+      },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "system": { "type": "boolean" },
+      "noteable_id": { "type": "integer" },
+      "noteable_iid": { "type": "integer" },
+      "noteable_type": { "type": "string" }
+    },
+    "required": [
+      "id", "body", "attachment", "author", "created_at", "updated_at",
+      "system", "noteable_id", "noteable_type"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/projects.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "name": { "type": "string" },
+      "name_with_namespace": { "type": "string" },
+      "description": { "type": ["string", "null"] },
+      "path": { "type": "string" },
+      "path_with_namespace": { "type": "string" },
+      "created_at": { "type": "date" },
+      "default_branch": { "type": ["string", "null"] },
+      "tag_list": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
+      "ssh_url_to_repo": { "type": "string" },
+      "http_url_to_repo": { "type": "string" },
+      "web_url": { "type": "string" },
+      "avatar_url": { "type": ["string", "null"] },
+      "star_count": { "type": "integer" },
+      "forks_count": { "type": "integer" },
+      "last_activity_at": { "type": "date" }
+    },
+    "required": [
+      "id", "name", "name_with_namespace", "description", "path",
+      "path_with_namespace", "created_at", "default_branch", "tag_list",
+      "ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
+      "star_count", "last_activity_at"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/snippets.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "project_id": { "type": ["integer", "null"] },
+      "title": { "type": "string" },
+      "file_name": { "type": ["string", "null"] },
+      "description": { "type": ["string", "null"] },
+      "web_url": { "type": "string" },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "author": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "username": { "type": "string" },
+          "id": { "type": "integer" },
+          "state": { "type": "string" },
+          "avatar_url": { "type": "uri" },
+          "web_url": { "type": "uri" }
+        },
+        "additionalProperties": false
+      }
+    },
+    "required": [
+      "id", "title", "file_name", "description", "web_url",
+      "created_at", "updated_at", "author"
+    ],
+    "additionalProperties": false
+  }
+}

spec/lib/gitlab/search_results_spec.rb

@@ -20,9 +20,13 @@ describe Gitlab::SearchResults do
     end
 
     describe '#objects' do
-      it 'returns without_page collection by default' do
+      it 'returns without_counts collection by default' do
         expect(results.objects('projects')).to be_kind_of(Kaminari::PaginatableWithoutCount)
       end
+
+      it 'returns with counts collection when requested' do
+        expect(results.objects('projects', 1, false)).not_to be_kind_of(Kaminari::PaginatableWithoutCount)
+      end
     end
 
     describe '#projects_count' do

spec/support/matchers/pagination_matcher.rb

@@ -3,3 +3,9 @@ RSpec::Matchers.define :include_pagination_headers do |expected|
     expect(actual.headers).to include('X-Total', 'X-Total-Pages', 'X-Per-Page', 'X-Page', 'X-Next-Page', 'X-Prev-Page', 'Link')
   end
 end
+
+RSpec::Matchers.define :include_limited_pagination_headers do |expected|
+  match do |actual|
+    expect(actual.headers).to include('X-Per-Page', 'X-Page', 'X-Next-Page', 'X-Prev-Page', 'Link')
+  end
+end