Support search in API

app/services/search_service.rb

@@ -43,7 +43,7 @@ class SearchService
   end
 
   def search_objects
-    @search_objects ||= search_results.objects(scope, params[:page])
+    @search_objects ||= search_results.objects(scope, params[:page], params[:without_counts])
   end
 
   private

changelogs/unreleased/41763-search-api.yml

+---
+title: Add search support into the API
+merge_request: 16878
+author:
+type: added

lib/api/api.rb

@@ -146,6 +146,7 @@ module API
     mount ::API::Repositories
     mount ::API::Runner
     mount ::API::Runners
+    mount ::API::Search
     mount ::API::Services
     mount ::API::Settings
     mount ::API::SidekiqMetrics

lib/api/entities.rb

@@ -314,24 +314,20 @@ module API
       end
     end
 
-    class ProjectSnippet < Grape::Entity
+    class Snippet < Grape::Entity
       expose :id, :title, :file_name, :description
       expose :author, using: Entities::UserBasic
       expose :updated_at, :created_at
-
-      expose :web_url do |snippet, options|
+      expose :project_id
+      expose :web_url do |snippet|
         Gitlab::UrlBuilder.build(snippet)
       end
     end
 
-    class PersonalSnippet < Grape::Entity
-      expose :id, :title, :file_name, :description
-      expose :author, using: Entities::UserBasic
-      expose :updated_at, :created_at
+    class ProjectSnippet < Snippet
+    end
 
-      expose :web_url do |snippet|
-        Gitlab::UrlBuilder.build(snippet)
-      end
+    class PersonalSnippet < Snippet
       expose :raw_url do |snippet|
         Gitlab::UrlBuilder.build(snippet) + "/raw"
       end
@@ -1168,5 +1164,14 @@ module API
     class ApplicationWithSecret < Application
       expose :secret
     end
+
+    class Blob < Grape::Entity
+      expose :basename
+      expose :data
+      expose :filename
+      expose :id
+      expose :ref
+      expose :startline
+    end
   end
 end

lib/api/search.rb

+module API
+  class Search < Grape::API
+    include PaginationParams
+
+    before { authenticate! }
+
+    helpers do
+      SCOPE_ENTITY = {
+        merge_requests: Entities::MergeRequestBasic,
+        issues: Entities::IssueBasic,
+        projects: Entities::BasicProjectDetails,
+        milestones: Entities::Milestone,
+        notes: Entities::Note,
+        commits: Entities::Commit,
+        blobs: Entities::Blob,
+        wiki_blobs: Entities::Blob,
+        snippet_titles: Entities::Snippet,
+        snippet_blobs: Entities::Snippet
+      }.freeze
+
+      def search(additional_params = {})
+        search_params = {
+          scope: params[:scope],
+          search: params[:search],
+          snippets: snippets?,
+          page: params[:page],
+          per_page: params[:per_page],
+          without_counts: false
+        }.merge(additional_params)
+
+        results = SearchService.new(current_user, search_params).search_objects
+
+        process_results(results)
+      end
+
+      def process_results(results)
+        case params[:scope]
+        when 'wiki_blobs'
+          paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) }
+        when 'blobs'
+          paginate(results).map { |blob| blob[1] }
+        else
+          paginate(results)
+        end
+      end
+
+      def snippets?
+        %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s
+      end
+
+      def entity
+        SCOPE_ENTITY[params[:scope].to_sym]
+      end
+    end
+
+    resource :search do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope, type: String, desc: 'The scope of search, available scopes:
+          projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs',
+          values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs)
+        use :pagination
+      end
+      get do
+        present search, with: entity
+      end
+    end
+
+    resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope, type: String, desc: 'The scope of search, available scopes:
+          projects, issues, merge_requests, milestones',
+          values: %w(projects issues merge_requests milestones)
+        use :pagination
+      end
+      get ':id/-/search' do
+        find_group!(params[:id])
+
+        present search(group_id: params[:id]), with: entity
+      end
+    end
+
+    resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+      desc 'Search on GitLab' do
+        detail 'This feature was introduced in GitLab 10.5.'
+      end
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+        requires :search, type: String, desc: 'The expression it should be searched for'
+        requires :scope, type: String, desc: 'The scope of search, available scopes:
+          issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs',
+          values: %w(issues merge_requests milestones notes wiki_blobs commits blobs)
+        use :pagination
+      end
+      get ':id/-/search' do
+        find_project!(params[:id])
+
+        present search(project_id: params[:id]), with: entity
+      end
+    end
+  end
+end

lib/api/v3/projects.rb

@@ -174,7 +174,7 @@ module API
           use :pagination
         end
         get "/search/:query", requirements: { query: %r{[^/]+} } do
-          search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
+          search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute
           projects = search_service.objects('projects', params[:page], false)
           projects = projects.reorder(params[:order_by] => params[:sort])
 

lib/gitlab/project_search_results.rb

@@ -2,14 +2,15 @@ module Gitlab
   class ProjectSearchResults < SearchResults
     attr_reader :project, :repository_ref
 
-    def initialize(current_user, project, query, repository_ref = nil)
+    def initialize(current_user, project, query, repository_ref = nil, per_page: 20)
       @current_user = current_user
       @project = project
       @repository_ref = repository_ref.presence || project.default_branch
       @query = query
+      @per_page = per_page
     end
 
-    def objects(scope, page = nil)
+    def objects(scope, page = nil, without_counts = true)
       case scope
       when 'notes'
         notes.page(page).per(per_page)
@@ -20,7 +21,7 @@ module Gitlab
       when 'commits'
         Kaminari.paginate_array(commits).page(page).per(per_page)
       else
-        super(scope, page, false)
+        super(scope, page, without_counts)
       end
     end
 

lib/gitlab/search_results.rb

@@ -10,6 +10,7 @@ module Gitlab
         @ref = opts.fetch(:ref, nil)
         @startline = opts.fetch(:startline, nil)
         @data = opts.fetch(:data, nil)
+        @per_page = opts.fetch(:per_page, 20)
       end
 
       def path
@@ -21,7 +22,7 @@ module Gitlab
       end
     end
 
-    attr_reader :current_user, :query
+    attr_reader :current_user, :query, :per_page
 
     # Limit search results by passed projects
     # It allows us to search only for projects user has access to
@@ -33,11 +34,12 @@ module Gitlab
     # query
     attr_reader :default_project_filter
 
-    def initialize(current_user, limit_projects, query, default_project_filter: false)
+    def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20)
       @current_user = current_user
       @limit_projects = limit_projects || Project.all
       @query = query
       @default_project_filter = default_project_filter
+      @per_page = per_page
     end
 
     def objects(scope, page = nil, without_count = true)
@@ -153,10 +155,6 @@ module Gitlab
       'projects'
     end
 
-    def per_page
-      20
-    end
-
     def project_ids_relation
       limit_projects.select(:id).reorder(nil)
     end

lib/gitlab/snippet_search_results.rb

@@ -9,14 +9,14 @@ module Gitlab
       @query = query
     end
 
-    def objects(scope, page = nil)
+    def objects(scope, page = nil, without_counts = true)
       case scope
       when 'snippet_titles'
         snippet_titles.page(page).per(per_page)
       when 'snippet_blobs'
         snippet_blobs.page(page).per(per_page)
       else
-        super(scope, nil, false)
+        super(scope, nil, without_counts)
       end
     end
 

spec/fixtures/api/schemas/public_api/v4/blobs.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "basename": { "type": "string" },
+      "data": { "type": "string" },
+      "filename": { "type": ["string"] },
+      "id": { "type": ["string", "null"] },
+      "ref": { "type": "string" },
+      "startline": { "type": "integer" }
+    },
+    "required": [
+      "basename", "data", "filename", "id", "ref", "startline"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/issues.json

@@ -20,7 +20,7 @@
         }
       },
       "milestone": {
-        "type": "object",
+        "type": ["object", "null"],
         "properties": {
           "id": { "type": "integer" },
           "iid": { "type": "integer" },

spec/fixtures/api/schemas/public_api/v4/merge_requests.json

@@ -28,7 +28,7 @@
         "additionalProperties": false
       },
       "assignee": {
-        "type": "object",
+        "type": ["object", "null"],
         "properties": {
           "name": { "type": "string" },
           "username": { "type": "string" },

spec/fixtures/api/schemas/public_api/v4/milestones.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "iid": { "type": "integer" },
+      "project_id": { "type": ["integer", "null"] },
+      "group_id": { "type": ["integer", "null"] },
+      "title": { "type": "string" },
+      "description": { "type": ["string", "null"] },
+      "state": { "type": "string" },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "start_date": { "type": "date" },
+      "due_date": { "type": "date" }
+    },
+    "required": [
+      "id", "iid", "title", "description", "state",
+      "state", "created_at", "updated_at", "start_date", "due_date"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/notes.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "body": { "type": "string" },
+      "attachment": { "type": ["string", "null"] },
+      "author": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "username": { "type": "string" },
+          "id": { "type": "integer" },
+          "state": { "type": "string" },
+          "avatar_url": { "type": "uri" },
+          "web_url": { "type": "uri" }
+        },
+        "additionalProperties": false
+      },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "system": { "type": "boolean" },
+      "noteable_id": { "type": "integer" },
+      "noteable_iid": { "type": "integer" },
+      "noteable_type": { "type": "string" }
+    },
+    "required": [
+      "id", "body", "attachment", "author", "created_at", "updated_at",
+      "system", "noteable_id", "noteable_type"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/projects.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "name": { "type": "string" },
+      "name_with_namespace": { "type": "string" },
+      "description": { "type": ["string", "null"] },
+      "path": { "type": "string" },
+      "path_with_namespace": { "type": "string" },
+      "created_at": { "type": "date" },
+      "default_branch": { "type": ["string", "null"] },
+      "tag_list": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
+      "ssh_url_to_repo": { "type": "string" },
+      "http_url_to_repo": { "type": "string" },
+      "web_url": { "type": "string" },
+      "avatar_url": { "type": ["string", "null"] },
+      "star_count": { "type": "integer" },
+      "forks_count": { "type": "integer" },
+      "last_activity_at": { "type": "date" }
+    },
+    "required": [
+      "id", "name", "name_with_namespace", "description", "path",
+      "path_with_namespace", "created_at", "default_branch", "tag_list",
+      "ssh_url_to_repo", "http_url_to_repo", "web_url", "avatar_url",
+      "star_count", "last_activity_at"
+    ],
+    "additionalProperties": false
+  }
+}

spec/fixtures/api/schemas/public_api/v4/snippets.json

+{
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties" : {
+      "id": { "type": "integer" },
+      "project_id": { "type": ["integer", "null"] },
+      "title": { "type": "string" },
+      "file_name": { "type": ["string", "null"] },
+      "description": { "type": ["string", "null"] },
+      "web_url": { "type": "string" },
+      "created_at": { "type": "date" },
+      "updated_at": { "type": "date" },
+      "author": {
+        "type": "object",
+        "properties": {
+          "name": { "type": "string" },
+          "username": { "type": "string" },
+          "id": { "type": "integer" },
+          "state": { "type": "string" },
+          "avatar_url": { "type": "uri" },
+          "web_url": { "type": "uri" }
+        },
+        "additionalProperties": false
+      }
+    },
+    "required": [
+      "id", "title", "file_name", "description", "web_url",
+      "created_at", "updated_at", "author"
+    ],
+    "additionalProperties": false
+  }
+}

spec/lib/gitlab/search_results_spec.rb

@@ -20,9 +20,13 @@ describe Gitlab::SearchResults do
     end
 
     describe '#objects' do
-      it 'returns without_page collection by default' do
+      it 'returns without_counts collection by default' do
         expect(results.objects('projects')).to be_kind_of(Kaminari::PaginatableWithoutCount)
       end
+
+      it 'returns with counts collection when requested' do
+        expect(results.objects('projects', 1, false)).not_to be_kind_of(Kaminari::PaginatableWithoutCount)
+      end
     end
 
     describe '#projects_count' do

spec/requests/api/search_spec.rb

+require 'spec_helper'
+
+describe API::Search do
+  set(:user) { create(:user) }
+  set(:group) { create(:group) }
+  set(:project) { create(:project, :public, name: 'awesome project', group: group) }
+  set(:repo_project) { create(:project, :public, :repository, group: group) }
+
+  shared_examples 'response is correct' do |schema:, size: 1|
+    it { expect(response).to have_gitlab_http_status(200) }
+    it { expect(response).to match_response_schema(schema) }
+    it { expect(response).to include_pagination_headers }
+    it { expect(json_response.size).to eq(size) }
+  end
+
+  describe 'GET /search'  do
+    context 'when user is not authenticated' do
+      it 'returns 401 error' do
+        get api('/search'), scope: 'projects', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(401)
+      end
+    end
+
+    context 'when scope is not supported' do
+      it 'returns 400 error' do
+        get api('/search', user), scope: 'unsupported', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'when scope is missing' do
+      it 'returns 400 error' do
+        get api('/search', user), search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'with correct params' do
+      context 'for projects scope' do
+        before do
+          get api('/search', user), scope: 'projects', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/projects'
+      end
+
+      context 'for issues scope' do
+        before do
+          create(:issue, project: project, title: 'awesome issue')
+
+          get api('/search', user), scope: 'issues', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/issues'
+      end
+
+      context 'for merge_requests scope' do
+        before do
+          create(:merge_request, source_project: repo_project, title: 'awesome mr')
+
+          get api('/search', user), scope: 'merge_requests', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/merge_requests'
+      end
+
+      context 'for milestones scope' do
+        before do
+          create(:milestone, project: project, title: 'awesome milestone')
+
+          get api('/search', user), scope: 'milestones', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/milestones'
+      end
+
+      context 'for snippet_titles scope' do
+        before do
+          create(:snippet, :public, title: 'awesome snippet', content: 'snippet content')
+
+          get api('/search', user), scope: 'snippet_titles', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/snippets'
+      end
+
+      context 'for snippet_blobs scope' do
+        before do
+          create(:snippet, :public, title: 'awesome snippet', content: 'snippet content')
+
+          get api('/search', user), scope: 'snippet_blobs', search: 'content'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/snippets'
+      end
+    end
+  end
+
+  describe "GET /groups/:id/-/search" do
+    context 'when user is not authenticated' do
+      it 'returns 401 error' do
+        get api("/groups/#{group.id}/-/search"), scope: 'projects', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(401)
+      end
+    end
+
+    context 'when scope is not supported' do
+      it 'returns 400 error' do
+        get api("/groups/#{group.id}/-/search", user), scope: 'unsupported', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'when scope is missing' do
+      it 'returns 400 error' do
+        get api("/groups/#{group.id}/-/search", user), search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'when group does not exist' do
+      it 'returns 404 error' do
+        get api('/groups/9999/-/search', user), scope: 'issues', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when user does can not see the group' do
+      it 'returns 404 error' do
+        private_group = create(:group, :private)
+
+        get api("/groups/#{private_group.id}/-/search", user), scope: 'issues', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'with correct params' do
+      context 'for projects scope' do
+        before do
+          get api("/groups/#{group.id}/-/search", user), scope: 'projects', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/projects'
+      end
+
+      context 'for issues scope' do
+        before do
+          create(:issue, project: project, title: 'awesome issue')
+
+          get api("/groups/#{group.id}/-/search", user), scope: 'issues', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/issues'
+      end
+
+      context 'for merge_requests scope' do
+        before do
+          create(:merge_request, source_project: repo_project, title: 'awesome mr')
+
+          get api("/groups/#{group.id}/-/search", user), scope: 'merge_requests', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/merge_requests'
+      end
+
+      context 'for milestones scope' do
+        before do
+          create(:milestone, project: project, title: 'awesome milestone')
+
+          get api("/groups/#{group.id}/-/search", user), scope: 'milestones', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/milestones'
+      end
+    end
+  end
+
+  describe "GET /projects/:id/search" do
+    context 'when user is not authenticated' do
+      it 'returns 401 error' do
+        get api("/projects/#{project.id}/-/search"), scope: 'issues', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(401)
+      end
+    end
+
+    context 'when scope is not supported' do
+      it 'returns 400 error' do
+        get api("/projects/#{project.id}/-/search", user), scope: 'unsupported', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'when scope is missing' do
+      it 'returns 400 error' do
+        get api("/projects/#{project.id}/-/search", user), search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(400)
+      end
+    end
+
+    context 'when project does not exist' do
+      it 'returns 404 error' do
+        get api('/projects/9999/-/search', user), scope: 'issues', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'when user does can not see the project' do
+      it 'returns 404 error' do
+        project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+
+        get api("/projects/#{project.id}/-/search", user), scope: 'issues', search: 'awesome'
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+
+    context 'with correct params' do
+      context 'for issues scope' do
+        before do
+          create(:issue, project: project, title: 'awesome issue')
+
+          get api("/projects/#{project.id}/-/search", user), scope: 'issues', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/issues'
+      end
+
+      context 'for merge_requests scope' do
+        before do
+          create(:merge_request, source_project: repo_project, title: 'awesome mr')
+
+          get api("/projects/#{repo_project.id}/-/search", user), scope: 'merge_requests', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/merge_requests'
+      end
+
+      context 'for milestones scope' do
+        before do
+          create(:milestone, project: project, title: 'awesome milestone')
+
+          get api("/projects/#{project.id}/-/search", user), scope: 'milestones', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/milestones'
+      end
+
+      context 'for notes scope' do
+        before do
+          create(:note_on_merge_request, project: project, note: 'awesome note')
+
+          get api("/projects/#{project.id}/-/search", user), scope: 'notes', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/notes'
+      end
+
+      context 'for wiki_blobs scope' do
+        before do
+          wiki = create(:project_wiki, project: project)
+          create(:wiki_page, wiki: wiki, attrs: { title: 'home', content: "Awesome page" })
+
+          get api("/projects/#{project.id}/-/search", user), scope: 'wiki_blobs', search: 'awesome'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/blobs'
+      end
+
+      context 'for commits scope' do
+        before do
+          get api("/projects/#{repo_project.id}/-/search", user), scope: 'commits', search: '498214de67004b1da3d820901307bed2a68a8ef6'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/commits'
+      end
+
+      context 'for blobs scope' do
+        before do
+          get api("/projects/#{repo_project.id}/-/search", user), scope: 'blobs', search: 'monitors'
+        end
+
+        it_behaves_like 'response is correct', schema: 'public_api/v4/blobs', size: 2
+      end
+    end
+  end
+end