Merge branch '1565-global-wiki-search' into 'master'

Add global wiki search using Elasticsearch

Closes #1565

See merge request !1717

app/services/search/global_service.rb

@@ -38,7 +38,7 @@ module Search
     def scope
       @scope ||= begin
         allowed_scopes = %w[issues merge_requests milestones]
-        allowed_scopes += %w[blobs commits] if current_application_settings.elasticsearch_search?
+        allowed_scopes += %w[wiki_blobs blobs commits] if current_application_settings.elasticsearch_search?
 
         allowed_scopes.delete(params[:scope]) { 'projects' }
       end

app/views/search/_category.html.haml

@@ -83,3 +83,8 @@
             Commits
             %span.badge
               = @search_results.commits_count
+        %li{ class: active_when(@scope == 'wiki_blobs') }
+          = link_to search_filter_path(scope: 'wiki_blobs') do
+            Wiki
+            %span.badge
+              = @search_results.wiki_blobs_count

app/views/search/results/_wiki_blob.html.haml

+- project = @project || find_project_for_blob(wiki_blob)
 - wiki_blob = parse_search_result(wiki_blob)
+
 .blob-result
   .file-holder
     .js-file-title.file-title
-      = link_to namespace_project_wiki_path(@project.namespace, @project, wiki_blob.basename) do
+      = link_to namespace_project_wiki_path(project.namespace, project, wiki_blob.basename) do
         %i.fa.fa-file
         %strong
-          = wiki_blob.basename
+          - if @project
+            = wiki_blob.basename
+          - else
+            #{project.name_with_namespace}:
+            %i= wiki_blob.basename
     .file-content.code.term
       = render 'shared/file_highlight', blob: wiki_blob, first_line_number: wiki_blob.startline

changelogs/unreleased-ee/1565-global-wiki-search.yml

+---
+title: Add global wiki search using Elasticsearch
+merge_request:
+author:

lib/gitlab/elastic/search_results.rb

@@ -26,6 +26,8 @@ module Gitlab
           milestones.page(page).per(per_page).records
         when 'blobs'
           blobs.page(page).per(per_page)
+        when 'wiki_blobs'
+          wiki_blobs.page(page).per(per_page)
         when 'commits'
           commits(page: page, per_page: per_page)
         else
@@ -41,6 +43,10 @@ module Gitlab
         @blobs_count ||= blobs.total_count
       end
 
+      def wiki_blobs_count
+        @wiki_blobs_count ||= wiki_blobs.total_count
+      end
+
       def commits_count
         @commits_count ||= commits.total_count
       end
@@ -147,6 +153,22 @@ module Gitlab
         end
       end
 
+      def wiki_blobs
+        if query.blank?
+          Kaminari.painate_array([])
+        else
+          opt = {
+            additional_filter: wiki_filter
+          }
+
+          ProjectWiki.search(
+            query,
+            type: :blob,
+            options: opt.merge({ highlight: true })
+          )[:blobs][:results].response
+        end
+      end
+
       def commits(page: 1, per_page: 20)
         if query.blank?
           Kaminari.paginate_array([])
@@ -164,7 +186,15 @@ module Gitlab
         end
       end
 
+      def wiki_filter
+        blob_filter(:wiki_access_level)
+      end
+
       def repository_filter
+        blob_filter(:repository_access_level)
+      end
+
+      def blob_filter(project_feature_name)
         conditions =
           if non_guest_project_ids == :any
             [{ exists: { field: "id" } }]
@@ -177,7 +207,7 @@ module Gitlab
                           bool: {
                             filter: [
                               { term: { visibility_level: Project::PUBLIC } },
-                              { term: { repository_access_level: ProjectFeature::ENABLED } }
+                              { term: { project_feature_name => ProjectFeature::ENABLED } }
                             ]
                           }
                         }
@@ -187,7 +217,7 @@ module Gitlab
                             bool: {
                               filter: [
                                 { term: { visibility_level: Project::INTERNAL } },
-                                { term: { repository_access_level: ProjectFeature::ENABLED } }
+                                { term: { project_feature_name => ProjectFeature::ENABLED } }
                               ]
                             }
                           }
@@ -200,7 +230,7 @@ module Gitlab
             query: {
               bool: {
                 should: conditions,
-                must_not: { term: { repository_access_level: ProjectFeature::DISABLED } }
+                must_not: { term: { project_feature_name => ProjectFeature::DISABLED } }
               }
             }
           }

spec/features/es_global_search_spec.rb

@@ -56,6 +56,28 @@ feature 'Global elastic search', feature: true do
     end
   end
 
+  describe 'I search through the wiki blobs' do
+    before do
+      project.wiki.create_page('test.md', '# term')
+      project.wiki.index_blobs
+
+      Gitlab::Elastic::Helper.refresh_index
+    end
+
+    it "finds files" do
+      visit dashboard_projects_path
+
+      fill_in "search", with: "term"
+      click_button "Go"
+
+      select_filter("Wiki")
+
+      expect(page).to have_selector('.file-content .code')
+
+      expect(page).to have_selector("span.line[lang='markdown']")
+    end
+  end
+
   describe 'I search through the commits' do
     before do
       project.repository.index_commits

spec/features/es_group_search_spec.rb

@@ -68,6 +68,31 @@ feature 'Group elastic search', js: true, feature: true do
     end
   end
 
+  describe 'wiki search' do
+    let(:wiki) { ProjectWiki.new(project, user) }
+
+    before do
+      wiki.create_page('test.md', '# term')
+      wiki.index_blobs
+
+      Gitlab::Elastic::Helper.refresh_index
+    end
+
+    it "finds pages" do
+      visit search_path
+      
+      choose_group group
+      fill_in "search", with: "term"
+      click_button "Search"
+
+      select_filter("Wiki")
+
+      expect(page).to have_selector('.file-content .code')
+
+      expect(page).to have_selector("span.line[lang='markdown']")
+    end
+  end
+
   describe 'commit search' do
     before do
       project.repository.index_commits

spec/lib/gitlab/elastic/search_results_spec.rb

@@ -401,6 +401,75 @@ describe Gitlab::Elastic::SearchResults, lib: true do
     end
   end
 
+  describe 'Wikis' do
+    let(:results) { described_class.new(user, 'term', limit_project_ids) }
+    subject(:wiki_blobs) { results.objects('wiki_blobs') }
+
+    before do
+      project_1.wiki.create_page('index_page', 'term')
+      project_1.wiki.index_blobs
+
+      Gitlab::Elastic::Helper.refresh_index
+    end
+
+    it 'finds wiki blobs' do
+      blobs = results.objects('wiki_blobs')
+
+      expect(blobs.first["_source"]["blob"]["content"]).to include("term")
+      expect(results.wiki_blobs_count).to eq 1
+    end
+
+    it 'finds wiki blobs from public projects only' do
+      project_2 = create :project, :private
+      project_2.wiki.create_page('index_page', 'term')
+      project_2.wiki.index_blobs
+      Gitlab::Elastic::Helper.refresh_index
+
+      expect(results.wiki_blobs_count).to eq 1
+
+      results = described_class.new(user, 'term', [project_1.id, project_2.id])
+      expect(results.wiki_blobs_count).to eq 2
+    end
+
+    it 'returns zero when wiki blobs are not found' do
+      results = described_class.new(user, 'asdfg', limit_project_ids)
+
+      expect(results.wiki_blobs_count).to eq 0
+    end
+
+    context 'when wiki is disabled' do
+      let(:project_1) { create(:project, :public, :wiki_disabled) }
+
+      context 'search by member' do
+        let(:limit_project_ids) { [project_1.id] }
+
+        it { is_expected.to be_empty }
+      end
+
+      context 'search by non-member' do
+        let(:limit_project_ids) { [] }
+
+        it { is_expected.to be_empty }
+      end
+    end
+
+    context 'when wiki is internal' do
+      let(:project_1) { create(:project, :public, :wiki_private) }
+
+      context 'search by member' do
+        let(:limit_project_ids) { [project_1.id] }
+
+        it { is_expected.not_to be_empty }
+      end
+
+      context 'search by non-member' do
+        let(:limit_project_ids) { [] }
+
+        it { is_expected.to be_empty }
+      end
+    end
+  end
+
   describe 'Commits' do
     before do
       project_1.repository.index_commits
@@ -554,6 +623,33 @@ describe Gitlab::Elastic::SearchResults, lib: true do
       end
     end
 
+    context 'Wikis' do
+      before do
+        [public_project, internal_project, private_project1, private_project2].each do |project|
+          project.wiki.create_page('index_page', 'term')
+          project.wiki.index_blobs
+        end
+
+        Gitlab::Elastic::Helper.refresh_index
+      end
+
+      it 'finds the right set of wiki blobs' do
+        # Authenticated search
+        results = described_class.new(user, 'term', limit_project_ids)
+        blobs = results.objects('wiki_blobs')
+
+        expect(blobs.map{|blob| blob._parent.to_i }).to match_array [internal_project.id, private_project2.id, public_project.id]
+        expect(results.wiki_blobs_count).to eq 3
+
+        # Unauthenticated search
+        results = described_class.new(nil, 'term', [])
+        blobs = results.objects('wiki_blobs')
+
+        expect(blobs.first._parent.to_i).to eq public_project.id
+        expect(results.wiki_blobs_count).to eq 1
+      end
+    end
+
     context 'Commits' do
       it 'finds right set of commits' do
         [internal_project, private_project1, private_project2, public_project].each do |project|