Use elasticsearch go indexer for wikis

Currently non-NFS (Gitaly) indexing of wiki blobs is totally broken, so
let's fix it!

ee/app/models/concerns/elastic/wiki_repositories_search.rb

@@ -23,10 +23,18 @@ module Elastic
         self.__elasticsearch__.client
       end
 
+      def index_wiki_blobs
+        if ::Gitlab::CurrentSettings.elasticsearch_experimental_indexer?
+          ElasticCommitIndexerWorker.perform_async(project.id, nil, nil, true)
+        else
+          project.wiki.index_blobs
+        end
+      end
+
       def self.import
         Project.with_wiki_enabled.find_each do |project|
           if project.use_elasticsearch? && !project.wiki.empty?
-            project.wiki.index_blobs
+            project.wiki.index_wiki_blobs
           end
         end
       end

ee/app/models/ee/project_wiki.rb

@@ -16,7 +16,7 @@ module EE
     end
 
     def update_elastic_index
-      index_blobs if project.use_elasticsearch?
+      index_wiki_blobs if project.use_elasticsearch?
     end
 
     def path_to_repo

ee/app/workers/ee/post_receive.rb

@@ -31,7 +31,7 @@ module EE
     def update_wiki_es_indexes(post_received)
       return unless post_received.project.use_elasticsearch?
 
-      post_received.project.wiki.index_blobs
+      post_received.project.wiki.index_wiki_blobs
     end
   end
 end

ee/app/workers/elastic_commit_indexer_worker.rb

@@ -5,13 +5,13 @@ class ElasticCommitIndexerWorker
 
   sidekiq_options retry: 2
 
-  def perform(project_id, oldrev = nil, newrev = nil)
+  def perform(project_id, oldrev = nil, newrev = nil, wiki = false)
     return true unless Gitlab::CurrentSettings.elasticsearch_indexing?
 
     project = Project.find(project_id)
 
     return true unless project.use_elasticsearch?
 
-    Gitlab::Elastic::Indexer.new(project).run(newrev)
+    Gitlab::Elastic::Indexer.new(project, wiki: wiki).run(newrev)
   end
 end

ee/changelogs/unreleased/11269-index-wikis-using-new-go-indexer.yml

+---
+title: Use elasticsearch go indexer for wikis
+merge_request: 13743
+author:
+type: fixed

ee/lib/gitlab/elastic/indexer.rb

@@ -24,8 +24,9 @@ module Gitlab
 
       attr_reader :project, :index_status
 
-      def initialize(project)
+      def initialize(project, wiki: false)
         @project = project
+        @wiki = wiki
 
         # We accept any form of settings, including string and array
         # This is why JSON is needed
@@ -41,7 +42,7 @@ module Gitlab
         end
 
         # Use the eager-loaded association if available.
-        @index_status = project.index_status
+        @index_status = project.index_status unless wiki?
       end
 
       def run(to_sha = nil)
@@ -50,20 +51,24 @@ module Gitlab
         head_commit = repository.try(:commit)
 
         if repository.nil? || !repository.exists? || repository.empty? || head_commit.nil?
-          update_index_status(Gitlab::Git::BLANK_SHA)
+          update_index_status(Gitlab::Git::BLANK_SHA) unless wiki?
           return
         end
 
         run_indexer!(to_sha)
-        update_index_status(to_sha)
+        update_index_status(to_sha) unless wiki?
 
         true
       end
 
       private
 
+      def wiki?
+        @wiki
+      end
+
       def repository
-        project.repository
+        wiki? ? project.wiki.repository : project.repository
       end
 
       def path_to_indexer
@@ -76,16 +81,26 @@ module Gitlab
 
       def use_experimental_indexer?
         strong_memoize(:use_experimental_indexer) do
-          Gitlab::CurrentSettings.elasticsearch_experimental_indexer? && self.class.experimental_indexer_present?
+          if wiki?
+            raise '`gitlab-elasticsearch-indexer` is required for indexing wikis' unless self.class.experimental_indexer_present?
+
+            true
+          else
+            Gitlab::CurrentSettings.elasticsearch_experimental_indexer? && self.class.experimental_indexer_present?
+          end
         end
       end
 
       def run_indexer!(to_sha)
         if index_status && !repository_contains_last_indexed_commit?
-          project.repository.delete_index_for_commits_and_blobs
+          repository.delete_index_for_commits_and_blobs
         end
 
-        command = [path_to_indexer, project.id.to_s, repository_path]
+        command = if wiki?
+                    [path_to_indexer, "--blob-type=wiki_blob", "--skip-commits", project.id.to_s, repository_path]
+                  else
+                    [path_to_indexer, project.id.to_s, repository_path]
+                  end
 
         vars = @vars.merge('FROM_SHA' => from_sha, 'TO_SHA' => to_sha)
 

ee/lib/tasks/gitlab/elastic.rake

@@ -55,8 +55,8 @@ namespace :gitlab do
           puts "Indexing wiki of #{project.full_name}..."
 
           begin
-            project.wiki.index_blobs
-            puts "Done!".color(:green)
+            project.wiki.index_wiki_blobs
+            puts "Enqueued!".color(:green)
           rescue StandardError => e
             puts "#{e.message}, trace - #{e.backtrace}"
           end

ee/spec/elastic_integration/global_search_spec.rb

@@ -156,7 +156,7 @@ describe 'GlobalSearch' do
 
       project.repository.index_blobs
       project.repository.index_commits
-      project.wiki.index_blobs
+      project.wiki.index_wiki_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end

ee/spec/features/search/elastic/global_search_spec.rb

@@ -145,7 +145,7 @@ describe 'Global elastic search', :elastic do
   describe 'I search through the wiki blobs' do
     before do
       project.wiki.create_page('test.md', '# term')
-      project.wiki.index_blobs
+      project.wiki.index_wiki_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end

ee/spec/features/search/elastic/group_search_spec.rb

@@ -73,7 +73,7 @@ describe 'Group elastic search', :js do
 
     before do
       wiki.create_page('test.md', '# term')
-      wiki.index_blobs
+      wiki.index_wiki_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end

ee/spec/lib/gitlab/elastic/indexer_spec.rb

@@ -30,6 +30,52 @@ describe Gitlab::Elastic::Indexer do
     end
   end
 
+  context 'wikis' do
+    let!(:project) { create(:project, :wiki_repo) }
+    let(:indexer) { described_class.new(project, wiki: true) }
+
+    before do
+      project.wiki.create_page('test.md', '# term')
+    end
+
+    it 'does not ask for IndexStatus' do
+      expect(project).not_to receive(:index_status)
+      expect(project.wiki).not_to receive(:index_status)
+      expect_popen.and_return(popen_success)
+
+      indexer.run
+    end
+
+    it 'raises if it cannot find gitlab-elasticsearch-indexer' do
+      expect(described_class).to receive(:experimental_indexer_present?).and_return(false)
+
+      expect { indexer.run }.to raise_error('`gitlab-elasticsearch-indexer` is required for indexing wikis')
+    end
+
+    it 'runs the indexer with the right flags' do
+      expect(described_class).to receive(:experimental_indexer_present?).and_return(true)
+
+      expect_popen.with(
+        [
+          'gitlab-elasticsearch-indexer',
+          '--blob-type=wiki_blob',
+          '--skip-commits',
+          project.id.to_s,
+          "#{project.wiki.repository.disk_path}.git"
+        ],
+        nil,
+        hash_including(
+          'ELASTIC_CONNECTION_INFO' => Gitlab::CurrentSettings.elasticsearch_config.to_json,
+          'RAILS_ENV'               => Rails.env,
+          'FROM_SHA'                => expected_from_sha,
+          'TO_SHA'                  => nil
+        )
+      ).and_return(popen_success)
+
+      indexer.run
+    end
+  end
+
   context 'repository has unborn head' do
     it 'updates the index status without running the indexing command' do
       allow(project.repository).to receive(:exists?).and_return(false)

ee/spec/lib/gitlab/elastic/project_search_results_spec.rb

@@ -47,9 +47,9 @@ describe Gitlab::Elastic::ProjectSearchResults do
 
       # Wiki
       project.wiki.create_page('index_page', 'term')
-      project.wiki.index_blobs
+      project.wiki.index_wiki_blobs
       project1.wiki.create_page('index_page', ' term')
-      project1.wiki.index_blobs
+      project1.wiki.index_wiki_blobs
 
       Gitlab::Elastic::Helper.refresh_index
 
@@ -70,7 +70,7 @@ describe Gitlab::Elastic::ProjectSearchResults do
 
         # Wiki
         project.wiki.create_page('index_page', 'term')
-        project.wiki.index_blobs
+        project.wiki.index_wiki_blobs
 
         Gitlab::Elastic::Helper.refresh_index
 

ee/spec/lib/gitlab/elastic/search_results_spec.rb

@@ -555,7 +555,7 @@ describe Gitlab::Elastic::SearchResults, :elastic do
     before do
       if project_1.wiki_enabled?
         project_1.wiki.create_page('index_page', 'term')
-        project_1.wiki.index_blobs
+        project_1.wiki.index_wiki_blobs
       end
 
       Gitlab::Elastic::Helper.refresh_index
@@ -579,7 +579,7 @@ describe Gitlab::Elastic::SearchResults, :elastic do
     it 'finds wiki blobs from public projects only' do
       project_2 = create :project, :repository, :private, :wiki_repo
       project_2.wiki.create_page('index_page', 'term')
-      project_2.wiki.index_blobs
+      project_2.wiki.index_wiki_blobs
       Gitlab::Elastic::Helper.refresh_index
 
       expect(results.wiki_blobs_count).to eq 1
@@ -870,7 +870,7 @@ describe Gitlab::Elastic::SearchResults, :elastic do
       before do
         [public_project, internal_project, private_project1, private_project2].each do |project|
           project.wiki.create_page('index_page', 'term')
-          project.wiki.index_blobs
+          project.wiki.index_wiki_blobs
         end
 
         Gitlab::Elastic::Helper.refresh_index

ee/spec/models/concerns/elastic/project_wiki_spec.rb

 require 'spec_helper'
 
 describe ProjectWiki, :elastic do
+  set(:project) { create(:project, :wiki_repo) }
+
   before do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
-  end
-
-  it "searches wiki page" do
-    project = create :project, :wiki_repo
 
     Sidekiq::Testing.inline! do
       project.wiki.create_page("index_page", "Bla bla term1")
       project.wiki.create_page("omega_page", "Bla bla term2")
-      project.wiki.index_blobs
+      project.wiki.index_wiki_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end
+  end
 
+  it "searches wiki page" do
     expect(project.wiki.search('term1', type: :wiki_blob)[:wiki_blobs][:total_count]).to eq(1)
     expect(project.wiki.search('term1 | term2', type: :wiki_blob)[:wiki_blobs][:total_count]).to eq(2)
   end
+
+  context 'with old indexer' do
+    before do
+      stub_ee_application_setting(elasticsearch_experimental_indexer: false)
+    end
+
+    it 'searches wiki page' do
+      expect(project.wiki.search('term1', type: :wiki_blob)[:wiki_blobs][:total_count]).to eq(1)
+      expect(project.wiki.search('term1 | term2', type: :wiki_blob)[:wiki_blobs][:total_count]).to eq(2)
+    end
+  end
+
+  it 'uses the experimental indexer if enabled' do
+    stub_ee_application_setting(elasticsearch_experimental_indexer: true)
+
+    expect(project.wiki).not_to receive(:index_blobs)
+    expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id, nil, nil, true)
+
+    project.wiki.index_wiki_blobs
+  end
+
+  it 'indexes inside Rails if experiemntal indexer is not enabled' do
+    stub_ee_application_setting(elasticsearch_experimental_indexer: false)
+
+    expect(project.wiki).to receive(:index_blobs)
+    expect(ElasticCommitIndexerWorker).not_to receive(:perform_async)
+
+    project.wiki.index_wiki_blobs
+  end
 end

ee/spec/requests/api/search_spec.rb

@@ -49,7 +49,7 @@ describe API::Search do
         wiki = create(:project_wiki, project: project)
         create(:wiki_page, wiki: wiki, attrs: { title: 'home', content: "Awesome page" })
 
-        project.wiki.index_blobs
+        project.wiki.index_wiki_blobs
         Gitlab::Elastic::Helper.refresh_index
 
         get api(endpoint, user), params: { scope: 'wiki_blobs', search: 'awesome' }

ee/spec/workers/elastic_commit_indexer_worker_spec.rb

@@ -23,5 +23,14 @@ describe ElasticCommitIndexerWorker do
 
       expect(subject.perform(1)).to be_truthy
     end
+
+    it 'runs indexer in wiki mode if asked to' do
+      indexer = double
+
+      expect(indexer).to receive(:run)
+      expect(Gitlab::Elastic::Indexer).to receive(:new).with(project, wiki: true).and_return(indexer)
+
+      subject.perform(project.id, nil, nil, true)
+    end
   end
 end

ee/spec/workers/post_receive_spec.rb

@@ -72,7 +72,7 @@ describe PostReceive do
     it 'triggers wiki index update when ElasticSearch is enabled', :elastic do
       stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
 
-      expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
+      expect_any_instance_of(ProjectWiki).to receive(:index_wiki_blobs)
 
       described_class.new.perform(gl_repository, key_id, base64_changes)
     end
@@ -88,7 +88,7 @@ describe PostReceive do
 
       context 'when the project is not enabled specifically' do
         it 'does not trigger wiki index update' do
-          expect_any_instance_of(ProjectWiki).not_to receive(:index_blobs)
+          expect_any_instance_of(ProjectWiki).not_to receive(:index_wiki_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end
@@ -100,7 +100,7 @@ describe PostReceive do
         end
 
         it 'triggers wiki index update' do
-          expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
+          expect_any_instance_of(ProjectWiki).to receive(:index_wiki_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end
@@ -116,7 +116,7 @@ describe PostReceive do
         end
 
         it 'triggers wiki index update' do
-          expect_any_instance_of(ProjectWiki).to receive(:index_blobs)
+          expect_any_instance_of(ProjectWiki).to receive(:index_wiki_blobs)
 
           described_class.new.perform(gl_repository, key_id, base64_changes)
         end