Keep the project (repository) index status up to date

Prior to this commit, the `index_statuses` table was allowed to go stale after
initial indexing. This made it impossible to provide status details, and also
made treating the initial index as a 'backfill' job harder.

Now, if a repository is indexed via `git push`, the initial indexing will
recognise that it has been indexed, and not re-index it.

We introduce an asynchronous initial indexer and an index status rake task at
the same time.

app/workers/elastic_batch_project_indexer_worker.rb

@@ -2,50 +2,37 @@ class ElasticBatchProjectIndexerWorker
   include Sidekiq::Worker
   include Gitlab::CurrentSettings
 
-  sidekiq_options queue: :elasticsearch, retry: 2
+  # Batch indexing is a generally a onetime option, so give finer control over
+  # queuing and concurrency
+  include DedicatedSidekiqQueue
+
+  # This worker is long-running, but idempotent, so retry many times if
+  # necessary
+  sidekiq_options retry: 10
 
   def perform(start, finish, update_index = false)
     projects = build_relation(start, finish, update_index)
-    indexer = Gitlab::Elastic::Indexer.new
-
-    projects.find_each do |project|
-      repository = project.repository
-      next unless repository.exists? && !repository.empty?
-
-      begin
-        logger.info "Indexing #{project.name_with_namespace} (ID=#{project.id})..."
-
-        index_status = project.index_status || project.build_index_status
-        head_commit = repository.commit
-
-        if !head_commit || index_status.last_commit == head_commit.sha
-          logger.info("Skipped".color(:yellow))
-          next
-        end
-
-        indexer.run(
-          project.id,
-          repository.path_to_repo,
-          index_status.last_commit
-        )
-
-        # During indexing the new commits can be pushed,
-        # the last_commit parameter only indicates that at least this commit is in index
-        index_status.last_commit = head_commit.sha
-        index_status.indexed_at = DateTime.now
-        index_status.save
-
-        logger.info("Done!".color(:green))
-      rescue => err
-        logger.warn("#{err.message}, trace - #{err.backtrace}")
-      end
-    end
+
+    projects.find_each { |project| run_indexer(project) }
+  end
+
+  private
+
+  def run_indexer(project)
+    logger.info "Indexing #{project.name_with_namespace} (ID=#{project.id})..."
+
+    last_commit = project.index_status.try(:last_commit)
+    Gitlab::Elastic::Indexer.new(project).run(last_commit)
+
+    logger.info "Indexing #{project.name_with_namespace} (ID=#{project.id}) is done!"
+  rescue => err
+    logger.warn("#{err.message} indexing #{project.name_with_namespace} (ID=#{project.id}), trace - #{err.backtrace}")
   end
 
   def build_relation(start, finish, update_index)
     relation = Project.includes(:index_status)
 
-    if update_index
+    unless update_index
       relation = relation.where('index_statuses.id IS NULL').references(:index_statuses)
     end
 

app/workers/elastic_commit_indexer_worker.rb

@@ -8,16 +8,7 @@ class ElasticCommitIndexerWorker
     return true unless current_application_settings.elasticsearch_indexing?
 
     project = Project.find(project_id)
-    repository = project.repository
 
-    return true unless repository.exists?
-
-    indexer = Gitlab::Elastic::Indexer.new
-    indexer.run(
-      project_id,
-      repository.path_to_repo,
-      oldrev,
-      newrev
-    )
+    Gitlab::Elastic::Indexer.new(project).run(oldrev, newrev)
   end
 end

doc/integration/elasticsearch.md

@@ -22,11 +22,10 @@ searching in:
 - Snippets
 - Wiki
 
-Once the data is added to the database or repository and [Elasticsearch is enabled in the admin area](#enable-elasticsearch) the search index will be updated
-automatically.
-Elasticsearch can be installed on the same machine that GitLab
-is installed or on a separate server.
-
+Once the data is added to the database or repository and [Elasticsearch is
+enabled in the admin area](#enable-elasticsearch) the search index will be
+updated automatically. Elasticsearch can be installed on the same machine as
+GitLab, or on a separate server.
 
 ## Requirements
 
@@ -77,6 +76,37 @@ bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production
 
 Then enable Elasticsearch indexing and run repository indexing tasks:
 
+```
+# Omnibus installations
+sudo gitlab-rake gitlab:elastic:index_repositories_async
+
+# Installations from source
+bundle exec rake gitlab:elastic:index_repositories_async RAILS_ENV=production
+```
+
+This enqueues a number of Sidekiq jobs to index your existing repositories.
+You can view the jobs in the admin panel (they are placed in the `elastic_batch_project_indexer`)
+queue), or you can query indexing status using a rake task:
+
+```
+# Omnibus installations
+sudo gitlab-rake gitlab:elastic:index_repositories_status
+
+# Installations from source
+bundle exec rake gitlab:elastic:index_repositories_status RAILS_ENV=production
+
+Indexing is 65.55% complete (6555/10000 projects)
+```
+
+By default, one job is created for every 300 projects. For large numbers of
+projects, you may wish to increase the batch size, by setting the `BATCH`
+environment variable. You may also wish to consider [throttling](../administration/operations/sidekiq_job_throttling.md)
+the `elastic_batch_project_indexer` queue , as this step can be I/O-intensive.
+
+You can also run the initial indexing synchronously - this is most useful if
+you have a small number of projects, or need finer-grained control over indexing
+than Sidekiq permits:
+
 ```
 # Omnibus installations
 sudo gitlab-rake gitlab:elastic:index_repositories
@@ -103,12 +133,24 @@ ID_FROM=1001 ID_TO=2000 sudo gitlab-rake gitlab:elastic:index_repositories
 ID_FROM=2001 sudo gitlab-rake gitlab:elastic:index_repositories
 ```
 
-Sometimes your repository index process `gitlab:elastic:index_repositories` get interupted due to various reasons, in this case you can safely run it again and it will skip those repositories that already have been indexed. As the indexer stores the last commit SHA of every indexed repository in the database you can run the indexer with the special parameter `UPDATE_INDEX` and it will check every project repository again to make sure that every commit in that repository is indexed, it can be useful in case if your index is outdated:
+Sometimes your repository index process `gitlab:elastic:index_repositories` or
+`gitlab:elastic:index_repositories_async` can get interrupted. This may happen
+for many reasons, but it's always safe to run the indexing job again - it will
+skip those repositories that have already been indexed.
+
+As the indexer stores the last commit SHA of every indexed repository in the
+database, you can run the indexer with the special parameter `UPDATE_INDEX` and
+it will check every project repository again to make sure that every commit in
+that repository is indexed, it can be useful in case if your index is outdated:
 
 ```
 UPDATE_INDEX=true ID_TO=1000 sudo gitlab-rake gitlab:elastic:index_repositories
 ```
 
+You can also use the `gitlab:elastic:clear_index_status` Rake task to force the
+indexer to "forget" all progresss, so retrying the indexing process from the
+start.
+
 To index all wikis:
 
 ```

lib/gitlab/elastic/indexer.rb

@@ -8,7 +8,11 @@ module Gitlab
 
       Error = Class.new(StandardError)
 
-      def initialize
+      attr_reader :project
+
+      def initialize(project)
+        @project = project
+
         connection_info = {
           host: current_application_settings.elasticsearch_host,
           port: current_application_settings.elasticsearch_port
@@ -22,20 +26,62 @@ module Gitlab
         }
       end
 
-      def run(project_id, repo_path, from_sha = nil, to_sha = nil)
+      def run(from_sha = nil, to_sha = nil)
         to_sha = nil if to_sha == Gitlab::Git::BLANK_SHA
 
-        vars = @vars.merge({ 'FROM_SHA' => from_sha, 'TO_SHA' => to_sha })
+        head_commit = repository.try(:commit)
+
+        if repository.nil? || !repository.exists? || repository.empty? || head_commit.nil?
+          update_index_status(Gitlab::Git::BLANK_SHA)
+          return
+        end
+
+        run_indexer!(from_sha, to_sha)
+        update_index_status(to_sha)
+
+        true
+      end
+
+      private
 
-        path_to_indexer = File.join(Rails.root, 'bin/elastic_repo_indexer')
+      def repository
+        project.repository
+      end
+
+      def path_to_indexer
+        File.join(Rails.root, 'bin/elastic_repo_indexer')
+      end
 
-        command = [path_to_indexer, project_id.to_s, repo_path]
+      def run_indexer!(from_sha, to_sha)
+        command = [path_to_indexer, project.id.to_s, repository.path_to_repo]
+        vars = @vars.merge('FROM_SHA' => from_sha, 'TO_SHA' => to_sha)
 
         output, status = Gitlab::Popen.popen(command, nil, vars)
 
         raise Error, output unless status.zero?
+      end
 
-        true
+      def update_index_status(to_sha)
+        head_commit = repository.try(:commit)
+
+        # Use the eager-loaded association if available. An index_status should
+        # always be created, even if the repository is empty, so we know it's
+        # been looked at.
+        index_status = project.index_status
+        index_status ||=
+          begin
+            IndexStatus.find_or_create_by(project_id: project.id)
+          rescue ActiveRecord::RecordNotUnique
+            retry
+          end
+
+        # Don't update the index status if we never reached HEAD
+        return if head_commit && to_sha && head_commit.sha != to_sha
+
+        sha = head_commit.try(:sha)
+        sha ||= Gitlab::Git::BLANK_SHA
+        index_status.update_attributes(last_commit: sha, indexed_at: Time.now)
+        project.index_status(true)
       end
     end
   end

lib/tasks/gitlab/elastic.rake

@@ -21,13 +21,21 @@ namespace :gitlab do
       puts "OK"
     end
 
+    desc "GitLab | ElasticSearch | Check project repository indexing status"
+    task index_repositories_status: :environment do
+      indexed = IndexStatus.count
+      projects = Project.count
+      percent = (indexed / projects.to_f) * 100.0
+
+      puts "Indexing is %.2f%% complete (%d/%d projects)" % [percent, indexed, projects]
+    end
+
     desc "GitLab | Elasticsearch | Index project repositories"
     task index_repositories: :environment  do
       print "Indexing project repositories..."
 
       Sidekiq::Logging.logger = Logger.new(STDOUT)
       project_id_batches do |start, finish|
-        puts [start, finish].inspect
         ElasticBatchProjectIndexerWorker.new.perform(start, finish, ENV['UPDATE_INDEX'])
       end
     end
@@ -99,7 +107,7 @@ namespace :gitlab do
     def project_id_batches(&blk)
       relation = Project
 
-      if ENV['UPDATE_INDEX']
+      unless ENV['UPDATE_INDEX']
         relation = relation.includes(:index_status).where('index_statuses.id IS NULL').references(:index_statuses)
       end
 

spec/lib/gitlab/elastic/indexer_spec.rb

 require 'spec_helper'
 
-describe "Indexer" do
+describe Gitlab::Elastic::Indexer do
   include StubENV
 
   before do
@@ -8,21 +8,100 @@ describe "Indexer" do
     stub_application_setting(es_host: ['elastic-host1', 'elastic-host2'])
   end
 
-  it "runs commands" do
-    expect(Gitlab::Popen).to receive(:popen).with(
-      [File.join(Rails.root, 'bin/elastic_repo_indexer'), '1', 'full_repo_path'],
-      nil,
-      hash_including(
-        'ELASTIC_CONNECTION_INFO' => {
-                                       host: current_application_settings.elasticsearch_host,
-                                       port: current_application_settings.elasticsearch_port
-                                     }.to_json,
-        'RAILS_ENV'               => Rails.env,
-        'FROM_SHA' => '000000',
-        'TO_SHA' => '1d1f2d'
-      )
-    ).and_return([[''], 0])
-
-    Gitlab::Elastic::Indexer.new.run(1, 'full_repo_path', '000000', '1d1f2d')
+  let(:project)  { create(:project) }
+  let(:from_sha) { Gitlab::Git::BLANK_SHA }
+  let(:to_sha)   { project.commit.try(:sha) }
+  let(:indexer)  { described_class.new(project)  }
+
+  let(:popen_success) { [[''], 0] }
+  let(:popen_failure) { [['error'], 1] }
+
+  let(:elastic_connection_info) do
+    {
+      host: current_application_settings.elasticsearch_host,
+      port: current_application_settings.elasticsearch_port,
+    }
+  end
+
+  context 'empty project' do
+    let(:project) { create(:empty_project) }
+
+    it 'updates the index status without running the indexing command' do
+      expect_popen.never
+
+      indexer.run
+
+      expect_index_status(Gitlab::Git::BLANK_SHA)
+    end
+  end
+
+  context 'repository has unborn head' do
+    it 'updates the index status without running the indexing command' do
+      allow(project.repository).to receive(:exists?).and_return(false)
+      expect_popen.never
+
+      indexer.run
+
+      expect_index_status(Gitlab::Git::BLANK_SHA)
+    end
+  end
+
+  context 'test project' do
+    let(:project) { create(:project) }
+
+    it 'runs the indexing command' do
+      expect_popen.with(
+        [
+          File.join(Rails.root, 'bin/elastic_repo_indexer'),
+          project.id.to_s,
+          project.repository.path_to_repo
+        ],
+        nil,
+        hash_including(
+          'ELASTIC_CONNECTION_INFO' => elastic_connection_info.to_json,
+          'RAILS_ENV'               => Rails.env,
+          'FROM_SHA'                => from_sha,
+          'TO_SHA'                  => to_sha
+        )
+      ).and_return(popen_success)
+
+      indexer.run(from_sha, to_sha)
+    end
+
+    it 'updates the index status when the indexing is a success' do
+      expect_popen.and_return(popen_success)
+
+      indexer.run(from_sha, to_sha)
+
+      expect_index_status(to_sha)
+    end
+
+    it 'leaves the index status untouched when indexing a non-HEAD commit' do
+      expect_popen.and_return(popen_success)
+
+      indexer.run(from_sha, project.repository.commit('HEAD~1'))
+
+      expect(project.index_status).to be_nil
+    end
+
+    it 'leaves the index status untouched when the indexing fails' do
+      expect_popen.and_return(popen_failure)
+
+      expect { indexer.run }.to raise_error(Gitlab::Elastic::Indexer::Error)
+
+      expect(project.index_status).to be_nil
+    end
+  end
+
+  def expect_popen(*with)
+    expect(Gitlab::Popen).to receive(:popen)
+  end
+
+  def expect_index_status(sha)
+    status = project.index_status
+
+    expect(status).not_to be_nil
+    expect(status.indexed_at).not_to be_nil
+    expect(status.last_commit).to eq(sha)
   end
 end

spec/workers/elastic_batch_project_indexer_worker_spec.rb

+require 'spec_helper'
+
+describe ElasticBatchProjectIndexerWorker do
+  subject(:worker) { described_class.new }
+  let(:projects) { create_list(:empty_project, 2) }
+
+  describe '#perform' do
+    it 'runs the indexer for projects in the batch range' do
+      projects.each {|project| expect_index(project) }
+
+      worker.perform(projects.first.id, projects.last.id)
+    end
+
+    it 'skips projects not in the batch range' do
+      expect_index(projects.first).never
+      expect_index(projects.last)
+
+      worker.perform(projects.last.id, projects.last.id)
+    end
+
+    context 'update_index = false' do
+      it 'skips projects that were already indexed' do
+        projects.first.create_index_status!
+
+        expect_index(projects.first).never
+
+        worker.perform(projects.first.id, projects.first.id)
+      end
+    end
+
+    context 'with update_index' do
+      it 'reindexes projects that were already indexed' do
+        projects.first.create_index_status!
+
+        expect_index(projects.first)
+        expect_index(projects.last)
+
+        worker.perform(projects.first.id, projects.last.id, true)
+      end
+
+      it 'starts indexing at the last indexed commit' do
+        projects.first.create_index_status!(last_commit: 'foo')
+
+        expect_index(projects.first).and_call_original
+        expect_any_instance_of(Gitlab::Elastic::Indexer).to receive(:run).with('foo')
+
+        worker.perform(projects.first.id, projects.first.id, true)
+      end
+    end
+  end
+
+  def expect_index(project)
+    expect(worker).to receive(:run_indexer).with(project)
+  end
+end

spec/workers/elastic_commit_indexer_worker_spec.rb

@@ -15,23 +15,6 @@ describe ElasticCommitIndexerWorker do
       subject.perform(project.id, '0000', '0000')
     end
 
-    it 'does not run indexer when project is empty' do
-      empty_project = create :empty_project
-
-      expect_any_instance_of(Gitlab::Elastic::Indexer).not_to receive(:run)
-
-      subject.perform(empty_project.id, '0000', '0000')
-    end
-
-    it 'returns true if repository has unborn head' do
-      project = create :project
-      repository = double('repository')
-      expect(repository).to receive(:exists?).and_return(false)
-      expect_any_instance_of(Project).to receive(:repository).and_return(repository)
-
-      expect(subject.perform(project.id)).to be_truthy
-    end
-
     it 'returns true if ES disabled' do
       stub_application_setting(elasticsearch_indexing: false)