Reset blob/commit index if last_commit is no longer in repository

During code push, check to see if last_commit is still in repository.
If it is not, we can no longer diff against it,
therefore a full reindex on blob/commit is required.

ee/lib/elasticsearch/git/repository.rb

@@ -318,6 +318,36 @@ module Elasticsearch
           res
         end
 
+        def delete_index_for_commits_and_blobs
+          client_for_indexing.delete_by_query(
+            index: self.class.index_name,
+            routing: es_parent,
+            body: {
+              query: {
+                bool: {
+                  filter: [
+                    {
+                      terms: {
+                        type: %w{commit blob}
+                      }
+                    },
+                    {
+                      has_parent: {
+                        parent_type: 'project',
+                        query: {
+                          term: {
+                            id: project_id
+                          }
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+            }
+          )
+        end
+
         def search(query, type: :all, page: 1, per: 20, options: {})
           options[:repository_id] = repository_id if options[:repository_id].nil?
           self.class.search(query, type: type, page: page, per: per, options: options)

ee/lib/gitlab/elastic/indexer.rb

@@ -16,7 +16,7 @@ module Gitlab
         Gitlab::Utils.which(EXPERIMENTAL_INDEXER).present?
       end
 
-      attr_reader :project
+      attr_reader :project, :index_status
 
       def initialize(project)
         @project = project
@@ -75,6 +75,10 @@ module Gitlab
       end
 
       def run_indexer!(to_sha)
+        if index_status && !repository_contains_last_indexed_commit?
+          project.repository.delete_index_for_commits_and_blobs
+        end
+
         command = [path_to_indexer, project.id.to_s, repository_path]
 
         vars = @vars.merge('FROM_SHA' => from_sha, 'TO_SHA' => to_sha)
@@ -85,7 +89,7 @@ module Gitlab
       end
 
       def last_commit
-        @index_status&.last_commit
+        index_status&.last_commit
       end
 
       def from_sha

ee/spec/lib/gitlab/elastic/indexer_spec.rb

@@ -79,20 +79,6 @@ describe Gitlab::Elastic::Indexer do
           expect_index_status(to_sha)
         end
       end
-
-      context 'when last_commit no longer exists' do
-        before do
-          project.create_index_status!(last_commit: '51749675fc22nononononononono343bd54a3c95')
-        end
-
-        it 'use nil as from_sha' do
-          expect_popen.and_return(popen_success)
-
-          indexer.run(to_sha)
-
-          expect_index_status(to_sha)
-        end
-      end
     end
 
     it 'updates the index status when the indexing is a success' do
@@ -174,8 +160,12 @@ describe Gitlab::Elastic::Indexer do
     let!(:initial_commit) { project.repository.commit('master').sha }
     let(:ee_application_setting) { nil }
 
+    before do
+      stub_ee_application_setting(elasticsearch_indexing: true)
+    end
+
     def change_repository_and_index(project, &blk)
-      yield blk
+      yield blk if blk
 
       current_commit = project.repository.commit('master').sha
 
@@ -194,25 +184,29 @@ describe Gitlab::Elastic::Indexer do
       end
     end
 
-    context 'when IndexStatus#last_commit is no longer in repository', :pending do
+    context 'when IndexStatus#last_commit is no longer in repository' do
       before do
+        ElasticIndexerWorker.new.perform("index", "Project", project.id, project.es_id)
+      end
+
+      it 'reindexes from scratch if IndexStatus#last_commit is no longer in repository' do
+        sha_for_reset = nil
+
         change_repository_and_index(project) do
-          project.repository.create_file(user, '12', '', message: '12', branch_name: 'master')
+          sha_for_reset = project.repository.create_file(user, '12', '', message: '12', branch_name: 'master')
           project.repository.create_file(user, '23', '', message: '23', branch_name: 'master')
         end
 
         expect(indexed_file_paths_for('12')).to include('12')
         expect(indexed_file_paths_for('23')).to include('23')
 
-        project.index_status.update(last_commit: 'ABCDABCDABCD')
-      end
+        project.index_status.update(last_commit: '____________')
 
-      it 'reindexes from scratch if IndexStatus#last_commit is no longer in repository' do
         change_repository_and_index(project) do
-          project.repository.write_ref('master', initial_commit)
+          project.repository.write_ref('master', sha_for_reset)
         end
 
-        expect(indexed_file_paths_for('12')).not_to include('12')
+        expect(indexed_file_paths_for('12')).to include('12')
         expect(indexed_file_paths_for('23')).not_to include('23')
       end
     end
@@ -221,11 +215,9 @@ describe Gitlab::Elastic::Indexer do
       before do
         change_repository_and_index(project) do
           project.repository.create_file(user, '12', '', message: '12', branch_name: 'master')
-          project.repository.create_file(user, '23', '', message: '23', branch_name: 'master')
         end
 
         expect(indexed_file_paths_for('12')).to include('12')
-        expect(indexed_file_paths_for('23')).to include('23')
       end
 
       it 'reverses already indexed commits' do
@@ -234,7 +226,6 @@ describe Gitlab::Elastic::Indexer do
         end
 
         expect(indexed_file_paths_for('12')).not_to include('12')
-        expect(indexed_file_paths_for('23')).not_to include('23')
       end
     end
   end