Implement progressive elasticsearch indexing for project mirrors

app/models/ee/project/import_status_state_machine.rb

@@ -47,7 +47,10 @@ module EE
             end
 
             if current_application_settings.elasticsearch_indexing?
-              project.run_after_commit { ElasticCommitIndexerWorker.perform_async(project.id) }
+              project.run_after_commit do
+                last_indexed_commit = project.index_status&.last_commit
+                ElasticCommitIndexerWorker.perform_async(project.id, last_indexed_commit)
+              end
             end
           end
 

changelogs/unreleased-ee/2881-es-progessive-indexing-of-mirrors.yml

+---
+title: Implement progressive elasticsearch indexing for project mirrors
+merge_request: 2393
+author:

spec/models/project_spec.rb

@@ -1824,6 +1824,46 @@ describe Project, models: true do
 
         expect(housekeeping_service).not_to have_received(:execute)
       end
+
+      context 'elasticsearch indexing disabled' do
+        before do
+          stub_application_setting(elasticsearch_indexing: false)
+        end
+
+        it 'does not index the repository' do
+          project = create(:empty_project, :import_started, import_type: :github)
+
+          expect(ElasticCommitIndexerWorker).not_to receive(:perform_async)
+
+          project.import_finish
+        end
+      end
+
+      context 'elasticsearch indexing enabled' do
+        let(:project) { create(:project, :import_started, import_type: :github) }
+
+        before do
+          stub_application_setting(elasticsearch_indexing: true)
+        end
+
+        context 'no index status' do
+          it 'schedules a full index of the repository' do
+            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id, nil)
+
+            project.import_finish
+          end
+        end
+
+        context 'with index status' do
+          let!(:index_status) { project.create_index_status!(indexed_at: Time.now, last_commit: 'foo') }
+
+          it 'schedules a progressive index of the repository' do
+            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id, index_status.last_commit)
+
+            project.import_finish
+          end
+        end
+      end
     end
   end