ES: respo indexer increments index

app/models/index_status.rb

 class IndexStatus < ActiveRecord::Base
   belongs_to :project
+
+  validates :project_id, uniqueness: true
 end

lib/tasks/gitlab/elastic.rake

@@ -4,30 +4,27 @@ namespace :gitlab do
     task index_repositories: :environment  do
       Repository.__elasticsearch__.create_index!
 
-      projects = Project
-
-      if ENV['ID_FROM']
-        projects = projects.where("id >= ?", ENV['ID_FROM'])
-      end
-
-      if ENV['ID_TO']
-        projects = projects.where("id <= ?", ENV['ID_TO'])
-      end
+      projects = apply_project_filters(Project)
 
       projects.find_each do |project|
         if project.repository.exists? && !project.repository.empty?
-          puts "Indexing #{project.name_with_namespace}..."
+          puts "Indexing #{project.name_with_namespace} (ID=#{project.id})..."
 
-          begin
-            # During indexing the new commits can be pushed,
-            # this parameter only indicates that at least this commit is in index
-            heeads_sha = project.repository.commit.sha
-            IndexStatus.find_or_create_by(last_commit: heeads_sha, project: project)
+          index_status = IndexStatus.find_or_create_by(project: project)
+          heads_sha = project.repository.commit.sha
 
-            project.repository.index_commits
-            project.repository.index_blobs
+          if index_status.last_commit == heads_sha
+            puts "Skipped".yellow
+            next
+          end
+
+          begin
+            project.repository.index_commits(from_rev: project.index_status.last_commit)
+            project.repository.index_blobs(from_rev: project.index_status.last_commit)
 
-            project.index_status.update(indexed_at: DateTime.now)
+            # During indexing the new commits can be pushed,
+            # the last_commit parameter only indicates that at least this commit is in index
+            index_status.update(last_commit: heads_sha, indexed_at: DateTime.now)
             puts "Done!".green
           rescue StandardError => e
             puts "#{e.message}, trace - #{e.backtrace}"
@@ -40,7 +37,9 @@ namespace :gitlab do
     task index_wikis: :environment  do
       ProjectWiki.__elasticsearch__.create_index!
 
-      Project.where(wiki_enabled: true).find_each do |project|
+      projects = apply_project_filters(Project.where(wiki_enabled: true))
+
+      projects.find_each do |project|
         unless project.wiki.empty?
           puts "Indexing wiki of #{project.name_with_namespace}..."
           begin
@@ -81,5 +80,17 @@ namespace :gitlab do
         klass.__elasticsearch__.create_index!
       end
     end
+
+    def apply_project_filters(projects)
+      if ENV['ID_FROM']
+        projects = projects.where("id >= ?", ENV['ID_FROM'])
+      end
+
+      if ENV['ID_TO']
+        projects = projects.where("id <= ?", ENV['ID_TO'])
+      end
+
+      projects
+    end
   end
 end
\ No newline at end of file