Merge branch 'process-commit-worker-large-batches' into 'master'

Improve ProcessCommitWorker for large push payloads

Closes #25827

See merge request !8267

app/services/git_push_service.rb

@@ -3,6 +3,9 @@ class GitPushService < BaseService
   include Gitlab::CurrentSettings
   include Gitlab::Access
 
+  # The N most recent commits to process in a single push payload.
+  PROCESS_COMMIT_LIMIT = 100
+
   # This method will be called after each git update
   # and only if the provided user and project are present in GitLab.
   #
@@ -77,6 +80,16 @@ class GitPushService < BaseService
     ProjectCacheWorker.perform_async(@project.id, types)
   end
 
+  # Schedules processing of commit messages.
+  def process_commit_messages
+    default = is_default_branch?
+
+    push_commits.last(PROCESS_COMMIT_LIMIT).each do |commit|
+      ProcessCommitWorker.
+        perform_async(project.id, current_user.id, commit.to_hash, default)
+    end
+  end
+
   protected
 
   def execute_related_hooks
@@ -128,17 +141,6 @@ class GitPushService < BaseService
     end
   end
 
-  # Extract any GFM references from the pushed commit messages. If the configured issue-closing regex is matched,
-  # close the referenced Issue. Create cross-reference Notes corresponding to any other referenced Mentionables.
-  def process_commit_messages
-    default = is_default_branch?
-
-    @push_commits.each do |commit|
-      ProcessCommitWorker.
-        perform_async(project.id, current_user.id, commit.to_hash, default)
-    end
-  end
-
   def build_push_data
     @push_data ||= Gitlab::DataBuilder::Push.build(
       @project,

changelogs/unreleased/process-commit-worker-large-batches.yml

+---
+title: Push payloads schedule at most 100 commits, instead of all commits
+merge_request:
+author:

spec/services/git_push_service_spec.rb

@@ -604,6 +604,25 @@ describe GitPushService, services: true do
     end
   end
 
+  describe '#process_commit_messages' do
+    let(:service) do
+      described_class.new(project,
+                          user,
+                          oldrev: sample_commit.parent_id,
+                          newrev: sample_commit.id,
+                          ref: 'refs/heads/master')
+    end
+
+    it 'only schedules a limited number of commits' do
+      allow(service).to receive(:push_commits).
+        and_return(Array.new(1000, double(:commit, to_hash: {})))
+
+      expect(ProcessCommitWorker).to receive(:perform_async).exactly(100).times
+
+      service.process_commit_messages
+    end
+  end
+
   def execute_service(project, user, oldrev, newrev, ref)
     service = described_class.new(project, user, oldrev: oldrev, newrev: newrev, ref: ref )
     service.execute