Merge branch 'limit-amount-of-created-pipelines' into 'master'

Process at most 4 pipelines during push

See merge request gitlab-org/gitlab-ce!27205

app/services/git/base_hooks_service.rb

@@ -51,6 +51,8 @@ module Git
     end
 
     def create_pipelines
+      return unless params.fetch(:create_pipelines, true)
+
       Ci::CreatePipelineService
         .new(project, current_user, push_data)
         .execute(:push, pipeline_options)

app/workers/post_receive.rb

@@ -3,6 +3,8 @@
 class PostReceive
   include ApplicationWorker
 
+  PIPELINE_PROCESS_LIMIT = 4
+
   def perform(gl_repository, identifier, changes, push_options = {})
     project, repo_type = Gitlab::GlRepository.parse(gl_repository)
 
@@ -36,23 +38,24 @@ class PostReceive
       return false
     end
 
-    post_received.changes_refs do |oldrev, newrev, ref|
-      if Gitlab::Git.tag_ref?(ref)
-        Git::TagPushService.new(
-          post_received.project,
-          @user,
-          oldrev: oldrev,
-          newrev: newrev,
-          ref: ref,
-          push_options: post_received.push_options).execute
-      elsif Gitlab::Git.branch_ref?(ref)
-        Git::BranchPushService.new(
+    post_received.enum_for(:changes_refs).with_index do |(oldrev, newrev, ref), index|
+      service_klass =
+        if Gitlab::Git.tag_ref?(ref)
+          Git::TagPushService
+        elsif Gitlab::Git.branch_ref?(ref)
+          Git::BranchPushService
+        end
+
+      if service_klass
+        service_klass.new(
           post_received.project,
           @user,
           oldrev: oldrev,
           newrev: newrev,
           ref: ref,
-          push_options: post_received.push_options).execute
+          push_options: post_received.push_options,
+          create_pipelines: index < PIPELINE_PROCESS_LIMIT || Feature.enabled?(:git_push_create_all_pipelines, post_received.project)
+        ).execute
       end
 
       changes << Gitlab::DataBuilder::Repository.single_change(oldrev, newrev, ref)

changelogs/unreleased/limit-amount-of-created-pipelines.yml

+---
+title: Process at most 4 pipelines during push
+merge_request: 27205
+author:
+type: performance

doc/ci/yaml/README.md

@@ -2701,6 +2701,15 @@ Not to be confused with [`trigger`](#trigger-premium).
 
 [Read more in the triggers documentation.](../triggers/README.md)
 
+## Processing Git pushes
+
+GitLab will create at most 4 branch and tags pipelines when
+doing pushing multiple changes in single `git push` invocation.
+
+This limitation does not affect any of the updated Merge Request pipelines,
+all updated Merge Requests will have a pipeline created when using
+[pipelines for merge requests](../merge_request_pipelines/index.md).
+
 ## Skipping jobs
 
 If your commit message contains `[ci skip]` or `[skip ci]`, using any

spec/workers/post_receive_spec.rb

@@ -99,11 +99,21 @@ describe PostReceive do
       end
 
       context "gitlab-ci.yml" do
-        let(:changes) { "123456 789012 refs/heads/feature\n654321 210987 refs/tags/tag" }
+        let(:changes) do
+          <<-EOF.strip_heredoc
+            123456 789012 refs/heads/feature
+            654321 210987 refs/tags/tag
+            123456 789012 refs/heads/feature2
+            123458 789013 refs/heads/feature3
+            123459 789015 refs/heads/feature4
+          EOF
+        end
+
+        let(:changes_count) { changes.lines.count }
 
         subject { described_class.new.perform(gl_repository, key_id, base64_changes) }
 
-        context "creates a Ci::Pipeline for every change" do
+        context "with valid .gitlab-ci.yml" do
           before do
             stub_ci_pipeline_to_return_yaml_file
 
@@ -116,7 +126,33 @@ describe PostReceive do
               .and_return(true)
           end
 
-          it { expect { subject }.to change { Ci::Pipeline.count }.by(2) }
+          context 'when git_push_create_all_pipelines is disabled' do
+            before do
+              stub_feature_flags(git_push_create_all_pipelines: false)
+            end
+
+            it "creates pipeline for branches and tags" do
+              subject
+
+              expect(Ci::Pipeline.pluck(:ref)).to contain_exactly("feature", "tag", "feature2", "feature3")
+            end
+
+            it "creates exactly #{described_class::PIPELINE_PROCESS_LIMIT} pipelines" do
+              expect(changes_count).to be > described_class::PIPELINE_PROCESS_LIMIT
+
+              expect { subject }.to change { Ci::Pipeline.count }.by(described_class::PIPELINE_PROCESS_LIMIT)
+            end
+          end
+
+          context 'when git_push_create_all_pipelines is enabled' do
+            before do
+              stub_feature_flags(git_push_create_all_pipelines: true)
+            end
+
+            it "creates all pipelines" do
+              expect { subject }.to change { Ci::Pipeline.count }.by(changes_count)
+            end
+          end
         end
 
         context "does not create a Ci::Pipeline" do