Merge branch 'support-ci-resource-group-in-cross-project-pipeline' into 'master'

Support Resrouce Group in Cross-Project/Parent-Child pipelines

See merge request gitlab-org/gitlab!53007

app/models/ci/bridge.rb

@@ -27,7 +27,7 @@ module Ci
     # rubocop:enable Cop/ActiveRecordSerialize
 
     state_machine :status do
-      after_transition [:created, :manual] => :pending do |bridge|
+      after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
         next unless bridge.downstream_project
 
         bridge.run_after_commit do
@@ -156,6 +156,10 @@ module Ci
       false
     end
 
+    def any_unmet_prerequisites?
+      false
+    end
+
     def expanded_environment_name
     end
 

changelogs/unreleased/support-ci-resource-group-in-cross-project-pipeline.yml

+---
+title: Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
+merge_request: 53007
+author:
+type: added

doc/ci/yaml/README.md

@@ -3924,6 +3924,60 @@ It can't start or end with `/`.
 
 For more information, see [Deployments Safety](../environments/deployment_safety.md).
 
+#### Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39057) in GitLab 13.9.
+
+You can define `resource_group` for downstream pipelines that are sensitive to concurrent
+executions. The [`trigger` keyword](#trigger) can trigger downstream pipelines. The
+[`resource_group` keyword](#resource_group) can co-exist with it. This is useful to control the
+concurrency for deployment pipelines, while running non-sensitive jobs concurrently.
+
+This example has two pipeline configurations in a project. When a pipeline starts running,
+non-sensitive jobs are executed first and aren't affected by concurrent executions in other
+pipelines. However, GitLab ensures that there are no other deployment pipelines running before
+triggering a deployment (child) pipeline. If other deployment pipelines are running, GitLab waits
+until those pipelines finish before running another one.
+
+```yaml
+# .gitlab-ci.yml (parent pipeline)
+
+build:
+  stage: build
+  script: echo "Building..."
+
+test:
+  stage: test
+  script: echo "Testing..."
+
+deploy:
+  stage: deploy
+  trigger:
+    include: deploy.gitlab-ci.yml
+    strategy: depend
+  resource_group: AWS-production
+```
+
+```yaml
+# deploy.gitlab-ci.yml (child pipeline)
+
+stages:
+  - provision
+  - deploy
+
+provision:
+  stage: provision
+  script: echo "Provisioning..."
+
+deployment:
+  stage: deploy
+  script: echo "Deploying..."
+```
+
+Note that you must define [`strategy: depend`](#linking-pipelines-with-triggerstrategy)
+with the `trigger` keyword. This ensures that the lock isn't released until the downstream pipeline
+finishes.
+
 ### `release`
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2.

lib/gitlab/ci/pipeline/seed/build.rb

@@ -73,17 +73,28 @@ module Gitlab
 
           def to_resource
             strong_memoize(:resource) do
-              if bridge?
-                ::Ci::Bridge.new(attributes)
-              else
-                ::Ci::Build.new(attributes).tap do |build|
-                  build.assign_attributes(self.class.environment_attributes_for(build))
-                  build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource
-                end
+              processable = initialize_processable
+              assign_resource_group(processable)
+              processable
+            end
+          end
+
+          def initialize_processable
+            if bridge?
+              ::Ci::Bridge.new(attributes)
+            else
+              ::Ci::Build.new(attributes).tap do |build|
+                build.assign_attributes(self.class.environment_attributes_for(build))
               end
             end
           end
 
+          def assign_resource_group(processable)
+            processable.resource_group =
+              Seed::Processable::ResourceGroup.new(processable, @resource_group_key)
+                                              .to_resource
+          end
+
           def self.environment_attributes_for(build)
             return {} unless build.has_environment?
 

lib/gitlab/ci/pipeline/seed/build/resource_group.rb → lib/gitlab/ci/pipeline/seed/processable/resource_group.rb

@@ -4,7 +4,7 @@ module Gitlab
   module Ci
     module Pipeline
       module Seed
-        class Build
+        module Processable
           class ResourceGroup < Seed::Base
             include Gitlab::Utils::StrongMemoize
 

lib/gitlab/ci/status/bridge/factory.rb

@@ -8,6 +8,7 @@ module Gitlab
           def self.extended_statuses
             [[Status::Bridge::Failed],
              [Status::Bridge::Manual],
+             [Status::Bridge::WaitingForResource],
              [Status::Bridge::Play],
              [Status::Bridge::Action]]
           end

lib/gitlab/ci/status/bridge/waiting_for_resource.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Bridge
+        class WaitingForResource < Status::Processable::WaitingForResource
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/status/build/waiting_for_resource.rb

@@ -4,22 +4,7 @@ module Gitlab
   module Ci
     module Status
       module Build
-        class WaitingForResource < Status::Extended
-          ##
-          # TODO: image is shared with 'pending'
-          # until we get a dedicated one
-          #
-          def illustration
-            {
-              image: 'illustrations/pending_job_empty.svg',
-              size: 'svg-430',
-              title: _('This job is waiting for resource: ') + subject.resource_group.key
-            }
-          end
-
-          def self.matches?(build, _)
-            build.waiting_for_resource?
-          end
+        class WaitingForResource < Status::Processable::WaitingForResource
         end
       end
     end

lib/gitlab/ci/status/processable/waiting_for_resource.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Processable
+        class WaitingForResource < Status::Extended
+          ##
+          # TODO: image is shared with 'pending'
+          # until we get a dedicated one
+          #
+          def illustration
+            {
+              image: 'illustrations/pending_job_empty.svg',
+              size: 'svg-430',
+              title: _('This job is waiting for resource: ') + subject.resource_group.key
+            }
+          end
+
+          def self.matches?(processable, _)
+            processable.waiting_for_resource?
+          end
+        end
+      end
+    end
+  end
+end

spec/factories/ci/bridge.rb

 # frozen_string_literal: true
 
 FactoryBot.define do
-  factory :ci_bridge, class: 'Ci::Bridge' do
+  factory :ci_bridge, class: 'Ci::Bridge', parent: :ci_processable do
     name { 'bridge' }
-    stage { 'test' }
-    stage_idx { 0 }
-    ref { 'master' }
-    tag { false }
     created_at { '2013-10-29 09:50:00 CET' }
     status { :created }
-    scheduling_type { 'stage' }
-
-    pipeline factory: :ci_pipeline
 
     trait :variables do
       yaml_variables do

spec/factories/ci/builds.rb

@@ -3,15 +3,10 @@
 include ActionDispatch::TestProcess
 
 FactoryBot.define do
-  factory :ci_build, class: 'Ci::Build' do
+  factory :ci_build, class: 'Ci::Build', parent: :ci_processable do
     name { 'test' }
-    stage { 'test' }
-    stage_idx { 0 }
-    ref { 'master' }
-    tag { false }
     add_attribute(:protected) { false }
     created_at { 'Di 29. Okt 09:50:00 CET 2013' }
-    scheduling_type { 'stage' }
     pending
 
     options do
@@ -28,7 +23,6 @@ FactoryBot.define do
       ]
     end
 
-    pipeline factory: :ci_pipeline
     project { pipeline.project }
 
     trait :degenerated do
@@ -79,10 +73,6 @@ FactoryBot.define do
       status { 'created' }
     end
 
-    trait :waiting_for_resource do
-      status { 'waiting_for_resource' }
-    end
-
     trait :preparing do
       status { 'preparing' }
     end
@@ -213,14 +203,6 @@ FactoryBot.define do
       trigger_request factory: :ci_trigger_request
     end
 
-    trait :resource_group do
-      waiting_for_resource_at { 5.minutes.ago }
-
-      after(:build) do |build, evaluator|
-        build.resource_group = create(:ci_resource_group, project: build.project)
-      end
-    end
-
     trait :with_deployment do
       after(:build) do |build, evaluator|
         ##

spec/factories/ci/processable.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :ci_processable, class: 'Ci::Processable' do
+    name { 'processable' }
+    stage { 'test' }
+    stage_idx { 0 }
+    ref { 'master' }
+    tag { false }
+    pipeline factory: :ci_pipeline
+    project { pipeline.project }
+    scheduling_type { 'stage' }
+
+    trait :waiting_for_resource do
+      status { 'waiting_for_resource' }
+    end
+
+    trait :resource_group do
+      waiting_for_resource_at { 5.minutes.ago }
+
+      after(:build) do |processable, evaluator|
+        processable.resource_group = create(:ci_resource_group, project: processable.project)
+      end
+    end
+  end
+end

spec/features/projects/pipelines/pipeline_spec.rb

@@ -846,6 +846,28 @@ RSpec.describe 'Pipeline', :js do
             end
           end
         end
+
+        context 'when deploy job is a bridge to trigger a downstream pipeline' do
+          let!(:deploy_job) do
+            create(:ci_bridge, :created, stage: 'deploy', name: 'deploy',
+              stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group)
+          end
+
+          it 'shows deploy job as waiting for resource' do
+            subject
+
+            within('.pipeline-header-container') do
+              expect(page).to have_content('waiting')
+            end
+
+            within('.pipeline-graph') do
+              within '.stage-column:nth-child(2)' do
+                expect(page).to have_content('deploy')
+                expect(page).to have_css('.ci-status-icon-waiting-for-resource')
+              end
+            end
+          end
+        end
       end
     end
   end

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -383,14 +383,25 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
     end
 
     context 'when job is a bridge' do
-      let(:attributes) do
+      let(:base_attributes) do
         {
           name: 'rspec', ref: 'master', options: { trigger: 'my/project' }, scheduling_type: :stage
         }
       end
 
+      let(:attributes) { base_attributes }
+
       it { is_expected.to be_a(::Ci::Bridge) }
       it { is_expected.to be_valid }
+
+      context 'when job belongs to a resource group' do
+        let(:attributes) { base_attributes.merge(resource_group_key: 'iOS') }
+
+        it 'returns a job with resource group' do
+          expect(subject.resource_group).not_to be_nil
+          expect(subject.resource_group.key).to eq('iOS')
+        end
+      end
     end
 
     it 'memoizes a resource object' do

spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb → spec/lib/gitlab/ci/pipeline/seed/processable/resource_group_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::ResourceGroup do
+RSpec.describe Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup do
   let_it_be(:project) { create(:project) }
   let(:job) { build(:ci_build, project: project) }
   let(:seed) { described_class.new(job, resource_group_key) }

spec/lib/gitlab/ci/status/bridge/factory_spec.rb

@@ -117,14 +117,31 @@ RSpec.describe Gitlab::Ci::Status::Bridge::Factory do
     end
   end
 
+  context 'when bridge is waiting for resource' do
+    let(:bridge) { create_bridge(:waiting_for_resource, :resource_group) }
+
+    it 'matches correct core status' do
+      expect(factory.core_status).to be_a Gitlab::Ci::Status::WaitingForResource
+    end
+
+    it 'fabricates status with correct details' do
+      expect(status.text).to eq 'waiting'
+      expect(status.group).to eq 'waiting-for-resource'
+      expect(status.icon).to eq 'status_pending'
+      expect(status.favicon).to eq 'favicon_pending'
+      expect(status.illustration).to include(:image, :size, :title)
+      expect(status).not_to have_details
+    end
+  end
+
   private
 
-  def create_bridge(trait)
+  def create_bridge(*traits)
     upstream_project = create(:project, :repository)
     downstream_project = create(:project, :repository)
     upstream_pipeline = create(:ci_pipeline, :running, project: upstream_project)
     trigger = { trigger: { project: downstream_project.full_path, branch: 'feature' } }
 
-    create(:ci_bridge, trait, options: trigger, pipeline: upstream_pipeline)
+    create(:ci_bridge, *traits, options: trigger, pipeline: upstream_pipeline)
   end
 end

spec/lib/gitlab/ci/status/bridge/waiting_for_resource_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Ci::Status::Bridge::WaitingForResource do
+  it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource }
+end

spec/lib/gitlab/ci/status/build/waiting_for_resource_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Ci::Status::Build::WaitingForResource do
+  it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource }
+end

spec/lib/gitlab/ci/status/processable/waiting_for_resource_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource do
+  let(:user) { create(:user) }
+
+  subject do
+    processable = create(:ci_build, :waiting_for_resource, :resource_group)
+    described_class.new(Gitlab::Ci::Status::Core.new(processable, user))
+  end
+
+  describe '#illustration' do
+    it { expect(subject.illustration).to include(:image, :size, :title) }
+  end
+
+  describe '.matches?' do
+    subject {described_class.matches?(processable, user) }
+
+    context 'when processable is waiting for resource' do
+      let(:processable) { create(:ci_build, :waiting_for_resource) }
+
+      it 'is a correct match' do
+        expect(subject).to be true
+      end
+    end
+
+    context 'when processable is not waiting for resource' do
+      let(:processable) { create(:ci_build) }
+
+      it 'does not match' do
+        expect(subject).to be false
+      end
+    end
+  end
+end

spec/models/ci/bridge_spec.rb

@@ -80,6 +80,14 @@ RSpec.describe Ci::Bridge do
         end
       end
 
+      it "schedules downstream pipeline creation when the status is waiting for resource" do
+        bridge.status = :waiting_for_resource
+
+        expect(bridge).to receive(:schedule_downstream_pipeline!)
+
+        bridge.enqueue_waiting_for_resource!
+      end
+
       it 'raises error when the status is failed' do
         bridge.status = :failed
 

spec/services/ci/create_pipeline_service/cross_project_pipeline_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService, '#execute' do
+  let_it_be(:group) { create(:group, name: 'my-organization') }
+  let(:upstream_project) { create(:project, :repository, name: 'upstream', group: group) }
+  let(:downstram_project) { create(:project, :repository, name: 'downstream', group: group) }
+  let(:user) { create(:user) }
+
+  let(:service) do
+    described_class.new(upstream_project, user, ref: 'master')
+  end
+
+  before do
+    upstream_project.add_developer(user)
+    downstram_project.add_developer(user)
+    create_gitlab_ci_yml(upstream_project, upstream_config)
+    create_gitlab_ci_yml(downstram_project, downstream_config)
+  end
+
+  context 'with resource group', :aggregate_failures do
+    let(:upstream_config) do
+      <<~YAML
+      instrumentation_test:
+        stage: test
+        resource_group: iOS
+        trigger:
+          project: my-organization/downstream
+          strategy: depend
+      YAML
+    end
+
+    let(:downstream_config) do
+      <<~YAML
+      test:
+        script: echo "Testing..."
+      YAML
+    end
+
+    it 'creates bridge job with resource group' do
+      pipeline = create_pipeline!
+
+      test = pipeline.statuses.find_by(name: 'instrumentation_test')
+      expect(pipeline).to be_created_successfully
+      expect(pipeline.triggered_pipelines).not_to be_exist
+      expect(upstream_project.resource_groups.count).to eq(1)
+      expect(test).to be_a Ci::Bridge
+      expect(test).to be_waiting_for_resource
+      expect(test.resource_group.key).to eq('iOS')
+    end
+
+    context 'when sidekiq processes the job', :sidekiq_inline do
+      it 'transitions to pending status and triggers a downstream pipeline' do
+        pipeline = create_pipeline!
+
+        test = pipeline.statuses.find_by(name: 'instrumentation_test')
+        expect(test).to be_pending
+        expect(pipeline.triggered_pipelines.count).to eq(1)
+      end
+
+      context 'when the resource is occupied by the other bridge' do
+        before do
+          resource_group = create(:ci_resource_group, project: upstream_project, key: 'iOS')
+          resource_group.assign_resource_to(create(:ci_build, project: upstream_project))
+        end
+
+        it 'stays waiting for resource' do
+          pipeline = create_pipeline!
+
+          test = pipeline.statuses.find_by(name: 'instrumentation_test')
+          expect(test).to be_waiting_for_resource
+          expect(pipeline.triggered_pipelines.count).to eq(0)
+        end
+      end
+    end
+  end
+
+  def create_pipeline!
+    service.execute(:push)
+  end
+
+  def create_gitlab_ci_yml(project, content)
+    project.repository.create_file(user, '.gitlab-ci.yml', content, branch_name: 'master', message: 'test')
+  end
+end

spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb

@@ -84,21 +84,46 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do
           stage: test
           resource_group: iOS
           trigger:
-            include:
-              - local: path/to/child.yml
+            include: path/to/child.yml
+            strategy: depend
         YAML
       end
 
-      # TODO: This test will be properly implemented in the next MR
-      # for https://gitlab.com/gitlab-org/gitlab/-/issues/39057.
-      it 'creates bridge job but still resource group is no-op', :aggregate_failures do
+      it 'creates bridge job with resource group', :aggregate_failures do
         pipeline = create_pipeline!
 
         test = pipeline.statuses.find_by(name: 'instrumentation_test')
-
-        expect(pipeline).to be_persisted
+        expect(pipeline).to be_created_successfully
+        expect(pipeline.triggered_pipelines).not_to be_exist
+        expect(project.resource_groups.count).to eq(1)
         expect(test).to be_a Ci::Bridge
-        expect(project.resource_groups.count).to eq(0)
+        expect(test).to be_waiting_for_resource
+        expect(test.resource_group.key).to eq('iOS')
+      end
+
+      context 'when sidekiq processes the job', :sidekiq_inline do
+        it 'transitions to pending status and triggers a downstream pipeline' do
+          pipeline = create_pipeline!
+
+          test = pipeline.statuses.find_by(name: 'instrumentation_test')
+          expect(test).to be_pending
+          expect(pipeline.triggered_pipelines.count).to eq(1)
+        end
+
+        context 'when the resource is occupied by the other bridge' do
+          before do
+            resource_group = create(:ci_resource_group, project: project, key: 'iOS')
+            resource_group.assign_resource_to(create(:ci_build, project: project))
+          end
+
+          it 'stays waiting for resource' do
+            pipeline = create_pipeline!
+
+            test = pipeline.statuses.find_by(name: 'instrumentation_test')
+            expect(test).to be_waiting_for_resource
+            expect(pipeline.triggered_pipelines.count).to eq(0)
+          end
+        end
       end
     end
   end