Create downstream pipeline inside same project

Allow CreateCrossProjectPipelineService to create
a pipeline when the same project is provided for
target and source.

app/models/ci/pipeline.rb

@@ -694,6 +694,19 @@ module Ci
       all_merge_requests.order(id: :desc)
     end
 
+    # If pipeline is a child of another pipeline, include the parent
+    # and the siblings, otherwise return only itself.
+    def same_family_pipeline_ids
+      upstream_pipeline = triggered_by_pipeline
+
+      if upstream_pipeline && upstream_pipeline.project == self.project
+        child_pipeline_ids = upstream_pipeline&.triggered_pipelines&.pluck(:id) || []
+        child_pipeline_ids + [upstream_pipeline.id]
+      else
+        [self.id]
+      end
+    end
+
     def detailed_status(current_user)
       Gitlab::Ci::Status::Pipeline::Factory
         .new(self, current_user)

app/models/ci/pipeline_enums.rb

@@ -23,10 +23,11 @@ module Ci
         schedule: 4,
         api: 5,
         external: 6,
-        pipeline: 7,
+        cross_project_pipeline: 7,
         chat: 8,
         merge_request_event: 10,
-        external_pull_request_event: 11
+        external_pull_request_event: 11,
+        parent_pipeline: 12
       }
     end
 
@@ -38,7 +39,8 @@ module Ci
         repository_source: 1,
         auto_devops_source: 2,
         remote_source: 4,
-        external_project_source: 5
+        external_project_source: 5,
+        bridge_source: 6
       }
     end
 

app/services/ci/create_pipeline_service.rb

@@ -23,7 +23,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze
 
     # rubocop: disable Metrics/ParameterLists
-    def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, external_pull_request: nil, **options, &block)
+    def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, external_pull_request: nil, config_content: nil, **options, &block)
       @pipeline = Ci::Pipeline.new
 
       command = Gitlab::Ci::Pipeline::Chain::Command.new(
@@ -46,6 +46,7 @@ module Ci
         current_user: current_user,
         push_options: params[:push_options] || {},
         chat_data: params[:chat_data],
+        config_content: config_content,
         **extra_options(options))
 
       sequence = Gitlab::Ci::Pipeline::Chain::Sequence
@@ -104,14 +105,14 @@ module Ci
       if Feature.enabled?(:ci_support_interruptible_pipelines, project, default_enabled: true)
         project.ci_pipelines
           .where(ref: pipeline.ref)
-          .where.not(id: pipeline.id)
+          .where.not(id: pipeline.same_family_pipeline_ids)
           .where.not(sha: project.commit(pipeline.ref).try(:id))
           .alive_or_scheduled
           .with_only_interruptible_builds
       else
         project.ci_pipelines
           .where(ref: pipeline.ref)
-          .where.not(id: pipeline.id)
+          .where.not(id: pipeline.same_family_pipeline_ids)
           .where.not(sha: project.commit(pipeline.ref).try(:id))
           .created_or_pending
       end

app/services/ci/pipeline_trigger_service.rb

@@ -44,7 +44,7 @@ module Ci
       return error("400 Job has to be running", 400) unless job.running?
 
       pipeline = Ci::CreatePipelineService.new(project, job.user, ref: params[:ref])
-        .execute(:pipeline, ignore_skip_ci: true) do |pipeline|
+        .execute(:cross_project_pipeline, ignore_skip_ci: true) do |pipeline|
           source = job.sourced_pipelines.build(
             source_pipeline: job.pipeline,
             source_project: job.project,

changelogs/unreleased/create-downstream-pipeline-in-same-project.yml

+---
+title: Allow an upstream pipeline to create a downstream pipeline in the same project
+merge_request: 20930
+author:
+type: added

ee/app/models/ee/ci/bridge.rb

@@ -102,12 +102,22 @@ module EE
         downstream_project || upstream_project
       end
 
+      def triggers_child_pipeline?
+        same_project? && yaml_for_downstream.present?
+      end
+
       def downstream_project
         strong_memoize(:downstream_project) do
           options&.dig(:trigger, :project)
         end
       end
 
+      def yaml_for_downstream
+        strong_memoize(:yaml_for_downstream) do
+          options&.dig(:trigger, :yaml)
+        end
+      end
+
       def upstream_project
         strong_memoize(:upstream_project) do
           options&.dig(:bridge_needs, :pipeline)
@@ -138,6 +148,12 @@ module EE
           end
         end
       end
+
+      private
+
+      def same_project?
+        ::Project.find_by_full_path(downstream_project) == project
+      end
     end
   end
 end

ee/app/services/ci/create_cross_project_pipeline_service.rb

 # frozen_string_literal: true
 
 module Ci
+  # TODO: rename this (and worker) to CreateDownstreamPipelineService
   class CreateCrossProjectPipelineService < ::BaseService
     include Gitlab::Utils::StrongMemoize
 
@@ -11,7 +12,7 @@ module Ci
         return bridge.drop!(:downstream_bridge_project_not_found)
       end
 
-      if target_project == project
+      if target_project == project && !bridge.triggers_child_pipeline?
         return bridge.drop!(:invalid_bridge_trigger)
       end
 
@@ -19,7 +20,11 @@ module Ci
         return bridge.drop!(:insufficient_bridge_permissions)
       end
 
-      create_pipeline!
+      if bridge.triggers_child_pipeline?
+        create_child_pipeline!
+      else
+        create_cross_project_pipeline!
+      end
     end
 
     private
@@ -39,10 +44,34 @@ module Ci
       ::Gitlab::UserAccess.new(target_user, project: target_project).can_update_branch?(target_ref)
     end
 
-    def create_pipeline!
+    def create_cross_project_pipeline!
       ::Ci::CreatePipelineService
         .new(target_project, target_user, ref: target_ref)
-        .execute(:pipeline, ignore_skip_ci: true) do |pipeline|
+        .execute(:cross_project_pipeline, ignore_skip_ci: true) do |pipeline|
+          @bridge.sourced_pipelines.build(
+            source_pipeline: @bridge.pipeline,
+            source_project: @bridge.project,
+            project: target_project,
+            pipeline: pipeline)
+
+          pipeline.variables.build(@bridge.downstream_variables)
+        end
+    end
+
+    def create_child_pipeline!
+      return unless @bridge.triggers_child_pipeline?
+
+      parent_pipeline = @bridge.pipeline
+
+      ::Ci::CreatePipelineService
+        .new(@bridge.project, @bridge.user,
+          ref: parent_pipeline.ref,
+          checkout_sha: parent_pipeline.sha,
+          before: parent_pipeline.before_sha,
+          source_sha: parent_pipeline.source_sha,
+          target_sha: parent_pipeline.target_sha
+        )
+        .execute(:parent_pipeline, ignore_skip_ci: true, config_content: @bridge.yaml_for_downstream) do |pipeline|
           @bridge.sourced_pipelines.build(
             source_pipeline: @bridge.pipeline,
             source_project: @bridge.project,
@@ -53,6 +82,37 @@ module Ci
         end
     end
 
+    def create_child_pipeline!
+      parent_pipeline = @bridge.pipeline
+
+      ::Ci::CreatePipelineService
+        .new(@bridge.project, @bridge.user,
+          ref: parent_pipeline.ref,
+          checkout_sha: parent_pipeline.sha,
+          before: parent_pipeline.before_sha,
+          source_sha: parent_pipeline.source_sha,
+          target_sha: parent_pipeline.target_sha
+        )
+        .execute(:pipeline,
+          ignore_skip_ci: true,
+          config_content: downstream_yaml,
+          schedule: parent_pipeline.pipeline_schedule) do |pipeline|
+            @bridge.sourced_pipelines.build(
+              source_pipeline: @bridge.pipeline,
+              source_project: @bridge.project,
+              project: target_project,
+              pipeline: pipeline)
+
+            pipeline.variables.build(@bridge.downstream_variables)
+          end
+    end
+
+    def downstream_yaml
+      return unless @bridge.triggers_child_pipeline?
+
+      @bridge.downstream_yaml
+    end
+
     def target_user
       strong_memoize(:target_user) { @bridge.target_user }
     end

ee/spec/models/ci/bridge_spec.rb

@@ -289,4 +289,48 @@ describe Ci::Bridge do
       expect(bridge.metadata.config_options).to be bridge.options
     end
   end
+
+  describe '#triggers_child_pipeline?' do
+    subject { bridge.triggers_child_pipeline? }
+
+    context 'when downstream project is same as the bridge project' do
+      context 'when bridge defines a downstream YAML' do
+        let(:options) do
+          {
+            trigger: {
+              project: project.full_path,
+              yaml: YAML.dump(rspec: { script: 'rspec' })
+            }
+          }
+        end
+
+        it { is_expected.to be_truthy }
+      end
+
+      context 'when bridge does not define a downstream YAML' do
+        let(:options) do
+          {
+            trigger: {
+              project: project.full_path
+            }
+          }
+        end
+
+        it { is_expected.to be_falsey }
+      end
+    end
+
+    context 'when downstream project is different than bridge project' do
+      let(:options) do
+        {
+          trigger: {
+            project: 'my/project',
+            yaml: YAML.dump(rspec: { script: 'rspec' })
+          }
+        }
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
 end

ee/spec/requests/api/triggers_spec.rb

@@ -60,7 +60,7 @@ describe API::Triggers do
             expect { subject }.to change(Ci::Pipeline, :count)
 
             expect(response).to have_http_status(201)
-            expect(Ci::Pipeline.last.source).to eq('pipeline')
+            expect(Ci::Pipeline.last.source).to eq('cross_project_pipeline')
             expect(Ci::Pipeline.last.triggered_by_pipeline).not_to be_nil
             expect(Ci::Sources::Pipeline.last).to have_attributes(
               pipeline_id: (a_value > 0),
@@ -94,7 +94,7 @@ describe API::Triggers do
                                 .and change(Ci::PipelineVariable, :count)
 
               expect(response).to have_http_status(201)
-              expect(Ci::Pipeline.last.source).to eq('pipeline')
+              expect(Ci::Pipeline.last.source).to eq('cross_project_pipeline')
               expect(Ci::Pipeline.last.triggered_by_pipeline).not_to be_nil
               expect(Ci::Pipeline.last.variables.map { |v| { v.key => v.value } }.last).to eq(params[:variables])
             end

ee/spec/services/ci/create_cross_project_pipeline_service_spec.rb

@@ -4,10 +4,10 @@ require 'spec_helper'
 
 describe Ci::CreateCrossProjectPipelineService, '#execute' do
   set(:user) { create(:user) }
-  set(:upstream_project) { create(:project, :repository) }
+  let(:upstream_project) { create(:project, :repository) }
   set(:downstream_project) { create(:project, :repository) }
 
-  set(:upstream_pipeline) do
+  let!(:upstream_pipeline) do
     create(:ci_pipeline, :running, project: upstream_project)
   end
 
@@ -30,7 +30,6 @@ describe Ci::CreateCrossProjectPipelineService, '#execute' do
   let(:service) { described_class.new(upstream_project, user) }
 
   before do
-    stub_ci_pipeline_to_return_yaml_file
     upstream_project.add_developer(user)
   end
 
@@ -126,21 +125,100 @@ describe Ci::CreateCrossProjectPipelineService, '#execute' do
       end
     end
 
-    context 'when circular dependency is defined' do
+    context 'when downstream project is the same as the job project' do
       let(:trigger) do
         { trigger: { project: upstream_project.full_path } }
       end
 
-      it 'does not create a new pipeline' do
-        expect { service.execute(bridge) }
-          .not_to change { Ci::Pipeline.count }
+      before do
+        downstream_project.add_developer(user)
       end
 
-      it 'changes status of the bridge build' do
-        service.execute(bridge)
+      context 'detects a circular dependency' do
+        it 'does not create a new pipeline' do
+          expect { service.execute(bridge) }
+            .not_to change { Ci::Pipeline.count }
+        end
 
-        expect(bridge.reload).to be_failed
-        expect(bridge.failure_reason).to eq 'invalid_bridge_trigger'
+        it 'changes status of the bridge build' do
+          service.execute(bridge)
+
+          expect(bridge.reload).to be_failed
+          expect(bridge.failure_reason).to eq 'invalid_bridge_trigger'
+        end
+      end
+
+      context 'when a custom YAML is provided' do
+        shared_examples 'creates a child pipeline' do
+          it 'creates only one new pipeline' do
+            expect { service.execute(bridge) }
+              .to change { Ci::Pipeline.count }.by(1)
+          end
+
+          it 'creates a child pipeline in the same project' do
+            pipeline = service.execute(bridge)
+            pipeline.reload
+
+            expect(pipeline.builds.map(&:name)).to eq %w[rspec echo]
+            expect(pipeline.user).to eq bridge.user
+            expect(pipeline.project).to eq bridge.project
+            expect(bridge.sourced_pipelines.first.pipeline).to eq pipeline
+            expect(pipeline.triggered_by_pipeline).to eq upstream_pipeline
+            expect(pipeline.source_bridge).to eq bridge
+            expect(pipeline.source_bridge).to be_a ::Ci::Bridge
+          end
+
+          it 'updates bridge status when downstream pipeline gets proceesed' do
+            pipeline = service.execute(bridge)
+
+            expect(pipeline.reload).to be_pending
+            expect(bridge.reload).to be_success
+          end
+
+          it 'propagates parent pipeline settings to the child pipeline' do
+            pipeline = service.execute(bridge)
+            pipeline.reload
+
+            expect(pipeline.ref).to eq(upstream_pipeline.ref)
+            expect(pipeline.sha).to eq(upstream_pipeline.sha)
+            expect(pipeline.source_sha).to eq(upstream_pipeline.source_sha)
+            expect(pipeline.target_sha).to eq(upstream_pipeline.target_sha)
+            expect(pipeline.target_sha).to eq(upstream_pipeline.target_sha)
+
+            expect(pipeline.trigger_requests.last).to eq(bridge.trigger_request)
+          end
+        end
+
+        before do
+          file_content = YAML.dump(
+            rspec: { script: 'rspec' },
+            echo: { script: 'echo' })
+          upstream_project.repository.create_file(
+            user, 'child-pipeline.yml', file_content, message: 'message', branch_name: 'master')
+
+          upstream_pipeline.update!(sha: upstream_project.commit.id)
+        end
+
+        let(:trigger) do
+          {
+            trigger: {
+              project: upstream_project.full_path,
+              yaml: YAML.dump({ include: 'child-pipeline.yml' })
+            }
+          }
+        end
+
+        it_behaves_like 'creates a child pipeline'
+
+        context 'when latest sha for the ref changed in the meantime' do
+          before do
+            upstream_project.repository.create_file(
+              user, 'another-change', 'test', message: 'message', branch_name: 'master')
+          end
+
+          # it does not auto-cancel pipelines from the same family
+          it_behaves_like 'creates a child pipeline'
+        end
       end
     end
 

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -9,7 +9,7 @@ module Gitlab
             include Chain::Helpers
 
             SOURCES = [
-              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge,
               Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
               Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,
               Gitlab::Ci::Pipeline::Chain::Config::Content::Remote,
@@ -17,7 +17,7 @@ module Gitlab
             ].freeze
 
             LEGACY_SOURCES = [
-              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge,
               Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository,
               Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops
             ].freeze

lib/gitlab/ci/pipeline/chain/config/content/runtime.rb → lib/gitlab/ci/pipeline/chain/config/content/bridge.rb

@@ -6,20 +6,16 @@ module Gitlab
       module Chain
         module Config
           class Content
-            class Runtime < Source
+            # This case represents when a config content is passed in
+            # as parameter to Ci::CreatePipelineService from the outside.
+            # For example when creating a child pipeline.
+            class Bridge < Source
               def content
                 @command.config_content
               end
 
               def source
-                # The only case when this source is used is when the config content
-                # is passed in as parameter to Ci::CreatePipelineService.
-                # This would only occur with parent/child pipelines which is being
-                # implemented.
-                # TODO: change source to return :runtime_source
-                # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041
-
-                nil
+                :bridge_source
               end
             end
           end

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

@@ -15,6 +15,21 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         stub_feature_flags(ci_root_config_content: false)
       end
 
+      context 'when config is passed in as parameter and already available in command' do
+        let(:ci_config_path) { nil }
+
+        before do
+          command.config_content = 'the-content'
+        end
+
+        it 'returns the content already available in command' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'bridge_source'
+          expect(command.config_content).to eq 'the-content'
+        end
+      end
+
       context 'when config is defined in a custom path in the repository' do
         let(:ci_config_path) { 'path/to/config.yml' }
 
@@ -135,6 +150,21 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
       end
     end
 
+    context 'when config is passed in as parameter and already available in command' do
+      let(:ci_config_path) { nil }
+
+      before do
+        command.config_content = 'the-content'
+      end
+
+      it 'returns the content already available in command' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'bridge_source'
+        expect(command.config_content).to eq 'the-content'
+      end
+    end
+
     context 'when config is defined in a custom path in the repository' do
       let(:ci_config_path) { 'path/to/config.yml' }
       let(:config_content_result) do

spec/services/ci/create_pipeline_service/custom_config_content_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:admin) }
+  let(:ref) { 'refs/heads/master' }
+  let(:service) { described_class.new(project, user, { ref: ref }) }
+  let(:pipeline) { service.execute(:push, config_content: config_content) }
+
+  context 'custom config content' do
+    let(:config_content) do
+      YAML.dump(
+        rspec: { script: 'rspec' },
+        custom: { script: 'custom' }
+      )
+    end
+
+    it 'creates a pipeline using the content passed in as param' do
+      expect(pipeline).to be_persisted
+      expect(pipeline.builds.map(&:name)).to eq %w[rspec custom]
+      expect(pipeline.config_source).to eq 'bridge_source'
+    end
+  end
+end