Merge branch 'feature/gb/cross-project-pipeline-trigger' into 'master'

Cross-project pipeline triggers /CE

See merge request gitlab-org/gitlab-ce!24664

app/models/ci/bridge.rb

@@ -2,11 +2,13 @@
 
 module Ci
   class Bridge < CommitStatus
+    include Ci::Processable
     include Importable
     include AfterCommitQueue
     include Gitlab::Utils::StrongMemoize
 
     belongs_to :project
+    belongs_to :trigger_request
     validates :ref, presence: true
 
     def self.retry(bridge, current_user)
@@ -23,6 +25,21 @@ module Ci
         .fabricate!
     end
 
+    def schedulable?
+      false
+    end
+
+    def action?
+      false
+    end
+
+    def artifacts?
+      false
+    end
+
+    def expanded_environment_name
+    end
+
     def predefined_variables
       raise NotImplementedError
     end
@@ -30,5 +47,9 @@ module Ci
     def execute_hooks
       raise NotImplementedError
     end
+
+    def to_partial_path
+      'projects/generic_commit_statuses/generic_commit_status'
+    end
   end
 end

app/models/ci/build.rb

@@ -3,6 +3,7 @@
 module Ci
   class Build < CommitStatus
     prepend ArtifactMigratable
+    include Ci::Processable
     include TokenAuthenticatable
     include AfterCommitQueue
     include ObjectStorage::BackgroundMove
@@ -638,10 +639,6 @@ module Ci
       super || project.try(:build_coverage_regex)
     end
 
-    def when
-      read_attribute(:when) || 'on_success'
-    end
-
     def options
       read_metadata_attribute(:options, :config_options, {})
     end

app/models/ci/pipeline.rb

@@ -25,6 +25,8 @@ module Ci
 
     has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline
+    has_many :processables, -> { processables },
+             class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline
     has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline
     has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
     has_many :variables, class_name: 'Ci::PipelineVariable'

app/models/ci/stage.rb

@@ -14,6 +14,7 @@ module Ci
 
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
     has_many :builds, foreign_key: :stage_id
+    has_many :bridges, foreign_key: :stage_id
 
     with_options unless: :importing? do
       validates :project, presence: true

app/models/commit_status.rb

@@ -41,6 +41,7 @@ class CommitStatus < ActiveRecord::Base
   scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) }
   scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
+  scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) }
 
   # We use `CommitStatusEnums.failure_reasons` here so that EE can more easily
   # extend this `Hash` with new values.

app/models/concerns/ci/processable.rb

+# frozen_string_literal: true
+
+module Ci
+  ##
+  # This module implements methods that need to be implemented by CI/CD
+  # entities that are supposed to go through pipeline processing
+  # services.
+  #
+  #
+  module Processable
+    def schedulable?
+      raise NotImplementedError
+    end
+
+    def action?
+      raise NotImplementedError
+    end
+
+    def when
+      read_attribute(:when) || 'on_success'
+    end
+
+    def expanded_environment_name
+      raise NotImplementedError
+    end
+  end
+end

app/services/ci/process_pipeline_service.rb

@@ -10,7 +10,7 @@ module Ci
       update_retried
 
       new_builds =
-        stage_indexes_of_created_builds.map do |index|
+        stage_indexes_of_created_processables.map do |index|
           process_stage(index)
         end
 
@@ -27,7 +27,7 @@ module Ci
       return if HasStatus::BLOCKED_STATUS.include?(current_status)
 
       if HasStatus::COMPLETED_STATUSES.include?(current_status)
-        created_builds_in_stage(index).select do |build|
+        created_processables_in_stage(index).select do |build|
           Gitlab::OptimisticLocking.retry_lock(build) do |subject|
             Ci::ProcessBuildService.new(project, @user)
               .execute(build, current_status)
@@ -43,19 +43,19 @@ module Ci
     # rubocop: enable CodeReuse/ActiveRecord
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def stage_indexes_of_created_builds
-      created_builds.order(:stage_idx).pluck('distinct stage_idx')
+    def stage_indexes_of_created_processables
+      created_processables.order(:stage_idx).pluck('distinct stage_idx')
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def created_builds_in_stage(index)
-      created_builds.where(stage_idx: index)
+    def created_processables_in_stage(index)
+      created_processables.where(stage_idx: index)
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    def created_builds
-      pipeline.builds.created
+    def created_processables
+      pipeline.processables.created
     end
 
     # This method is for compatibility and data consistency and should be removed with 9.3 version of GitLab

lib/gitlab/ci/config/entry/job.rb

@@ -67,7 +67,7 @@ module Gitlab
 
           entry :only, Entry::Policy,
             description: 'Refs policy this job will be executed for.',
-            default: { refs: %w[branches tags] }
+            default: Entry::Policy::DEFAULT_ONLY
 
           entry :except, Entry::Policy,
             description: 'Refs policy this job will be executed for.'

lib/gitlab/ci/config/entry/jobs.rb

@@ -28,11 +28,15 @@ module Gitlab
             name.to_s.start_with?('.')
           end
 
+          def node_type(name)
+            hidden?(name) ? Entry::Hidden : Entry::Job
+          end
+
           # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
               @config.each do |name, config|
-                node = hidden?(name) ? Entry::Hidden : Entry::Job
+                node = node_type(name)
 
                 factory = ::Gitlab::Config::Entry::Factory.new(node)
                   .value(config || {})

lib/gitlab/ci/config/entry/policy.rb

@@ -11,6 +11,8 @@ module Gitlab
           strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) }
           strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) }
 
+          DEFAULT_ONLY = { refs: %w[branches tags] }.freeze
+
           class RefsPolicy < ::Gitlab::Config::Entry::Node
             include ::Gitlab::Config::Entry::Validatable
 

lib/gitlab/ci/pipeline/seed/build.rb

@@ -38,8 +38,15 @@ module Gitlab
             )
           end
 
+          def bridge?
+            @attributes.to_h.dig(:options, :trigger).present?
+          end
+
           def to_resource
             strong_memoize(:resource) do
+              if bridge?
+                ::Ci::Bridge.new(attributes)
+              else
                 ::Ci::Build.new(attributes)
               end
             end
@@ -47,4 +54,5 @@ module Gitlab
         end
       end
     end
+  end
 end

lib/gitlab/ci/pipeline/seed/stage.rb

@@ -39,7 +39,13 @@ module Gitlab
           def to_resource
             strong_memoize(:stage) do
               ::Ci::Stage.new(attributes).tap do |stage|
-                seeds.each { |seed| stage.builds << seed.to_resource }
+                seeds.each do |seed|
+                  if seed.bridge?
+                    stage.bridges << seed.to_resource
+                  else
+                    stage.builds << seed.to_resource
+                  end
+                end
               end
             end
           end

lib/gitlab/ci/status/bridge/common.rb

@@ -18,7 +18,6 @@ module Gitlab
           end
 
           def details_path
-            raise NotImplementedError
           end
         end
       end

lib/gitlab/ci/yaml_processor.rb

@@ -33,7 +33,7 @@ module Gitlab
 
         { stage_idx: @stages.index(job[:stage]),
           stage: job[:stage],
-          tag_list: job[:tags] || [],
+          tag_list: job[:tags],
           name: job[:name].to_s,
           allow_failure: job[:ignore],
           when: job[:when] || 'on_success',
@@ -53,8 +53,9 @@ module Gitlab
             retry: job[:retry],
             parallel: job[:parallel],
             instance: job[:instance],
-            start_in: job[:start_in]
-          }.compact }
+            start_in: job[:start_in],
+            trigger: job[:trigger]
+          }.compact }.compact
       end
 
       def stage_builds_attributes(stage)

spec/factories/ci/bridge.rb

@@ -10,8 +10,16 @@ FactoryBot.define do
 
     pipeline factory: :ci_pipeline
 
+    transient { downstream nil }
+
     after(:build) do |bridge, evaluator|
       bridge.project ||= bridge.pipeline.project
+
+      if evaluator.downstream.present?
+        bridge.options = bridge.options.to_h.merge(
+          trigger: { project: evaluator.downstream.full_path }
+        )
+      end
     end
   end
 end

spec/features/projects/pipelines/pipeline_spec.rb

@@ -286,6 +286,49 @@ describe 'Pipeline', :js do
     end
   end
 
+  context 'when a bridge job exists' do
+    include_context 'pipeline builds'
+
+    let(:project) { create(:project, :repository) }
+    let(:downstream) { create(:project, :repository) }
+
+    let(:pipeline) do
+      create(:ci_pipeline, project: project,
+                           ref: 'master',
+                           sha: project.commit.id,
+                           user: user)
+    end
+
+    let!(:bridge) do
+      create(:ci_bridge, pipeline: pipeline,
+                         name: 'cross-build',
+                         user: user,
+                         downstream: downstream)
+    end
+
+    describe 'GET /:project/pipelines/:id' do
+      before do
+        visit project_pipeline_path(project, pipeline)
+      end
+
+      it 'shows the pipeline with a bridge job' do
+        expect(page).to have_selector('.pipeline-visualization')
+        expect(page).to have_content('cross-build')
+      end
+    end
+
+    describe 'GET /:project/pipelines/:id/builds' do
+      before do
+        visit builds_project_pipeline_path(project, pipeline)
+      end
+
+      it 'shows a bridge job on a list' do
+        expect(page).to have_content('cross-build')
+        expect(page).to have_content(bridge.id)
+      end
+    end
+  end
+
   describe 'GET /:project/pipelines/:id/builds' do
     include_context 'pipeline builds'
 

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

@@ -163,14 +163,14 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
         ->(pipeline) { pipeline.variables.create!(key: 'VAR', value: '123') }
       end
 
-      it 'raises exception' do
+      it 'wastes pipeline iid' do
         expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved)
-      end
 
-      it 'wastes pipeline iid' do
-        expect { step.perform! }.to raise_error
+        last_iid = InternalId.ci_pipelines
+          .where(project_id: project.id)
+          .last.last_value
 
-        expect(InternalId.ci_pipelines.where(project_id: project.id).last.last_value).to be > 0
+        expect(last_iid).to be > 0
       end
     end
   end

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -5,8 +5,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
   let(:pipeline) { create(:ci_empty_pipeline, project: project) }
 
   let(:attributes) do
-    { name: 'rspec',
-      ref: 'master' }
+    { name: 'rspec', ref: 'master' }
   end
 
   subject do
@@ -21,11 +20,46 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
     end
   end
 
+  describe '#bridge?' do
+    context 'when job is a bridge' do
+      let(:attributes) do
+        { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } }
+      end
+
+      it { is_expected.to be_bridge }
+    end
+
+    context 'when trigger definition is empty' do
+      let(:attributes) do
+        { name: 'rspec', ref: 'master', options: { trigger: '' } }
+      end
+
+      it { is_expected.not_to be_bridge }
+    end
+
+    context 'when job is not a bridge' do
+      it { is_expected.not_to be_bridge }
+    end
+  end
+
   describe '#to_resource' do
+    context 'when job is not a bridge' do
       it 'returns a valid build resource' do
         expect(subject.to_resource).to be_a(::Ci::Build)
         expect(subject.to_resource).to be_valid
       end
+    end
+
+    context 'when job is a bridge' do
+      let(:attributes) do
+        { name: 'rspec', ref: 'master', options: { trigger: 'my/project' } }
+      end
+
+      it 'returns a valid bridge resource' do
+        expect(subject.to_resource).to be_a(::Ci::Bridge)
+        expect(subject.to_resource).to be_valid
+      end
+    end
 
     it 'memoizes a resource object' do
       build = subject.to_resource

spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb

@@ -62,8 +62,18 @@ describe Gitlab::Ci::Pipeline::Seed::Stage do
       expect(subject.seeds.map(&:attributes)).to all(include(ref: 'master'))
       expect(subject.seeds.map(&:attributes)).to all(include(tag: false))
       expect(subject.seeds.map(&:attributes)).to all(include(project: pipeline.project))
+    end
+
+    context 'when a legacy trigger exists' do
+      before do
+        create(:ci_trigger_request, pipeline: pipeline)
+      end
+
+      it 'returns build seeds including legacy trigger' do
+        expect(pipeline.legacy_trigger).not_to be_nil
         expect(subject.seeds.map(&:attributes))
-        .to all(include(trigger_request: pipeline.trigger_requests.first))
+          .to all(include(trigger_request: pipeline.legacy_trigger))
+      end
     end
 
     context 'when a ref is protected' do

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -21,15 +21,12 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "rspec",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 before_script: ["pwd"],
                 script: ["rspec"]
               },
               allow_failure: false,
               when: "on_success",
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -154,12 +151,9 @@ module Gitlab
              builds:
                [{ stage_idx: 1,
                   stage: "test",
-                  tag_list: [],
                   name: "rspec",
                   allow_failure: false,
                   when: "on_success",
-                  environment: nil,
-                  coverage_regex: nil,
                   yaml_variables: [],
                   options: { script: ["rspec"] },
                   only: { refs: ["branches"] },
@@ -169,12 +163,9 @@ module Gitlab
              builds:
                [{ stage_idx: 2,
                   stage: "deploy",
-                  tag_list: [],
                   name: "prod",
                   allow_failure: false,
                   when: "on_success",
-                  environment: nil,
-                  coverage_regex: nil,
                   yaml_variables: [],
                   options: { script: ["cap prod"] },
                   only: { refs: ["tags"] },
@@ -344,8 +335,6 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "rspec",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 before_script: ["pwd"],
                 script: ["rspec"],
@@ -356,7 +345,6 @@ module Gitlab
               },
               allow_failure: false,
               when: "on_success",
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -378,8 +366,6 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "rspec",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 before_script: ["pwd"],
                 script: ["rspec"],
@@ -390,7 +376,6 @@ module Gitlab
               },
               allow_failure: false,
               when: "on_success",
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -410,8 +395,6 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "rspec",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 before_script: ["pwd"],
                 script: ["rspec"],
@@ -420,7 +403,6 @@ module Gitlab
               },
               allow_failure: false,
               when: "on_success",
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -438,8 +420,6 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "rspec",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 before_script: ["pwd"],
                 script: ["rspec"],
@@ -448,7 +428,6 @@ module Gitlab
               },
               allow_failure: false,
               when: "on_success",
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -763,8 +742,6 @@ module Gitlab
             stage: "test",
             stage_idx: 1,
             name: "rspec",
-            coverage_regex: nil,
-            tag_list: [],
             options: {
               before_script: ["pwd"],
               script: ["rspec"],
@@ -779,7 +756,6 @@ module Gitlab
             },
             when: "on_success",
             allow_failure: false,
-            environment: nil,
             yaml_variables: []
           })
         end
@@ -976,14 +952,11 @@ module Gitlab
               stage: "test",
               stage_idx: 1,
               name: "normal_job",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 script: ["test"]
               },
               when: "on_success",
               allow_failure: false,
-              environment: nil,
               yaml_variables: []
             })
           end
@@ -1023,28 +996,22 @@ module Gitlab
               stage: "build",
               stage_idx: 0,
               name: "job1",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 script: ["execute-script-for-job"]
               },
               when: "on_success",
               allow_failure: false,
-              environment: nil,
               yaml_variables: []
             })
             expect(subject.second).to eq({
               stage: "build",
               stage_idx: 0,
               name: "job2",
-              coverage_regex: nil,
-              tag_list: [],
               options: {
                 script: ["execute-script-for-job"]
               },
               when: "on_success",
               allow_failure: false,
-              environment: nil,
               yaml_variables: []
             })
           end

spec/lib/gitlab/import_export/all_models.yml

@@ -114,6 +114,7 @@ ci_pipelines:
 - stages
 - statuses
 - builds
+- processables
 - trigger_requests
 - variables
 - auto_canceled_by
@@ -137,6 +138,7 @@ stages:
 - pipeline
 - statuses
 - builds
+- bridges
 statuses:
 - project
 - pipeline

spec/models/ci/pipeline_spec.rb

@@ -39,6 +39,29 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '.processables' do
+    before do
+      create(:ci_build, name: 'build', pipeline: pipeline)
+      create(:ci_bridge, name: 'bridge', pipeline: pipeline)
+      create(:commit_status, name: 'commit status', pipeline: pipeline)
+      create(:generic_commit_status, name: 'generic status', pipeline: pipeline)
+    end
+
+    it 'has an association with processable CI/CD entities' do
+      pipeline.processables.pluck('name').yield_self do |processables|
+        expect(processables).to match_array %w[build bridge]
+      end
+    end
+
+    it 'makes it possible to append a new processable' do
+      pipeline.processables << build(:ci_bridge)
+
+      pipeline.save!
+
+      expect(pipeline.processables.reload.count).to eq 3
+    end
+  end
+
   describe '.sort_by_merge_request_pipelines' do
     subject { described_class.sort_by_merge_request_pipelines }