Initial implementation for cross project artifacts

Add changes to start a discussion.

app/models/ci/build.rb

@@ -762,6 +762,10 @@ module Ci
       Gitlab::Ci::Build::Credentials::Factory.new(self).create!
     end
 
+    def all_dependencies
+      (dependencies + cross_dependencies).uniq
+    end
+
     def dependencies
       return [] if empty_dependencies?
 
@@ -782,6 +786,10 @@ module Ci
       depended_jobs
     end
 
+    def cross_dependencies
+      []
+    end
+
     def empty_dependencies?
       options[:dependencies]&.empty?
     end

app/models/commit_status.rb

@@ -47,6 +47,12 @@ class CommitStatus < ApplicationRecord
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
   scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) }
   scope :for_ids, -> (ids) { where(id: ids) }
+  scope :for_ref, -> (ref) { where(ref: ref) }
+  scope :by_name, -> (name) { where(name: name) }
+
+  scope :for_project_paths, -> (paths) do
+    where(project: Project.where_full_path_in(Array(paths)))
+  end
 
   scope :with_preloads, -> do
     preload(:project, :user)

db/migrate/20191205094702_add_index_for_cross_projects_dependencies_to_ci_builds.rb

+# frozen_string_literal: true
+
+class AddIndexForCrossProjectsDependenciesToCiBuilds < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_builds, [:project_id, :name, :ref],
+      where: "type = 'Ci::Build' AND status = 'success' AND (retried = FALSE OR retried IS NULL)"
+  end
+
+  def down
+    remove_concurrent_index :ci_builds, [:project_id, :name, :ref],
+      where: "type = 'Ci::Build' AND status = 'success' AND (retried = FALSE OR retried IS NULL)"
+  end
+end

db/schema.rb

@@ -693,6 +693,7 @@ ActiveRecord::Schema.define(version: 2019_12_16_183532) do
     t.index ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref"
     t.index ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))"
     t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id"
+    t.index ["project_id", "name", "ref"], name: "index_ci_builds_on_project_id_and_name_and_ref", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = 'success'::text) AND ((retried = false) OR (retried IS NULL)))"
     t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))"
     t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))"
     t.index ["protected"], name: "index_ci_builds_on_protected"

ee/app/models/ee/ci/build.rb

@@ -8,6 +8,7 @@ module EE
     # and be included in the `Build` model
     module Build
       extend ActiveSupport::Concern
+      extend ::Gitlab::Utils::Override
 
       LICENSED_PARSER_FEATURES = {
         sast: :sast,
@@ -18,11 +19,18 @@ module EE
 
       prepended do
         include UsageStatistics
+        include FromUnion
 
         after_save :stick_build_if_status_changed
         delegate :service_specification, to: :runner_session, allow_nil: true
 
         scope :license_scan, -> { joins(:job_artifacts).merge(::Ci::JobArtifact.license_management) }
+        scope :max_build_id_by, -> (build_name, ref, project_path) do
+          select('max(ci_builds.id) as id')
+            .by_name(build_name)
+            .for_ref(ref)
+            .for_project_paths(project_path)
+        end
       end
 
       def shared_runners_minutes_limit_enabled?
@@ -109,8 +117,46 @@ module EE
         !merge_train_pipeline? && super
       end
 
+      override :cross_dependencies
+      def cross_dependencies
+        return [] unless user_id
+        return [] unless ::Feature.enabled?(:cross_project_need_artifacts, project, default_enabled: false)
+        return [] unless project.feature_available?(:cross_project_pipelines)
+
+        cross_dependencies_relationship
+          .preload(project: [:project_feature])
+          .select { |job| user.can?(:read_build, job) }
+      end
+
       private
 
+      def cross_dependencies_relationship
+        deps = Array(options[:cross_dependencies])
+        return ::Ci::Build.none unless deps.any?
+
+        relationship_fragments = build_cross_dependencies_fragments(deps, ::Ci::Build.latest.success)
+        return ::Ci::Build.none unless relationship_fragments.any?
+
+        ::Ci::Build
+          .from_union(relationship_fragments)
+          .limit(::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT)
+      end
+
+      def build_cross_dependencies_fragments(deps, search_scope)
+        deps.inject([]) do |fragments, dep|
+          next fragments unless dep[:artifacts]
+
+          fragments << build_cross_dependency_relationship_fragment(dep, search_scope)
+        end
+      end
+
+      def build_cross_dependency_relationship_fragment(dependency, search_scope)
+        args = dependency.values_at(:job, :ref, :project)
+        dep_id = search_scope.max_build_id_by(*args)
+
+        ::Ci::Build.id_in(dep_id)
+      end
+
       def name_in?(names)
         name.in?(Array(names))
       end

ee/changelogs/unreleased/cross-projects-artifacts-dependencies.yml

+---
+title: Create a new database composite index to support cross-project artifacts downloads
+merge_request: 20721
+author:
+type: added

ee/lib/ee/gitlab/ci/config/entry/need.rb

@@ -11,7 +11,11 @@ module EE
             prepended do
               strategy :BridgeHash,
                 class: EE::Gitlab::Ci::Config::Entry::Need::BridgeHash,
-                if: -> (config) { config.is_a?(Hash) && !config.key?(:job) }
+                if: -> (config) { config.is_a?(Hash) && !config.key?(:job) && !config.key?(:project) }
+
+              strategy :CrossDependency,
+                class: EE::Gitlab::Ci::Config::Entry::Need::CrossDependency,
+                if: -> (config) { config.is_a?(Hash) && (config.key?(:project) || config.key?(:ref)) }
             end
 
             class BridgeHash < ::Gitlab::Config::Entry::Node
@@ -31,6 +35,31 @@ module EE
                 :bridge
               end
             end
+
+            class CrossDependency < ::Gitlab::Config::Entry::Node
+              include ::Gitlab::Config::Entry::Validatable
+              include ::Gitlab::Config::Entry::Attributable
+
+              ALLOWED_KEYS = %i[project ref job artifacts].freeze
+              attributes :project, :ref, :job, :artifacts
+
+              validations do
+                validates :config, presence: true
+                validates :config, allowed_keys: ALLOWED_KEYS
+                validates :project,  type: String, presence: true
+                validates :ref,      type: String, presence: true
+                validates :job,      type: String, presence: true
+                validates :artifacts, boolean: true, allow_nil: true
+              end
+
+              def type
+                :cross_dependency
+              end
+
+              def value
+                super.merge(artifacts: artifacts || artifacts.nil?)
+              end
+            end
           end
         end
       end

ee/lib/ee/gitlab/ci/config/entry/needs.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Config
+        module Entry
+          module Needs
+            extend ActiveSupport::Concern
+
+            NEEDS_CROSS_DEPENDENCIES_LIMIT = 5
+
+            prepended do
+              validations do
+                validate on: :composed do
+                  cross_dependencies = value[:cross_dependency].to_a
+                  if cross_dependencies.size > NEEDS_CROSS_DEPENDENCIES_LIMIT
+                    errors.add(:config, "must be less than or equal to #{NEEDS_CROSS_DEPENDENCIES_LIMIT}")
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/ci/config/entry/bridge_spec.rb

@@ -241,6 +241,31 @@ describe EE::Gitlab::Ci::Config::Entry::Bridge do
       end
     end
 
+    context 'when bridge has only cross projects dependencies' do
+      let(:config) do
+        {
+          needs: [
+            {
+              project: 'some/project',
+              job: 'some/job',
+              ref: 'some/ref',
+              artifacts: true
+            }
+          ]
+        }
+      end
+
+      describe '#valid?' do
+        it { is_expected.not_to be_valid }
+      end
+
+      describe '#errors' do
+        it 'returns an error about cross dependencies' do
+          expect(subject.errors).to include('needs config uses invalid types: cross_dependency')
+        end
+      end
+    end
+
     context 'when bridge has bridge and job needs' do
       let(:config) do
         {
@@ -254,6 +279,33 @@ describe EE::Gitlab::Ci::Config::Entry::Bridge do
       end
     end
 
+    context 'when bridge has bridge and cross projects dependencies ' do
+      let(:config) do
+        {
+          trigger: 'other-project',
+          needs: [
+            { pipeline: 'some/other_project' },
+            {
+              project: 'some/project',
+              job: 'some/job',
+              ref: 'some/ref',
+              artifacts: true
+            }
+          ]
+        }
+      end
+
+      describe '#valid?' do
+        it { is_expected.not_to be_valid }
+      end
+
+      describe '#errors' do
+        it 'returns an error cross dependencies' do
+          expect(subject.errors).to contain_exactly('needs config uses invalid types: cross_dependency')
+        end
+      end
+    end
+
     context 'when bridge has more than one valid bridge needs' do
       let(:config) do
         {

ee/spec/lib/ee/gitlab/ci/config/entry/need_spec.rb

@@ -3,34 +3,131 @@
 require 'spec_helper'
 
 describe ::Gitlab::Ci::Config::Entry::Need do
-  subject { described_class.new(config) }
+  subject(:need) { described_class.new(config) }
 
-  context 'when upstream is specified' do
-    let(:config) { { pipeline: 'some/project' } }
+  context 'with Bridge config' do
+    context 'when upstream is specified' do
+      let(:config) { { pipeline: 'some/project' } }
 
-    describe '#valid?' do
-      it { is_expected.to be_valid }
+      describe '#valid?' do
+        it { is_expected.to be_valid }
+      end
+
+      describe '#value' do
+        it 'returns job needs configuration' do
+          expect(subject.value).to eq(pipeline: 'some/project')
+        end
+      end
     end
 
-    describe '#value' do
-      it 'returns job needs configuration' do
-        expect(subject.value).to eq(pipeline: 'some/project')
+    context 'when need is empty' do
+      let(:config) { {} }
+
+      describe '#valid?' do
+        it { is_expected.not_to be_valid }
+      end
+
+      describe '#errors' do
+        it 'is returns an error about an empty config' do
+          expect(subject.errors)
+            .to include("bridge hash config can't be blank")
+        end
       end
     end
   end
 
-  context 'when need is empty' do
-    let(:config) { {} }
+  context 'with CrossDependency config' do
+    describe '#artifacts' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:artifacts,      :value, :validity) do
+        { artifacts: true }  | true  | true
+        { artifacts: false } | false | true
+        { artifacts: nil }   | true  | true
+        {}                   | true  | true
+        { artifacts: 1 }     | 1     | false
+        { artifacts: 'str' } | 'str' | false
+      end
+
+      with_them do
+        let(:config) do
+          {
+            project: 'some/project',
+            job: 'some/job',
+            ref: 'some/ref'
+          }.merge(artifacts)
+        end
+
+        describe '#valid?' do
+          it { expect(subject.valid?).to eq(validity) }
+        end
+
+        describe '#value' do
+          it 'returns job needs configuration' do
+            expect(subject.value)
+              .to eq(artifacts: value, job: 'some/job',
+                project: 'some/project', ref: 'some/ref')
+          end
+        end
 
-    describe '#valid?' do
-      it { is_expected.not_to be_valid }
+        describe '#type' do
+          it { expect(subject.type).to eq(:cross_dependency) }
+        end
+      end
     end
 
-    describe '#errors' do
-      it 'is returns an error about an empty config' do
-        expect(subject.errors)
-          .to include("bridge hash config can't be blank")
+    shared_examples 'required string attribute' do |attribute|
+      describe "##{attribute}" do
+        using RSpec::Parameterized::TableSyntax
+
+        let(:general_config) do
+          {
+            job: 'some/job',
+            ref: 'some/ref',
+            project: 'some/project',
+            artifacts: true
+          }.tap { |config| config.delete(attribute) }
+        end
+
+        where(:value, :validity, :error) do
+          {}                           | false | "can't be blank"
+          { attribute => nil }         | false | "can't be blank"
+          { attribute => 'something' } | true  | ''
+          { attribute => :symbol }     | false | 'should be a string'
+          { attribute => 1 }           | false | 'should be a string'
+        end
+
+        with_them do
+          let(:config) { general_config.merge(value).freeze }
+
+          describe '#valid?' do
+            it { expect(subject.valid?).to eq(validity) }
+          end
+
+          describe '#value' do
+            it 'returns needs configuration' do
+              expect(subject.value).to eq(config)
+            end
+          end
+
+          describe '#type' do
+            it { expect(subject.type).to eq(:cross_dependency) }
+          end
+
+          describe '#errors' do
+            subject(:errors) { need.errors }
+
+            let(:error_message) { "cross dependency #{attribute} #{error}" }
+
+            it { is_expected.to(be_empty)               if validity }
+            it { is_expected.to(include(error_message)) unless validity }
+          end
+        end
       end
     end
+
+    it_behaves_like 'required string attribute', :project
+    it_behaves_like 'required string attribute', :job
+    it_behaves_like 'required string attribute', :ref
   end
 end

ee/spec/lib/ee/gitlab/ci/config/entry/needs_spec.rb

@@ -6,7 +6,7 @@ describe ::Gitlab::Ci::Config::Entry::Needs do
   subject(:needs) { described_class.new(config) }
 
   before do
-    needs.metadata[:allowed_needs] = %i[job bridge]
+    needs.metadata[:allowed_needs] = %i[job bridge cross_dependency]
   end
 
   describe 'validations' do
@@ -44,6 +44,32 @@ describe ::Gitlab::Ci::Config::Entry::Needs do
         it { is_expected.not_to be_valid }
       end
     end
+
+    context 'with too many cross dependencies' do
+      let(:limit) { described_class::NEEDS_CROSS_DEPENDENCIES_LIMIT }
+
+      let(:config) do
+        Array.new(limit.next) do |index|
+          {
+            project: "project-#{index}",
+            job: 'job-1',
+            ref: 'master',
+            artifacts: true
+          }
+        end
+      end
+
+      describe '#valid?' do
+        it { is_expected.not_to be_valid }
+      end
+
+      describe '#errors' do
+        it 'returns error about incorrect type' do
+          expect(needs.errors).to contain_exactly(
+            "needs config must be less than or equal to #{limit}")
+        end
+      end
+    end
   end
 
   describe '.compose!' do
@@ -52,7 +78,8 @@ describe ::Gitlab::Ci::Config::Entry::Needs do
         [
           'first_job_name',
           { job: 'second_job_name', artifacts: false },
-          { pipeline: 'some/project' }
+          { pipeline: 'some/project' },
+          { project: 'some/project', job: 'some/job', ref: 'some/ref', artifacts: true }
         ]
       end
 
@@ -71,14 +98,22 @@ describe ::Gitlab::Ci::Config::Entry::Needs do
               { name: 'first_job_name',  artifacts: true },
               { name: 'second_job_name', artifacts: false }
             ],
-            bridge: [{ pipeline: 'some/project' }]
+            bridge: [{ pipeline: 'some/project' }],
+            cross_dependency: [
+              {
+                project: 'some/project',
+                job: 'some/job',
+                ref: 'some/ref',
+                artifacts: true
+              }
+            ]
           )
         end
       end
 
       describe '#descendants' do
         it 'creates valid descendant nodes' do
-          expect(needs.descendants.count).to eq(3)
+          expect(needs.descendants.count).to eq(4)
           expect(needs.descendants)
             .to all(be_an_instance_of(::Gitlab::Ci::Config::Entry::Need))
         end

ee/spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -79,5 +79,124 @@ describe Gitlab::Ci::YamlProcessor do
         )
       end
     end
+
+    context 'needs cross projects artifacts' do
+      let(:config) do
+        {
+          build: { stage: 'build', script: 'test' },
+          test1: { stage: 'test', script: 'test', needs: needs },
+          test2: { stage: 'test', script: 'test' }
+        }
+      end
+
+      let(:needs) do
+        [
+          { job: 'build' },
+          {
+            project: 'some/project',
+            ref: 'some/ref',
+            job: 'build2',
+            artifacts: true
+          },
+          {
+            project: 'some/other/project',
+            ref: 'some/ref',
+            job: 'build3',
+            artifacts: false
+          },
+          {
+            project: 'project',
+            ref: 'master',
+            job: 'build4'
+          }
+        ]
+      end
+
+      it 'creates jobs with valid specification' do
+        expect(subject.builds.size).to eq(3)
+
+        expect(subject.builds[1]).to eq(
+          stage: 'test',
+          stage_idx: 2,
+          name: 'test1',
+          options: {
+            script: ['test'],
+            cross_dependencies: [
+              {
+                artifacts: true,
+                job: 'build2',
+                project: 'some/project',
+                ref: 'some/ref'
+              },
+              {
+                artifacts: false,
+                job: 'build3',
+                project: 'some/other/project',
+                ref: 'some/ref'
+              },
+              {
+                artifacts: true,
+                job: 'build4',
+                project: 'project',
+                ref: 'master'
+              }
+            ]
+          },
+          needs_attributes: [
+            { name: 'build', artifacts: true }
+          ],
+          only: { refs: %w[branches tags] },
+          when: 'on_success',
+          allow_failure: false,
+          yaml_variables: []
+        )
+      end
+    end
+
+    context 'needs cross projects artifacts and pipelines' do
+      let(:needs) do
+        [
+          {
+            project: 'some/project',
+            ref: 'some/ref',
+            job: 'build',
+            artifacts: true
+          },
+          {
+            pipeline: 'other/project'
+          }
+        ]
+      end
+
+      it 'returns errors' do
+        expect { subject }
+          .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
+                          'jobs:bridge config should contain either a trigger or a needs:pipeline')
+      end
+    end
+
+    context 'with invalid needs cross projects artifacts' do
+      let(:config) do
+        {
+          build: { stage: 'build', script: 'test' },
+          test: {
+            stage: 'test',
+            script: 'test',
+            needs: {
+              project: 'some/project',
+              ref: 1,
+              job: 'build',
+              artifacts: true
+            }
+          }
+        }
+      end
+
+      it 'returns errors' do
+        expect { subject }
+          .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
+                          'jobs:test:needs:need ref should be a string')
+      end
+    end
   end
 end

ee/spec/models/ci/build_spec.rb

@@ -361,4 +361,217 @@ describe Ci::Build do
       expect(described_class.license_scan).to contain_exactly(build_with_license_scan)
     end
   end
+
+  describe '#cross_dependencies' do
+    let(:user) { create(:user) }
+    let(:dependencies) { }
+
+    let!(:final) do
+      create(:ci_build,
+        pipeline: pipeline, name: 'final',
+        stage_idx: 3, stage: 'deploy', user: user, options: {
+          cross_dependencies: dependencies
+        }
+      )
+    end
+
+    subject { final.cross_dependencies }
+
+    before do
+      project.add_developer(user)
+      pipeline.update!(user: user)
+      stub_licensed_features(cross_project_pipelines: true)
+    end
+
+    context 'when cross_dependencies are not defined' do
+      it { is_expected.to be_empty }
+    end
+
+    context 'with missing dependency' do
+      let(:dependencies) do
+        [
+          {
+            project: 'some/project',
+            job: 'some/job',
+            ref: 'some/ref',
+            artifacts: true
+          }
+        ]
+      end
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'with cross_dependencies to the same pipeline' do
+      let!(:dependency) do
+        create(:ci_build, :success,
+          pipeline: pipeline, name: 'dependency',
+          stage_idx: 1, stage: 'build', user: user
+        )
+      end
+
+      let(:dependencies) do
+        [
+          {
+            project: project.full_path,
+            job: 'dependency',
+            ref: pipeline.ref,
+            artifacts: artifacts
+          }
+        ]
+      end
+
+      context 'with artifacts true' do
+        let(:artifacts) { true }
+
+        it { is_expected.to match(a_collection_containing_exactly(dependency)) }
+      end
+
+      context 'with artifacts false' do
+        let(:artifacts) { false }
+
+        it { is_expected.to be_empty }
+      end
+    end
+
+    context 'with cross_dependencies to other pipeline' do
+      let(:feature_pipeline) do
+        create(:ci_pipeline, project: project,
+                             sha: project.commit.id,
+                             ref: 'feature',
+                             status: 'success')
+      end
+
+      let(:dependencies) do
+        [
+          {
+            project: project.full_path,
+            job: 'dependency',
+            ref: feature_pipeline.ref,
+            artifacts: true
+          }
+        ]
+      end
+
+      let!(:dependency) do
+        create(:ci_build, :success,
+          pipeline: feature_pipeline, ref: feature_pipeline.ref,
+          name: 'dependency', stage_idx: 4, stage: 'deploy', user: user
+        )
+      end
+
+      it { is_expected.to match(a_collection_containing_exactly(dependency)) }
+    end
+
+    context 'with cross_dependencies to two pipelines' do
+      let(:other_project) { create(:project, :repository, group: group) }
+
+      let(:other_pipeline) do
+        create(:ci_pipeline, project: other_project,
+                             sha: other_project.commit.id,
+                             ref: other_project.default_branch,
+                             status: 'success',
+                             user: user)
+      end
+
+      let(:feature_pipeline) do
+        create(:ci_pipeline, project: project,
+                             sha: project.commit.id,
+                             ref: 'feature',
+                             status: 'success')
+      end
+
+      let(:dependencies) do
+        [
+          {
+            project: other_project.full_path,
+            job: 'other_dependency',
+            ref: other_pipeline.ref,
+            artifacts: true
+          },
+          {
+            project: project.full_path,
+            job: 'dependency',
+            ref: feature_pipeline.ref,
+            artifacts: true
+          }
+        ]
+      end
+
+      let!(:other_dependency) do
+        create(:ci_build, :success,
+          pipeline: other_pipeline, ref: other_pipeline.ref,
+          name: 'other_dependency', stage_idx: 4, stage: 'deploy', user: user
+        )
+      end
+
+      let!(:dependency) do
+        create(:ci_build, :success,
+          pipeline: feature_pipeline, ref: feature_pipeline.ref,
+          name: 'dependency', stage_idx: 4, stage: 'deploy', user: user
+        )
+      end
+
+      context 'with permissions to other_project' do
+        before do
+          other_project.add_developer(user)
+        end
+
+        it 'contains both dependencies' do
+          is_expected.to match(
+            a_collection_containing_exactly(dependency, other_dependency))
+        end
+
+        context 'when license does not have cross_project_pipelines' do
+          before do
+            stub_licensed_features(cross_project_pipelines: false)
+          end
+
+          it { is_expected.to be_empty }
+        end
+
+        context 'when feature is disabled' do
+          before do
+            stub_feature_flags(cross_project_need_artifacts: false)
+          end
+
+          it { is_expected.to be_empty }
+        end
+      end
+
+      context 'without permissions to other_project' do
+        it { is_expected.to match(a_collection_containing_exactly(dependency)) }
+      end
+    end
+
+    context 'with too many cross_dependencies' do
+      let(:cross_dependencies_limit) do
+        ::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT
+      end
+
+      before do
+        cross_dependencies_limit.next.times do |index|
+          create(:ci_build, :success,
+            pipeline: pipeline, name: "dependency-#{index}",
+            stage_idx: 1, stage: 'build', user: user
+          )
+        end
+      end
+
+      let(:dependencies) do
+        Array.new(cross_dependencies_limit.next) do |index|
+          {
+            project: project.full_path,
+            job: "dependency-#{index}",
+            ref: pipeline.ref,
+            artifacts: true
+          }
+        end
+      end
+
+      it 'has a limit' do
+        expect(subject.size).to eq(cross_dependencies_limit)
+      end
+    end
+  end
 end

ee/spec/services/ci/create_pipeline_service/cross_needs_artifacts_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  subject(:execute) { service.execute(:push) }
+
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:admin) }
+
+  let(:service) do
+    described_class.new(project, user, { ref: 'refs/heads/master' })
+  end
+
+  before do
+    stub_ci_pipeline_yaml_file(YAML.dump(config))
+  end
+
+  shared_examples 'supported cross project artifacts definitions' do
+    let(:config) do
+      {
+        build_job: {
+          stage: 'build',
+          needs: needs,
+          script: ['make']
+        }
+      }
+    end
+
+    let(:needs) do
+      [
+        { project: 'project-1', job: 'job-1', ref: 'ref-1', artifacts: true },
+        { project: 'project-2', job: 'job-2', ref: 'ref-2', artifacts: false },
+        { project: 'project-3', job: 'job-3', ref: 'ref-3', artifacts: nil },
+        { project: 'project-4', job: 'job-4', ref: 'ref-4' }
+      ]
+    end
+
+    let(:build_job) { subject.builds.find_by!(name: :build_job) }
+
+    it 'persists pipeline' do
+      is_expected.to be_persisted
+    end
+
+    it 'persists job' do
+      expect { execute }.to change(Ci::Build, :count).by(1)
+    end
+
+    it 'persists cross_dependencies' do
+      deps = build_job.options['cross_dependencies']
+      result = [
+        { job: "job-1", ref: "ref-1", project: "project-1", artifacts: true },
+        { job: "job-2", ref: "ref-2", project: "project-2", artifacts: false },
+        { job: "job-3", ref: "ref-3", project: "project-3", artifacts: true },
+        { job: "job-4", ref: "ref-4", project: "project-4", artifacts: true }
+      ]
+
+      expect(deps).to match(result)
+    end
+
+    it 'returns empty dependencies with non existing projects' do
+      expect(build_job.all_dependencies).to be_empty
+    end
+  end
+
+  shared_examples 'mixed artifacts definitions' do
+    let(:other_project) { create(:project, :repository) }
+
+    let(:other_pipeline) do
+      create(:ci_pipeline, project: other_project,
+        sha: other_project.commit.id,
+        ref: other_project.default_branch,
+        status: 'success',
+        user: user)
+    end
+
+    let!(:dependency) do
+      create(:ci_build, :success,
+        pipeline: other_pipeline, ref: other_pipeline.ref,
+        name: 'dependency', stage_idx: 3, stage: 'deploy', user: user
+      )
+    end
+
+    let(:config) do
+      {
+        build_job_1: {
+          stage: 'build',
+          script: ['make']
+        },
+        build_job_2: {
+          stage: 'build',
+          script: ['make']
+        },
+        test_job: {
+          stage: 'test',
+          needs: needs,
+          script: ['make']
+        }
+      }
+    end
+
+    let(:needs) do
+      [
+        'build_job_1',
+        { job: 'build_job_2', artifacts: false },
+        {
+          project: other_project.full_path,
+          job: dependency.name,
+          ref: other_pipeline.ref,
+          artifacts: true
+        }
+      ]
+    end
+
+    let(:dependencies_when_license_is_available) do
+      %w[dependency] + dependencies_when_license_is_not_available
+    end
+
+    let(:dependencies_when_license_is_not_available) do
+      %w[build_job_1]
+    end
+
+    let(:test_job) { subject.builds.find_by!(name: :test_job) }
+
+    it 'persists pipeline' do
+      is_expected.to be_persisted
+    end
+
+    it 'persists jobs' do
+      expect { execute }.to change(Ci::Build, :count).by(3)
+    end
+
+    it 'persists needs' do
+      expect { execute }.to change(Ci::BuildNeed, :count).by(2)
+
+      expect(test_job.needs.map(&:name)).to match(
+        a_collection_containing_exactly('build_job_1', 'build_job_2'))
+    end
+
+    it 'persists cross_dependencies' do
+      deps = test_job.options['cross_dependencies']
+      result = {
+        job: 'dependency',
+        ref: 'master',
+        project: other_project.full_path,
+        artifacts: true
+      }
+
+      expect(deps).to match(a_collection_containing_exactly(result))
+    end
+
+    it 'returns dependencies' do
+      names = test_job.all_dependencies.map(&:name)
+
+      expect(names).to match(
+        a_collection_containing_exactly(*expected_dependencies))
+    end
+  end
+
+  shared_examples 'broken artifacts definitions' do
+    let(:config) do
+      {
+        build_job: {
+          stage: 'build',
+          script: ['make'],
+          needs: [
+            { project: 'project-2', job: 'job', artifacts: true }
+          ]
+        }
+      }
+    end
+
+    it 'persists pipeline' do
+      is_expected.to be_persisted
+    end
+
+    it 'has errors' do
+      expect(subject.yaml_errors)
+        .to include('jobs:build_job:needs:need ref should be a string')
+    end
+  end
+
+  context 'with license' do
+    before do
+      stub_licensed_features(cross_project_pipelines: true)
+    end
+
+    it_behaves_like 'supported cross project artifacts definitions'
+    it_behaves_like 'broken artifacts definitions'
+    it_behaves_like 'mixed artifacts definitions' do
+      let(:expected_dependencies) { dependencies_when_license_is_available }
+    end
+  end
+
+  context 'without license' do
+    before do
+      stub_licensed_features(cross_project_pipelines: false)
+    end
+
+    it_behaves_like 'supported cross project artifacts definitions'
+    it_behaves_like 'broken artifacts definitions'
+    it_behaves_like 'mixed artifacts definitions' do
+      let(:expected_dependencies) { dependencies_when_license_is_not_available }
+    end
+  end
+end

lib/api/entities.rb

@@ -1672,7 +1672,7 @@ module API
         expose :artifacts, using: Artifacts
         expose :cache, using: Cache
         expose :credentials, using: Credentials
-        expose :dependencies, using: Dependency
+        expose :all_dependencies, as: :dependencies, using: Dependency
         expose :features
       end
     end

lib/gitlab/ci/config/entry/job.rb

@@ -134,7 +134,7 @@ module Gitlab
 
           entry :needs, Entry::Needs,
             description: 'Needs configuration for this job.',
-            metadata: { allowed_needs: %i[job] },
+            metadata: { allowed_needs: %i[job cross_dependency] },
             inherit: false
 
           entry :variables, Entry::Variables,

lib/gitlab/ci/config/entry/need.rb

@@ -6,7 +6,9 @@ module Gitlab
       module Entry
         class Need < ::Gitlab::Config::Entry::Simplifiable
           strategy :JobString, if: -> (config) { config.is_a?(String) }
-          strategy :JobHash, if: -> (config) { config.is_a?(Hash) && config.key?(:job) }
+
+          strategy :JobHash,
+            if: -> (config) { config.is_a?(Hash) && config.key?(:job) && !(config.key?(:project) || config.key?(:ref)) }
 
           class JobString < ::Gitlab::Config::Entry::Node
             include ::Gitlab::Config::Entry::Validatable

lib/gitlab/ci/config/entry/needs.rb

@@ -53,3 +53,5 @@ module Gitlab
     end
   end
 end
+
+::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs')

lib/gitlab/ci/yaml_processor.rb

@@ -69,6 +69,7 @@ module Gitlab
             services: job[:services],
             artifacts: job[:artifacts],
             dependencies: job[:dependencies],
+            cross_dependencies: job.dig(:needs, :cross_dependency),
             job_timeout: job[:timeout],
             before_script: job[:before_script],
             script: job[:script],

spec/models/ci/build_spec.rb

@@ -815,6 +815,27 @@ describe Ci::Build do
         it { is_expected.to contain_exactly(build, rspec_test, rubocop_test, staging) }
       end
     end
+
+    describe '#all_dependencies' do
+      let!(:final_build) do
+        create(:ci_build,
+          pipeline: pipeline, name: 'deploy',
+          stage_idx: 3, stage: 'deploy'
+        )
+      end
+
+      subject { final_build.all_dependencies }
+
+      it 'returns dependencies and cross_dependencies' do
+        dependencies = [1, 2, 3]
+        cross_dependencies = [3, 4]
+
+        allow(final_build).to receive(:dependencies).and_return(dependencies)
+        allow(final_build).to receive(:cross_dependencies).and_return(cross_dependencies)
+
+        is_expected.to match(a_collection_containing_exactly(1, 2, 3, 4))
+      end
+    end
   end
 
   describe '#triggered_by?' do

spec/models/commit_status_spec.rb

@@ -312,6 +312,72 @@ describe CommitStatus do
     end
   end
 
+  describe '.for_ref' do
+    subject { described_class.for_ref('bb').order(:id) }
+
+    let(:statuses) do
+      [create_status(ref: 'aa'),
+       create_status(ref: 'bb'),
+       create_status(ref: 'cc')]
+    end
+
+    it 'returns statuses with the specified ref' do
+      is_expected.to eq(statuses.values_at(1))
+    end
+  end
+
+  describe '.by_name' do
+    subject { described_class.by_name('bb').order(:id) }
+
+    let(:statuses) do
+      [create_status(name: 'aa'),
+       create_status(name: 'bb'),
+       create_status(name: 'cc')]
+    end
+
+    it 'returns statuses with the specified name' do
+      is_expected.to eq(statuses.values_at(1))
+    end
+  end
+
+  describe '.for_project_paths' do
+    subject do
+      described_class
+        .for_project_paths(paths)
+        .order(:id)
+    end
+
+    context 'with a single path' do
+      let(:other_project) { create(:project, :repository) }
+      let(:paths) { other_project.full_path }
+
+      let(:other_pipeline) do
+        create(:ci_pipeline, project: other_project, sha: other_project.commit.id)
+      end
+
+      let(:statuses) do
+        [create_status(pipeline: pipeline),
+         create_status(pipeline: other_pipeline)]
+      end
+
+      it 'returns statuses for other_project' do
+        is_expected.to eq(statuses.values_at(1))
+      end
+    end
+
+    context 'with array of paths' do
+      let(:paths) { [project.full_path] }
+
+      let(:statuses) do
+        [create_status(pipeline: pipeline)]
+      end
+
+      it 'returns statuses for project' do
+        is_expected.to eq(statuses.values_at(0))
+      end
+    end
+  end
+
   describe '.status' do
     context 'when there are multiple statuses present' do
       before do