Commit c340258b authored by Shinya Maeda's avatar Shinya Maeda

Merge branch 'move-dependencies-to-processable' into 'master'

Extract Ci::Processable::Dependencies class

See merge request gitlab-org/gitlab!27326
parents 0553d098 402898e2
......@@ -513,14 +513,6 @@ module Ci
success? && !deployment.try(:last?)
end
def depends_on_builds
# Get builds of the same type
latest_builds = self.pipeline.builds.latest
# Return builds from previous stages
latest_builds.where('stage_idx < ?', stage_idx)
end
def triggered_by?(current_user)
user == current_user
end
......@@ -825,41 +817,15 @@ module Ci
end
def all_dependencies
(dependencies + cross_dependencies).uniq
end
def dependencies
return [] if empty_dependencies?
depended_jobs = depends_on_builds
# find all jobs that are needed
if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && scheduling_type_dag?
depended_jobs = depended_jobs.where(name: needs.artifacts.select(:name))
end
# find all jobs that are dependent on
if options[:dependencies].present?
depended_jobs = depended_jobs.where(name: options[:dependencies])
end
# if both needs and dependencies are used,
# the end result will be an intersection between them
depended_jobs
end
def cross_dependencies
[]
end
def empty_dependencies?
options[:dependencies]&.empty?
dependencies.all
end
def has_valid_build_dependencies?
return true if Feature.enabled?('ci_disable_validates_dependencies')
dependencies.valid?
end
dependencies.all?(&:valid_dependency?)
def invalid_dependencies
dependencies.invalid_local
end
def valid_dependency?
......@@ -869,10 +835,6 @@ module Ci
true
end
def invalid_dependencies
dependencies.reject(&:valid_dependency?)
end
def runner_required_feature_names
strong_memoize(:runner_required_feature_names) do
RUNNER_FEATURES.select do |feature, method|
......@@ -950,6 +912,12 @@ module Ci
private
def dependencies
strong_memoize(:dependencies) do
Ci::Processable::Dependencies.new(self)
end
end
def build_data
@build_data ||= Gitlab::DataBuilder::Build.build(self)
end
......
# frozen_string_literal: true
module Ci
class Processable
class Dependencies
attr_reader :processable
def initialize(processable)
@processable = processable
end
def all
(local + cross_pipeline).uniq
end
# Dependencies local to the given pipeline
def local
return [] if no_local_dependencies_specified?
deps = model_class.where(pipeline_id: processable.pipeline_id).latest
deps = from_previous_stages(deps)
deps = from_needs(deps)
deps = from_dependencies(deps)
deps
end
# Dependencies that are defined in other pipelines
def cross_pipeline
[]
end
def invalid_local
local.reject(&:valid_dependency?)
end
def valid?
valid_local? && valid_cross_pipeline?
end
private
# Dependencies can only be of Ci::Build type because only builds
# can create artifacts
def model_class
::Ci::Build
end
def valid_local?
return true if Feature.enabled?('ci_disable_validates_dependencies')
local.all?(&:valid_dependency?)
end
def valid_cross_pipeline?
true
end
def project
processable.project
end
def no_local_dependencies_specified?
processable.options[:dependencies]&.empty?
end
def from_previous_stages(scope)
scope.before_stage(processable.stage_idx)
end
def from_needs(scope)
return scope unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
return scope unless processable.scheduling_type_dag?
needs_names = processable.needs.artifacts.select(:name)
scope.where(name: needs_names)
end
def from_dependencies(scope)
return scope unless processable.options[:dependencies].present?
scope.where(name: processable.options[:dependencies])
end
end
end
end
Ci::Processable::Dependencies.prepend_if_ee('EE::Ci::Processable::Dependencies')
......@@ -119,45 +119,8 @@ module EE
!merge_train_pipeline? && super
end
override :cross_dependencies
def cross_dependencies
return [] unless user_id
return [] unless project.feature_available?(:cross_project_pipelines)
cross_dependencies_relationship
.preload(project: [:project_feature])
.select { |job| user.can?(:read_build, job) }
end
private
def cross_dependencies_relationship
deps = Array(options[:cross_dependencies])
return ::Ci::Build.none unless deps.any?
relationship_fragments = build_cross_dependencies_fragments(deps, ::Ci::Build.latest.success)
return ::Ci::Build.none unless relationship_fragments.any?
::Ci::Build
.from_union(relationship_fragments)
.limit(::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT)
end
def build_cross_dependencies_fragments(deps, search_scope)
deps.inject([]) do |fragments, dep|
next fragments unless dep[:artifacts]
fragments << build_cross_dependency_relationship_fragment(dep, search_scope)
end
end
def build_cross_dependency_relationship_fragment(dependency, search_scope)
args = dependency.values_at(:job, :ref, :project)
dep_id = search_scope.max_build_id_by(*args)
::Ci::Build.id_in(dep_id)
end
def parse_security_artifact_blob(security_report, blob)
report_clone = security_report.clone_as_blank
::Gitlab::Ci::Parsers.fabricate!(security_report.type).parse!(blob, report_clone)
......
# frozen_string_literal: true
module EE
module Ci
module Processable
# this is a placeholder module where to nest
# EE extensions for Processable components
end
end
end
# frozen_string_literal: true
module EE
module Ci
module Processable
module Dependencies
extend ActiveSupport::Concern
extend ::Gitlab::Utils::Override
include ::Gitlab::Utils::StrongMemoize
LIMIT = ::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT
override :cross_pipeline
def cross_pipeline
strong_memoize(:cross_pipeline) do
fetch_cross_pipeline
end
end
private
override :valid_cross_pipeline?
def valid_cross_pipeline?
cross_pipeline.size == specified_cross_pipeline_dependencies.size
end
def fetch_cross_pipeline
return [] unless processable.user_id
return [] unless project.feature_available?(:cross_project_pipelines)
cross_dependencies_relationship
.preload(project: [:project_feature])
.select { |job| user.can?(:read_build, job) }
end
def cross_dependencies_relationship
deps = specified_cross_pipeline_dependencies
return model_class.none unless deps.any?
relationship_fragments = build_cross_dependencies_fragments(deps, model_class.latest.success)
return model_class.none unless relationship_fragments.any?
model_class.from_union(relationship_fragments).limit(LIMIT)
end
def build_cross_dependencies_fragments(deps, search_scope)
deps.inject([]) do |fragments, dep|
next fragments unless dep[:artifacts]
fragments << build_cross_dependency_relationship_fragment(dep, search_scope)
end
end
def build_cross_dependency_relationship_fragment(dependency, search_scope)
args = dependency.values_at(:job, :ref, :project)
dep_id = search_scope.max_build_id_by(*args)
model_class.id_in(dep_id)
end
def user
processable.user
end
def specified_cross_pipeline_dependencies
Array(processable.options[:cross_dependencies])
end
end
end
end
end
......@@ -383,209 +383,4 @@ describe Ci::Build do
expect(described_class.license_scan).to contain_exactly(build_with_license_scan)
end
end
describe '#cross_dependencies' do
let(:user) { create(:user) }
let(:dependencies) { }
let!(:final) do
create(:ci_build,
pipeline: pipeline, name: 'final',
stage_idx: 3, stage: 'deploy', user: user, options: {
cross_dependencies: dependencies
}
)
end
subject { final.cross_dependencies }
before do
project.add_developer(user)
pipeline.update!(user: user)
stub_licensed_features(cross_project_pipelines: true)
end
context 'when cross_dependencies are not defined' do
it { is_expected.to be_empty }
end
context 'with missing dependency' do
let(:dependencies) do
[
{
project: 'some/project',
job: 'some/job',
ref: 'some/ref',
artifacts: true
}
]
end
it { is_expected.to be_empty }
end
context 'with cross_dependencies to the same pipeline' do
let!(:dependency) do
create(:ci_build, :success,
pipeline: pipeline, name: 'dependency',
stage_idx: 1, stage: 'build', user: user
)
end
let(:dependencies) do
[
{
project: project.full_path,
job: 'dependency',
ref: pipeline.ref,
artifacts: artifacts
}
]
end
context 'with artifacts true' do
let(:artifacts) { true }
it { is_expected.to match(a_collection_containing_exactly(dependency)) }
end
context 'with artifacts false' do
let(:artifacts) { false }
it { is_expected.to be_empty }
end
end
context 'with cross_dependencies to other pipeline' do
let(:feature_pipeline) do
create(:ci_pipeline, project: project,
sha: project.commit.id,
ref: 'feature',
status: 'success')
end
let(:dependencies) do
[
{
project: project.full_path,
job: 'dependency',
ref: feature_pipeline.ref,
artifacts: true
}
]
end
let!(:dependency) do
create(:ci_build, :success,
pipeline: feature_pipeline, ref: feature_pipeline.ref,
name: 'dependency', stage_idx: 4, stage: 'deploy', user: user
)
end
it { is_expected.to match(a_collection_containing_exactly(dependency)) }
end
context 'with cross_dependencies to two pipelines' do
let(:other_project) { create(:project, :repository, group: group) }
let(:other_pipeline) do
create(:ci_pipeline, project: other_project,
sha: other_project.commit.id,
ref: other_project.default_branch,
status: 'success',
user: user)
end
let(:feature_pipeline) do
create(:ci_pipeline, project: project,
sha: project.commit.id,
ref: 'feature',
status: 'success')
end
let(:dependencies) do
[
{
project: other_project.full_path,
job: 'other_dependency',
ref: other_pipeline.ref,
artifacts: true
},
{
project: project.full_path,
job: 'dependency',
ref: feature_pipeline.ref,
artifacts: true
}
]
end
let!(:other_dependency) do
create(:ci_build, :success,
pipeline: other_pipeline, ref: other_pipeline.ref,
name: 'other_dependency', stage_idx: 4, stage: 'deploy', user: user
)
end
let!(:dependency) do
create(:ci_build, :success,
pipeline: feature_pipeline, ref: feature_pipeline.ref,
name: 'dependency', stage_idx: 4, stage: 'deploy', user: user
)
end
context 'with permissions to other_project' do
before do
other_project.add_developer(user)
end
it 'contains both dependencies' do
is_expected.to match(
a_collection_containing_exactly(dependency, other_dependency))
end
context 'when license does not have cross_project_pipelines' do
before do
stub_licensed_features(cross_project_pipelines: false)
end
it { is_expected.to be_empty }
end
end
context 'without permissions to other_project' do
it { is_expected.to match(a_collection_containing_exactly(dependency)) }
end
end
context 'with too many cross_dependencies' do
let(:cross_dependencies_limit) do
::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT
end
before do
cross_dependencies_limit.next.times do |index|
create(:ci_build, :success,
pipeline: pipeline, name: "dependency-#{index}",
stage_idx: 1, stage: 'build', user: user
)
end
end
let(:dependencies) do
Array.new(cross_dependencies_limit.next) do |index|
{
project: project.full_path,
job: "dependency-#{index}",
ref: pipeline.ref,
artifacts: true
}
end
end
it 'has a limit' do
expect(subject.size).to eq(cross_dependencies_limit)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Ci::Processable::Dependencies do
describe '#cross_pipeline' do
let_it_be(:user) { create(:user) }
let_it_be(:project, refind: true) { create(:project, :repository) }
let(:dependencies) { }
let(:pipeline) do
create(:ci_pipeline,
project: project,
sha: project.commit.id,
ref: project.default_branch,
status: 'success')
end
let!(:job) do
create(:ci_build,
pipeline: pipeline,
name: 'final',
stage_idx: 3,
stage: 'deploy',
user: user,
options: { cross_dependencies: dependencies })
end
subject { described_class.new(job).cross_pipeline }
before do
project.add_developer(user)
pipeline.update!(user: user)
stub_licensed_features(cross_project_pipelines: true)
end
context 'when cross_dependencies are not defined' do
it { is_expected.to be_empty }
end
context 'with missing dependency' do
let(:dependencies) do
[
{
project: 'some/project',
job: 'some/job',
ref: 'some/ref',
artifacts: true
}
]
end
it { is_expected.to be_empty }
end
context 'with cross_dependencies to the same pipeline' do
let!(:dependency) do
create(:ci_build, :success,
pipeline: pipeline,
name: 'dependency',
stage_idx: 1,
stage: 'build',
user: user
)
end
let(:dependencies) do
[
{
project: project.full_path,
job: 'dependency',
ref: pipeline.ref,
artifacts: artifacts
}
]
end
context 'with artifacts true' do
let(:artifacts) { true }
it { is_expected.to contain_exactly(dependency) }
end
context 'with artifacts false' do
let(:artifacts) { false }
it { is_expected.to be_empty }
end
end
context 'with cross_dependencies to another pipeline in same project' do
let(:another_pipeline) do
create(:ci_pipeline,
project: project,
sha: project.commit.id,
ref: 'feature',
status: 'success')
end
let(:dependencies) do
[
{
project: project.full_path,
job: 'dependency',
ref: another_pipeline.ref,
artifacts: true
}
]
end
let!(:dependency) do
create(:ci_build, :success,
pipeline: another_pipeline,
ref: another_pipeline.ref,
name: 'dependency',
stage_idx: 4,
stage: 'deploy',
user: user
)
end
it { is_expected.to contain_exactly(dependency) }
end
context 'with cross_dependencies to a pipeline in another project' do
let(:other_project) { create(:project, :repository) }
let(:other_pipeline) do
create(:ci_pipeline,
project: other_project,
sha: other_project.commit.id,
ref: other_project.default_branch,
status: 'success',
user: user)
end
let(:feature_pipeline) do
create(:ci_pipeline,
project: project,
sha: project.commit.id,
ref: 'feature',
status: 'success')
end
let(:dependencies) do
[
{
project: other_project.full_path,
job: 'other_dependency',
ref: other_pipeline.ref,
artifacts: true
},
{
project: project.full_path,
job: 'dependency',
ref: feature_pipeline.ref,
artifacts: true
}
]
end
let!(:other_dependency) do
create(:ci_build, :success,
pipeline: other_pipeline,
ref: other_pipeline.ref,
name: 'other_dependency',
stage_idx: 4,
stage: 'deploy',
user: user)
end
let!(:dependency) do
create(:ci_build, :success,
pipeline: feature_pipeline,
ref: feature_pipeline.ref,
name: 'dependency',
stage_idx: 4,
stage: 'deploy',
user: user)
end
context 'with permissions to other_project' do
before do
other_project.add_developer(user)
end
it 'contains both dependencies' do
is_expected.to contain_exactly(dependency, other_dependency)
end
context 'when license does not have cross_project_pipelines' do
before do
stub_licensed_features(cross_project_pipelines: false)
end
it { expect(subject).to be_empty }
end
end
context 'without permissions to other_project' do
it { is_expected.to contain_exactly(dependency) }
end
end
context 'with too many cross_dependencies' do
let(:cross_dependencies_limit) do
::Gitlab::Ci::Config::Entry::Needs::NEEDS_CROSS_DEPENDENCIES_LIMIT
end
before do
cross_dependencies_limit.next.times do |index|
create(:ci_build, :success,
pipeline: pipeline, name: "dependency-#{index}",
stage_idx: 1, stage: 'build', user: user
)
end
end
let(:dependencies) do
Array.new(cross_dependencies_limit.next) do |index|
{
project: project.full_path,
job: "dependency-#{index}",
ref: pipeline.ref,
artifacts: true
}
end
end
it 'returns a limited number of dependencies' do
expect(subject.size).to eq(cross_dependencies_limit)
end
end
end
end
......@@ -730,147 +730,6 @@ describe Ci::Build do
end
end
describe '#depends_on_builds' do
let!(:build) { create(:ci_build, pipeline: pipeline, name: 'build', stage_idx: 0, stage: 'build') }
let!(:rspec_test) { create(:ci_build, pipeline: pipeline, name: 'rspec', stage_idx: 1, stage: 'test') }
let!(:rubocop_test) { create(:ci_build, pipeline: pipeline, name: 'rubocop', stage_idx: 1, stage: 'test') }
let!(:staging) { create(:ci_build, pipeline: pipeline, name: 'staging', stage_idx: 2, stage: 'deploy') }
it 'expects to have no dependents if this is first build' do
expect(build.depends_on_builds).to be_empty
end
it 'expects to have one dependent if this is test' do
expect(rspec_test.depends_on_builds.map(&:id)).to contain_exactly(build.id)
end
it 'expects to have all builds from build and test stage if this is last' do
expect(staging.depends_on_builds.map(&:id)).to contain_exactly(build.id, rspec_test.id, rubocop_test.id)
end
it 'expects to have retried builds instead the original ones' do
project.add_developer(user)
retried_rspec = described_class.retry(rspec_test, user)
expect(staging.depends_on_builds.map(&:id))
.to contain_exactly(build.id, retried_rspec.id, rubocop_test.id)
end
describe '#dependencies' do
let(:dependencies) { }
let(:needs) { }
let!(:final) do
scheduling_type = needs.present? ? :dag : :stage
create(:ci_build,
pipeline: pipeline, name: 'final', scheduling_type: scheduling_type,
stage_idx: 3, stage: 'deploy', options: {
dependencies: dependencies
}
)
end
before do
needs.to_a.each do |need|
create(:ci_build_need, build: final, **need)
end
end
subject { final.dependencies }
context 'when dependencies are defined' do
let(:dependencies) { %w(rspec staging) }
it { is_expected.to contain_exactly(rspec_test, staging) }
end
context 'when needs are defined' do
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: true },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(build, rspec_test, staging) }
context 'when ci_dag_support is disabled' do
before do
stub_feature_flags(ci_dag_support: false)
end
it { is_expected.to contain_exactly(build, rspec_test, rubocop_test, staging) }
end
end
context 'when need artifacts are defined' do
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: false },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(build, staging) }
end
context 'when needs and dependencies are defined' do
let(:dependencies) { %w(rspec staging) }
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: true },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(rspec_test, staging) }
end
context 'when needs and dependencies contradict' do
let(:dependencies) { %w(rspec staging) }
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: false },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(staging) }
end
context 'when nor dependencies or needs are defined' do
it { is_expected.to contain_exactly(build, rspec_test, rubocop_test, staging) }
end
end
describe '#all_dependencies' do
let!(:final_build) do
create(:ci_build,
pipeline: pipeline, name: 'deploy',
stage_idx: 3, stage: 'deploy'
)
end
subject { final_build.all_dependencies }
it 'returns dependencies and cross_dependencies' do
dependencies = [1, 2, 3]
cross_dependencies = [3, 4]
allow(final_build).to receive(:dependencies).and_return(dependencies)
allow(final_build).to receive(:cross_dependencies).and_return(cross_dependencies)
is_expected.to match(a_collection_containing_exactly(1, 2, 3, 4))
end
end
end
describe '#triggered_by?' do
subject { build.triggered_by?(user) }
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::Processable::Dependencies do
let_it_be(:user) { create(:user) }
let_it_be(:project, reload: true) { create(:project, :repository) }
let_it_be(:pipeline, reload: true) do
create(:ci_pipeline, project: project,
sha: project.commit.id,
ref: project.default_branch,
status: 'success')
end
let!(:build) { create(:ci_build, pipeline: pipeline, name: 'build', stage_idx: 0, stage: 'build') }
let!(:rspec_test) { create(:ci_build, pipeline: pipeline, name: 'rspec', stage_idx: 1, stage: 'test') }
let!(:rubocop_test) { create(:ci_build, pipeline: pipeline, name: 'rubocop', stage_idx: 1, stage: 'test') }
let!(:staging) { create(:ci_build, pipeline: pipeline, name: 'staging', stage_idx: 2, stage: 'deploy') }
describe '#local' do
subject { described_class.new(job).local }
describe 'jobs from previous stages' do
context 'when job is in the first stage' do
let(:job) { build }
it { is_expected.to be_empty }
end
context 'when job is in the second stage' do
let(:job) { rspec_test }
it 'contains all jobs from the first stage' do
is_expected.to contain_exactly(build)
end
end
context 'when job is in the last stage' do
let(:job) { staging }
it 'contains all jobs from all previous stages' do
is_expected.to contain_exactly(build, rspec_test, rubocop_test)
end
context 'when a job is retried' do
before do
project.add_developer(user)
end
let(:retried_job) { Ci::Build.retry(rspec_test, user) }
it 'contains the retried job instead of the original one' do
is_expected.to contain_exactly(build, retried_job, rubocop_test)
end
end
end
end
describe 'jobs from specified dependencies' do
let(:dependencies) { }
let(:needs) { }
let!(:job) do
scheduling_type = needs.present? ? :dag : :stage
create(:ci_build,
pipeline: pipeline,
name: 'final',
scheduling_type: scheduling_type,
stage_idx: 3,
stage: 'deploy',
options: { dependencies: dependencies }
)
end
before do
needs.to_a.each do |need|
create(:ci_build_need, build: job, **need)
end
end
context 'when dependencies are defined' do
let(:dependencies) { %w(rspec staging) }
it { is_expected.to contain_exactly(rspec_test, staging) }
end
context 'when needs are defined' do
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: true },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(build, rspec_test, staging) }
context 'when ci_dag_support is disabled' do
before do
stub_feature_flags(ci_dag_support: false)
end
it { is_expected.to contain_exactly(build, rspec_test, rubocop_test, staging) }
end
end
context 'when need artifacts are defined' do
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: false },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(build, staging) }
end
context 'when needs and dependencies are defined' do
let(:dependencies) { %w(rspec staging) }
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: true },
{ name: 'staging', artifacts: true }
]
end
it { is_expected.to contain_exactly(rspec_test, staging) }
end
context 'when needs and dependencies contradict' do
let(:dependencies) { %w(rspec staging) }
let(:needs) do
[
{ name: 'build', artifacts: true },
{ name: 'rspec', artifacts: false },
{ name: 'staging', artifacts: true }
]
end
it 'returns only the intersection' do
is_expected.to contain_exactly(staging)
end
end
context 'when nor dependencies or needs are defined' do
it 'returns the jobs from previous stages' do
is_expected.to contain_exactly(build, rspec_test, rubocop_test, staging)
end
end
end
end
describe '#all' do
let!(:job) do
create(:ci_build, pipeline: pipeline, name: 'deploy', stage_idx: 3, stage: 'deploy')
end
let(:dependencies) { described_class.new(job) }
subject { dependencies.all }
it 'returns the union of all local dependencies and any cross pipeline dependencies' do
expect(dependencies).to receive(:local).and_return([1, 2, 3])
expect(dependencies).to receive(:cross_pipeline).and_return([3, 4])
expect(subject).to contain_exactly(1, 2, 3, 4)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment