Merge branch 'extract-ci-config-processor-4' into 'master'

Extract CI config processor into pipeline chain

See merge request gitlab-org/gitlab!19874

app/models/ci/build.rb

@@ -898,12 +898,6 @@ module Ci
         value.with_indifferent_access
       end
     end
-
-    def build_attributes_from_config
-      return {} unless pipeline.config_processor
-
-      pipeline.config_processor.build_attributes(name)
-    end
   end
 end
 

app/models/ci/pipeline.rb

@@ -551,23 +551,6 @@ module Ci
       end
     end
 
-    def stage_seeds
-      return [] unless config_processor
-
-      strong_memoize(:stage_seeds) do
-        seeds = config_processor.stages_attributes.inject([]) do |previous_stages, attributes|
-          seed = Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes, previous_stages)
-          previous_stages + [seed]
-        end
-
-        seeds.select(&:included?)
-      end
-    end
-
-    def seeds_size
-      stage_seeds.sum(&:size)
-    end
-
     def has_kubernetes_active?
       project.deployment_platform&.active?
     end
@@ -587,62 +570,14 @@ module Ci
       end
     end
 
-    def set_config_source
-      if ci_yaml_from_repo
-        self.config_source = :repository_source
-      elsif implied_ci_yaml_file
-        self.config_source = :auto_devops_source
-      end
-    end
-
-    ##
-    # TODO, setting yaml_errors should be moved to the pipeline creation chain.
-    #
-    def config_processor
-      return unless ci_yaml_file
-      return @config_processor if defined?(@config_processor)
-
-      @config_processor ||= begin
-        ::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha, user: user })
-      rescue Gitlab::Ci::YamlProcessor::ValidationError => e
-        self.yaml_errors = e.message
-        nil
-      rescue => ex
-        self.yaml_errors = "Undefined error (#{Labkit::Correlation::CorrelationId.current_id})"
-
-        Gitlab::Sentry.track_acceptable_exception(ex, extra: {
-          project_id: project.id,
-          sha: sha,
-          ci_yaml_file: ci_yaml_file_path
-        })
-        nil
-      end
-    end
-
-    def ci_yaml_file_path
+    # TODO: this logic is duplicate with Pipeline::Chain::Config::Content
+    # we should persist this is `ci_pipelines.config_path`
+    def config_path
       return unless repository_source? || unknown_source?
 
       project.ci_config_path.presence || '.gitlab-ci.yml'
     end
 
-    def ci_yaml_file
-      return @ci_yaml_file if defined?(@ci_yaml_file)
-
-      @ci_yaml_file =
-        if auto_devops_source?
-          implied_ci_yaml_file
-        else
-          ci_yaml_from_repo
-        end
-
-      if @ci_yaml_file
-        @ci_yaml_file
-      else
-        self.yaml_errors = "Failed to load CI/CD config file for #{sha}"
-        nil
-      end
-    end
-
     def has_yaml_errors?
       yaml_errors.present?
     end
@@ -711,7 +646,7 @@ module Ci
     def predefined_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.append(key: 'CI_PIPELINE_IID', value: iid.to_s)
-        variables.append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
+        variables.append(key: 'CI_CONFIG_PATH', value: config_path)
         variables.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
         variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s)
         variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s)
@@ -906,24 +841,6 @@ module Ci
 
     private
 
-    def ci_yaml_from_repo
-      return unless project
-      return unless sha
-      return unless ci_yaml_file_path
-
-      project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
-    rescue GRPC::NotFound, GRPC::Internal
-      nil
-    end
-
-    def implied_ci_yaml_file
-      return unless project
-
-      if project.auto_devops_enabled?
-        Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
-      end
-    end
-
     def pipeline_data
       Gitlab::DataBuilder::Pipeline.build(self)
     end

app/services/ci/create_pipeline_service.rb

@@ -7,12 +7,14 @@ module Ci
     CreateError = Class.new(StandardError)
 
     SEQUENCE = [Gitlab::Ci::Pipeline::Chain::Build,
-                Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
                 Gitlab::Ci::Pipeline::Chain::Validate::Abilities,
                 Gitlab::Ci::Pipeline::Chain::Validate::Repository,
-                Gitlab::Ci::Pipeline::Chain::Validate::Config,
+                Gitlab::Ci::Pipeline::Chain::Config::Content,
+                Gitlab::Ci::Pipeline::Chain::Config::Process,
+                Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
                 Gitlab::Ci::Pipeline::Chain::Skip,
                 Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules,
+                Gitlab::Ci::Pipeline::Chain::Seed,
                 Gitlab::Ci::Pipeline::Chain::Limit::Size,
                 Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::Create,

app/views/projects/pipelines/_with_tabs.html.haml

@@ -39,10 +39,6 @@
               %th
           = render partial: "projects/stage/stage", collection: pipeline.legacy_stages, as: :stage
 
-    - elsif pipeline.project.builds_enabled? && !pipeline.ci_yaml_file
-      .bs-callout.bs-callout-warning
-        = _("%{gitlab_ci_yml} not found in this commit") % { gitlab_ci_yml: ".gitlab-ci.yml" }
-
   - if @pipeline.failed_builds.present?
     #js-tab-failures.build-failures.tab-pane.build-page
       %table.table.responsive-table.ci-table.responsive-table-sm-rounded

ee/lib/ee/gitlab/ci/pipeline/chain/limit/size.rb

@@ -17,7 +17,7 @@ module EE
                 super
 
                 @limit = Pipeline::Quota::Size
-                  .new(project.namespace, pipeline)
+                  .new(project.namespace, pipeline, command)
               end
 
               override :perform!

ee/lib/ee/gitlab/ci/pipeline/quota/size.rb

@@ -9,9 +9,10 @@ module EE
             include ::Gitlab::Utils::StrongMemoize
             include ActionView::Helpers::TextHelper
 
-            def initialize(namespace, pipeline)
+            def initialize(namespace, pipeline, command)
               @namespace = namespace
               @pipeline = pipeline
+              @command = command
             end
 
             def enabled?
@@ -34,7 +35,7 @@ module EE
             private
 
             def excessive_seeds_count
-              @excessive ||= @pipeline.seeds_size - ci_pipeline_size_limit
+              @excessive ||= seeds_size - ci_pipeline_size_limit
             end
 
             def ci_pipeline_size_limit
@@ -42,6 +43,10 @@ module EE
                 @namespace.actual_limits.ci_pipeline_size
               end
             end
+
+            def seeds_size
+              @command.stage_seeds.sum(&:size) # rubocop: disable CodeReuse/ActiveRecord
+            end
           end
         end
       end

ee/spec/lib/ee/gitlab/ci/pipeline/quota/size_spec.rb

@@ -11,7 +11,12 @@ describe EE::Gitlab::Ci::Pipeline::Quota::Size do
 
   let(:pipeline) { build_stubbed(:ci_pipeline, project: project) }
 
-  subject { described_class.new(namespace, pipeline) }
+  let(:command) do
+    double(:command,
+      stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
+  end
+
+  subject { described_class.new(namespace, pipeline, command) }
 
   shared_context 'pipeline size limit exceeded' do
     before do

ee/spec/lib/gitlab/ci/pipeline/chain/limit/size_spec.rb

@@ -10,8 +10,10 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
   let(:pipeline) { build(:ci_pipeline, project: project) }
 
   let(:command) do
-    double('command', project: project,
-                      current_user: user)
+    double(:command,
+      project: project,
+      current_user: user,
+      stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
   end
 
   let(:step) { described_class.new(pipeline, command) }
@@ -31,9 +33,11 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
 
     context 'when saving incomplete pipelines' do
       let(:command) do
-        double('command', project: project,
+        double(:command,
+          project: project,
           current_user: user,
-                          save_incompleted: true)
+          save_incompleted: true,
+          stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
       end
 
       it 'drops the pipeline' do
@@ -79,9 +83,11 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
 
     context 'when not saving incomplete pipelines' do
       let(:command) do
-        double('command', project: project,
+        double(:command,
+          project: project,
           current_user: user,
-                          save_incompleted: false)
+          save_incompleted: false,
+          stage_seeds: [double(:seed_1, size: 1), double(:seed_2, size: 1)])
       end
 
       it 'does not drop the pipeline' do

ee/spec/models/ci/pipeline_spec.rb

@@ -471,27 +471,6 @@ describe Ci::Pipeline do
     end
   end
 
-  describe '#ci_yaml_file_path' do
-    subject { pipeline.ci_yaml_file_path }
-
-    context 'the source is the repository' do
-      let(:implied_yml) { Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content }
-
-      before do
-        pipeline.repository_source!
-      end
-
-      it 'returns the configuration if found' do
-        allow(pipeline.project.repository).to receive(:gitlab_ci_yml_for)
-          .and_return('config')
-
-        expect(pipeline.ci_yaml_file).to be_a(String)
-        expect(pipeline.ci_yaml_file).not_to eq(implied_yml)
-        expect(pipeline.yaml_errors).to be_nil
-      end
-    end
-  end
-
   describe '#latest_merge_request_pipeline?' do
     subject { pipeline.latest_merge_request_pipeline? }
 

ee/spec/services/ci/create_pipeline_service_spec.rb

@@ -64,7 +64,6 @@ describe Ci::CreatePipelineService, '#execute' do
 
         expect(pipeline).to be_persisted
         expect(pipeline).to be_failed
-        expect(pipeline.seeds_size).to be > 2
         expect(pipeline.statuses).to be_empty
         expect(pipeline.size_limit_exceeded?).to be true
       end

lib/gitlab/ci/pipeline/chain/base.rb

@@ -5,7 +5,7 @@ module Gitlab
     module Pipeline
       module Chain
         class Base
-          attr_reader :pipeline, :command
+          attr_reader :pipeline, :command, :config
 
           delegate :project, :current_user, to: :command
 

lib/gitlab/ci/pipeline/chain/build.rb

@@ -22,8 +22,6 @@ module Gitlab
               external_pull_request: @command.external_pull_request,
               variables_attributes: Array(@command.variables_attributes)
             )
-
-            @pipeline.set_config_source
           end
 
           def break?

lib/gitlab/ci/pipeline/chain/command.rb

@@ -10,7 +10,9 @@ module Gitlab
           :trigger_request, :schedule, :merge_request, :external_pull_request,
           :ignore_skip_ci, :save_incompleted,
           :seeds_block, :variables_attributes, :push_options,
-          :chat_data, :allow_mirror_update
+          :chat_data, :allow_mirror_update,
+          # These attributes are set by Chains during processing:
+          :config_content, :config_processor, :stage_seeds
         ) do
           include Gitlab::Utils::StrongMemoize
 

lib/gitlab/ci/pipeline/chain/config/content.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              return if @command.config_content
+
+              if content = content_from_repo
+                @command.config_content = content
+                @pipeline.config_source = :repository_source
+                # TODO: we should persist ci_config_path
+                # @pipeline.config_path = ci_config_path
+              elsif content = content_from_auto_devops
+                @command.config_content = content
+                @pipeline.config_source = :auto_devops_source
+              end
+
+              unless @command.config_content
+                return error("Missing #{ci_config_path} file")
+              end
+            end
+
+            def break?
+              @pipeline.errors.any? || @pipeline.persisted?
+            end
+
+            private
+
+            def content_from_repo
+              return unless project
+              return unless @pipeline.sha
+              return unless ci_config_path
+
+              project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
+            rescue GRPC::NotFound, GRPC::Internal
+              nil
+            end
+
+            def content_from_auto_devops
+              return unless project&.auto_devops_enabled?
+
+              Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
+            end
+
+            def ci_config_path
+              project.ci_config_path.presence || '.gitlab-ci.yml'
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/process.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Process < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              raise ArgumentError, 'missing config content' unless @command.config_content
+
+              @command.config_processor = ::Gitlab::Ci::YamlProcessor.new(
+                @command.config_content, {
+                  project: project,
+                  sha: @pipeline.sha,
+                  user: current_user
+                }
+              )
+            rescue Gitlab::Ci::YamlProcessor::ValidationError => ex
+              error(ex.message, config_error: true)
+            rescue => ex
+              Gitlab::Sentry.track_acceptable_exception(ex, extra: {
+                project_id: project.id,
+                sha: @pipeline.sha
+              })
+
+              error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})",
+                config_error: true)
+            end
+
+            def break?
+              @pipeline.errors.any? || @pipeline.persisted?
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb

@@ -41,7 +41,7 @@ module Gitlab
           end
 
           def workflow_config
-            @pipeline.config_processor.workflow_attributes || {}
+            @command.config_processor.workflow_attributes || {}
           end
         end
       end

lib/gitlab/ci/pipeline/chain/populate.rb

@@ -10,29 +10,12 @@ module Gitlab
           PopulateError = Class.new(StandardError)
 
           def perform!
-            # Allocate next IID. This operation must be outside of transactions of pipeline creations.
-            pipeline.ensure_project_iid!
-
-            # Protect the pipeline. This is assigned in Populate instead of
-            # Build to prevent erroring out on ambiguous refs.
-            pipeline.protected = @command.protected_ref?
-
-            ##
-            # Populate pipeline with block argument of CreatePipelineService#execute.
-            #
-            @command.seeds_block&.call(pipeline)
-
-            ##
-            # Gather all runtime build/stage errors
-            #
-            if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence
-              return error(seeds_errors.join("\n"), config_error: true)
-            end
+            raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds
 
             ##
             # Populate pipeline with all stages, and stages with builds.
             #
-            pipeline.stages = pipeline.stage_seeds.map(&:to_resource)
+            pipeline.stages = @command.stage_seeds.map(&:to_resource)
 
             if pipeline.stages.none?
               return error('No stages / jobs for this pipeline.')

lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb

@@ -6,11 +6,13 @@ module Gitlab
       module Chain
         class RemoveUnwantedChatJobs < Chain::Base
           def perform!
-            return unless pipeline.config_processor && pipeline.chat?
+            raise ArgumentError, 'missing config processor' unless @command.config_processor
+
+            return unless pipeline.chat?
 
             # When scheduling a chat pipeline we only want to run the build
             # that matches the chat command.
-            pipeline.config_processor.jobs.select! do |name, _|
+            @command.config_processor.jobs.select! do |name, _|
               name.to_s == command.chat_data[:command].to_s
             end
           end

lib/gitlab/ci/pipeline/chain/seed.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Seed < Chain::Base
+          include Chain::Helpers
+          include Gitlab::Utils::StrongMemoize
+
+          def perform!
+            raise ArgumentError, 'missing config processor' unless @command.config_processor
+
+            # Allocate next IID. This operation must be outside of transactions of pipeline creations.
+            pipeline.ensure_project_iid!
+
+            # Protect the pipeline. This is assigned in Populate instead of
+            # Build to prevent erroring out on ambiguous refs.
+            pipeline.protected = @command.protected_ref?
+
+            ##
+            # Populate pipeline with block argument of CreatePipelineService#execute.
+            #
+            @command.seeds_block&.call(pipeline)
+
+            ##
+            # Gather all runtime build/stage errors
+            #
+            if stage_seeds_errors
+              return error(stage_seeds_errors.join("\n"), config_error: true)
+            end
+
+            @command.stage_seeds = stage_seeds
+          end
+
+          def break?
+            pipeline.errors.any?
+          end
+
+          private
+
+          def stage_seeds_errors
+            stage_seeds.flat_map(&:errors).compact.presence
+          end
+
+          def stage_seeds
+            strong_memoize(:stage_seeds) do
+              seeds = stages_attributes.inject([]) do |previous_stages, attributes|
+                seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages)
+                previous_stages + [seed]
+              end
+
+              seeds.select(&:included?)
+            end
+          end
+
+          def stages_attributes
+            @command.config_processor.stages_attributes
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/validate/config.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Pipeline
-      module Chain
-        module Validate
-          class Config < Chain::Base
-            include Chain::Helpers
-
-            def perform!
-              unless @pipeline.config_processor
-                unless @pipeline.ci_yaml_file
-                  return error("Missing #{@pipeline.ci_yaml_file_path} file")
-                end
-
-                if @command.save_incompleted && @pipeline.has_yaml_errors?
-                  @pipeline.drop!(:config_error)
-                end
-
-                error(@pipeline.yaml_errors)
-              end
-            end
-
-            def break?
-              @pipeline.errors.any? || @pipeline.persisted?
-            end
-          end
-        end
-      end
-    end
-  end
-end

locale/gitlab.pot

@@ -261,9 +261,6 @@ msgstr ""
 msgid "%{from} to %{to}"
 msgstr ""
 
-msgid "%{gitlab_ci_yml} not found in this commit"
-msgstr ""
-
 msgid "%{group_docs_link_start}Groups%{group_docs_link_end} allow you to manage and collaborate across multiple projects. Members of a group have access to all of its projects."
 msgstr ""
 

spec/features/commits_spec.rb

@@ -157,39 +157,6 @@ describe 'Commits' do
         end
       end
     end
-
-    describe '.gitlab-ci.yml not found warning' do
-      before do
-        project.add_reporter(user)
-      end
-
-      context 'ci builds enabled' do
-        it 'does not show warning' do
-          visit pipeline_path(pipeline)
-
-          expect(page).not_to have_content '.gitlab-ci.yml not found in this commit'
-        end
-
-        it 'shows warning' do
-          stub_ci_pipeline_yaml_file(nil)
-
-          visit pipeline_path(pipeline)
-
-          expect(page).to have_content '.gitlab-ci.yml not found in this commit'
-        end
-      end
-
-      context 'ci builds disabled' do
-        it 'does not show warning' do
-          stub_ci_builds_disabled
-          stub_ci_pipeline_yaml_file(nil)
-
-          visit pipeline_path(pipeline)
-
-          expect(page).not_to have_content '.gitlab-ci.yml not found in this commit'
-        end
-      end
-    end
   end
 
   context 'viewing commits for a branch' do

spec/features/projects/commits/user_browses_commits_spec.rb

@@ -56,10 +56,6 @@ describe 'User browses commits' do
       project.enable_ci
 
       create(:ci_build, pipeline: pipeline)
-
-      allow_next_instance_of(Ci::Pipeline) do |instance|
-        allow(instance).to receive(:ci_yaml_file).and_return('')
-      end
     end
 
     it 'renders commit ci info' do

spec/lib/gitlab/ci/pipeline/chain/build_spec.rb

@@ -11,6 +11,7 @@ describe Gitlab::Ci::Pipeline::Chain::Build do
     [{ key: 'first', secret_value: 'world' },
      { key: 'second', secret_value: 'second_world' }]
   end
+
   let(:command) do
     Gitlab::Ci::Pipeline::Chain::Command.new(
       source: :push,
@@ -51,12 +52,6 @@ describe Gitlab::Ci::Pipeline::Chain::Build do
       .to eq variables_attributes.map(&:with_indifferent_access)
   end
 
-  it 'sets a valid config source' do
-    step.perform!
-
-    expect(pipeline.repository_source?).to be true
-  end
-
   it 'returns a valid pipeline' do
     step.perform!
 

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

@@ -18,19 +18,32 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       seeds_block: nil)
   end
 
+  let(:dependencies) do
+    [
+      Gitlab::Ci::Pipeline::Chain::Config::Content.new(pipeline, command),
+      Gitlab::Ci::Pipeline::Chain::Config::Process.new(pipeline, command),
+      Gitlab::Ci::Pipeline::Chain::Seed.new(pipeline, command)
+    ]
+  end
+
   let(:step) { described_class.new(pipeline, command) }
 
   let(:config) do
     { rspec: { script: 'rspec' } }
   end
 
+  def run_chain
+    dependencies.map(&:perform!)
+    step.perform!
+  end
+
   before do
     stub_ci_pipeline_yaml_file(YAML.dump(config))
   end
 
   context 'when pipeline doesn not have seeds block' do
     before do
-      step.perform!
+      run_chain
     end
 
     it 'does not persist the pipeline' do
@@ -66,7 +79,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
     end
 
     before do
-      step.perform!
+      run_chain
     end
 
     it 'breaks the chain' do
@@ -84,16 +97,16 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
   end
 
   describe 'pipeline protect' do
-    subject { step.perform! }
-
     context 'when ref is protected' do
       before do
         allow(project).to receive(:protected_for?).with('master').and_return(true)
         allow(project).to receive(:protected_for?).with('refs/heads/master').and_return(true)
+
+        dependencies.map(&:perform!)
       end
 
       it 'does not protect the pipeline' do
-        subject
+        run_chain
 
         expect(pipeline.protected).to eq(true)
       end
@@ -101,7 +114,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
 
     context 'when ref is not protected' do
       it 'does not protect the pipeline' do
-        subject
+        run_chain
 
         expect(pipeline.protected).to eq(false)
       end
@@ -114,7 +127,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
     end
 
     before do
-      step.perform!
+      run_chain
     end
 
     it 'breaks the chain' do
@@ -146,7 +159,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       end
 
       it 'populates pipeline with resources described in the seeds block' do
-        step.perform!
+        run_chain
 
         expect(pipeline).not_to be_persisted
         expect(pipeline.variables).not_to be_empty
@@ -156,7 +169,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       end
 
       it 'has pipeline iid' do
-        step.perform!
+        run_chain
 
         expect(pipeline.iid).to be > 0
       end
@@ -168,7 +181,7 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
       end
 
       it 'wastes pipeline iid' do
-        expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved)
+        expect { run_chain }.to raise_error(ActiveRecord::RecordNotSaved)
 
         last_iid = InternalId.ci_pipelines
           .where(project_id: project.id)
@@ -183,14 +196,14 @@ describe Gitlab::Ci::Pipeline::Chain::Populate do
     let(:pipeline) { create(:ci_pipeline, project: project) }
 
     it 'raises error' do
-      expect { step.perform! }.to raise_error(described_class::PopulateError)
+      expect { run_chain }.to raise_error(described_class::PopulateError)
     end
   end
 
   context 'when variables policy is specified' do
     shared_examples_for 'a correct pipeline' do
       it 'populates pipeline according to used policies' do
-        step.perform!
+        run_chain
 
         expect(pipeline.stages.size).to eq 1
         expect(pipeline.stages.first.statuses.size).to eq 1

spec/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs_spec.rb

@@ -2,36 +2,38 @@
 
 require 'spec_helper'
 
-describe Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs do
-  let(:project) { create(:project, :repository) }
+describe ::Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs do
+  let(:project) { create(:project) }
 
   let(:pipeline) do
     build(:ci_pipeline, project: project)
   end
 
   let(:command) do
-    double(:command, project: project, chat_data: { command: 'echo' })
-  end
-
-  before do
-    stub_ci_pipeline_yaml_file(YAML.dump(rspec: { script: 'rspec' }))
+    double(:command,
+      config_processor: double(:processor,
+        jobs: { echo: double(:job_echo), rspec: double(:job_rspec) }),
+      project: project,
+      chat_data: { command: 'echo' })
   end
 
   describe '#perform!' do
-    it 'removes unwanted jobs for chat pipelines' do
-      allow(pipeline).to receive(:chat?).and_return(true)
+    subject { described_class.new(pipeline, command).perform! }
 
-      pipeline.config_processor.jobs[:echo] = double(:job)
+    it 'removes unwanted jobs for chat pipelines' do
+      expect(pipeline).to receive(:chat?).and_return(true)
 
-      described_class.new(pipeline, command).perform!
+      subject
 
-      expect(pipeline.config_processor.jobs.keys).to eq([:echo])
-    end
+      expect(command.config_processor.jobs.keys).to eq([:echo])
     end
 
-  it 'does not remove any jobs for non-chat pipelines' do
-    described_class.new(pipeline, command).perform!
+    it 'does not remove any jobs for non chat-pipelines' do
+      expect(pipeline).to receive(:chat?).and_return(false)
+
+      subject
 
-    expect(pipeline.config_processor.jobs.keys).to eq([:rspec])
+      expect(command.config_processor.jobs.keys).to eq([:echo, :rspec])
+    end
   end
 end

spec/lib/gitlab/ci/pipeline/chain/seed_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Chain::Seed do
+  let(:project) { create(:project, :repository) }
+  let(:user) { create(:user, developer_projects: [project]) }
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(
+      project: project,
+      current_user: user,
+      origin_ref: 'master',
+      seeds_block: nil)
+  end
+
+  def run_chain(pipeline, command)
+    [
+      Gitlab::Ci::Pipeline::Chain::Config::Content.new(pipeline, command),
+      Gitlab::Ci::Pipeline::Chain::Config::Process.new(pipeline, command)
+    ].map(&:perform!)
+
+    described_class.new(pipeline, command).perform!
+  end
+
+  let(:pipeline) { build(:ci_pipeline, project: project) }
+
+  describe '#perform!' do
+    before do
+      stub_ci_pipeline_yaml_file(YAML.dump(config))
+      run_chain(pipeline, command)
+    end
+
+    let(:config) do
+      { rspec: { script: 'rake' } }
+    end
+
+    it 'allocates next IID' do
+      expect(pipeline.iid).to be_present
+    end
+
+    it 'sets the seeds in the command object' do
+      expect(command.stage_seeds).to all(be_a Gitlab::Ci::Pipeline::Seed::Base)
+      expect(command.stage_seeds.count).to eq 1
+    end
+
+    context 'when no ref policy is specified' do
+      let(:config) do
+        {
+          production: { stage: 'deploy', script: 'cap prod' },
+          rspec: { stage: 'test', script: 'rspec' },
+          spinach: { stage: 'test', script: 'spinach' }
+        }
+      end
+
+      it 'correctly fabricates a stage seeds object' do
+        seeds = command.stage_seeds
+        expect(seeds.size).to eq 2
+        expect(seeds.first.attributes[:name]).to eq 'test'
+        expect(seeds.second.attributes[:name]).to eq 'deploy'
+        expect(seeds.dig(0, 0, :name)).to eq 'rspec'
+        expect(seeds.dig(0, 1, :name)).to eq 'spinach'
+        expect(seeds.dig(1, 0, :name)).to eq 'production'
+      end
+    end
+
+    context 'when refs policy is specified' do
+      let(:pipeline) do
+        build(:ci_pipeline, project: project, ref: 'feature', tag: true)
+      end
+
+      let(:config) do
+        {
+          production: { stage: 'deploy', script: 'cap prod', only: ['master'] },
+          spinach: { stage: 'test', script: 'spinach', only: ['tags'] }
+        }
+      end
+
+      it 'returns stage seeds only assigned to master' do
+        seeds = command.stage_seeds
+
+        expect(seeds.size).to eq 1
+        expect(seeds.first.attributes[:name]).to eq 'test'
+        expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+      end
+    end
+
+    context 'when source policy is specified' do
+      let(:pipeline) { create(:ci_pipeline, source: :schedule) }
+
+      let(:config) do
+        {
+          production: { stage: 'deploy', script: 'cap prod', only: ['triggers'] },
+          spinach: { stage: 'test', script: 'spinach', only: ['schedules'] }
+        }
+      end
+
+      it 'returns stage seeds only assigned to schedules' do
+        seeds = command.stage_seeds
+
+        expect(seeds.size).to eq 1
+        expect(seeds.first.attributes[:name]).to eq 'test'
+        expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+      end
+    end
+
+    context 'when kubernetes policy is specified' do
+      let(:config) do
+        {
+          spinach: { stage: 'test', script: 'spinach' },
+          production: {
+            stage: 'deploy',
+            script: 'cap',
+            only: { kubernetes: 'active' }
+          }
+        }
+      end
+
+      context 'when kubernetes is active' do
+        context 'when user configured kubernetes from CI/CD > Clusters' do
+          let!(:cluster) { create(:cluster, :project, :provided_by_gcp) }
+          let(:project) { cluster.project }
+          let(:pipeline) { build(:ci_pipeline, project: project) }
+
+          it 'returns seeds for kubernetes dependent job' do
+            seeds = command.stage_seeds
+
+            expect(seeds.size).to eq 2
+            expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+            expect(seeds.dig(1, 0, :name)).to eq 'production'
+          end
+        end
+      end
+
+      context 'when kubernetes is not active' do
+        it 'does not return seeds for kubernetes dependent job' do
+          seeds = command.stage_seeds
+
+          expect(seeds.size).to eq 1
+          expect(seeds.dig(0, 0, :name)).to eq 'spinach'
+        end
+      end
+    end
+
+    context 'when variables policy is specified' do
+      let(:config) do
+        {
+          unit: { script: 'minitest', only: { variables: ['$CI_PIPELINE_SOURCE'] } },
+          feature: { script: 'spinach', only: { variables: ['$UNDEFINED'] } }
+        }
+      end
+
+      it 'returns stage seeds only when variables expression is truthy' do
+        seeds = command.stage_seeds
+
+        expect(seeds.size).to eq 1
+        expect(seeds.dig(0, 0, :name)).to eq 'unit'
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb

-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe Gitlab::Ci::Pipeline::Chain::Validate::Config do
-  set(:project) { create(:project, :repository) }
-  set(:user) { create(:user) }
-
-  let(:command) do
-    Gitlab::Ci::Pipeline::Chain::Command.new(
-      project: project,
-      current_user: user,
-      save_incompleted: true)
-  end
-
-  let(:pipeline) do
-    build(:ci_pipeline, project: project)
-  end
-
-  let!(:step) { described_class.new(pipeline, command) }
-
-  subject { step.perform! }
-
-  context 'when pipeline has no YAML configuration' do
-    let(:pipeline) do
-      build_stubbed(:ci_pipeline, project: project)
-    end
-
-    it 'appends errors about missing configuration' do
-      subject
-
-      expect(pipeline.errors.to_a)
-        .to include 'Missing .gitlab-ci.yml file'
-    end
-
-    it 'breaks the chain' do
-      subject
-
-      expect(step.break?).to be true
-    end
-  end
-
-  context 'when YAML configuration contains errors' do
-    before do
-      stub_ci_pipeline_yaml_file('invalid YAML')
-      subject
-    end
-
-    it 'appends errors about YAML errors' do
-      expect(pipeline.errors.to_a)
-        .to include 'Invalid configuration format'
-    end
-
-    it 'breaks the chain' do
-      expect(step.break?).to be true
-    end
-
-    context 'when saving incomplete pipeline is allowed' do
-      let(:command) do
-        double('command', project: project,
-                          current_user: user,
-                          save_incompleted: true)
-      end
-
-      it 'fails the pipeline' do
-        subject
-
-        expect(pipeline.reload).to be_failed
-      end
-
-      it 'sets a config error failure reason' do
-        subject
-
-        expect(pipeline.reload.config_error?).to eq true
-      end
-    end
-
-    context 'when saving incomplete pipeline is not allowed' do
-      let(:command) do
-        double('command', project: project,
-                          current_user: user,
-                          save_incompleted: false)
-      end
-
-      it 'does not drop pipeline' do
-        subject
-
-        expect(pipeline).not_to be_failed
-        expect(pipeline).not_to be_persisted
-      end
-    end
-  end
-
-  context 'when pipeline contains configuration validation errors' do
-    before do
-      stub_ci_pipeline_yaml_file(YAML.dump({
-        rspec: {
-          before_script: 10,
-          script: 'ls -al'
-        }
-      }))
-
-      subject
-    end
-
-    it 'appends configuration validation errors to pipeline errors' do
-      expect(pipeline.errors.to_a)
-        .to include "jobs:rspec:before_script config should be an array containing strings and arrays of strings"
-    end
-
-    it 'breaks the chain' do
-      expect(step.break?).to be true
-    end
-  end
-
-  context 'when pipeline is correct and complete' do
-    before do
-      stub_ci_pipeline_yaml_file(YAML.dump({
-        rspec: {
-          script: 'rspec'
-        }
-      }))
-      subject
-    end
-
-    it 'does not invalidate the pipeline' do
-      expect(pipeline).to be_valid
-    end
-
-    it 'does not break the chain' do
-      expect(step.break?).to be false
-    end
-  end
-
-  context 'when pipeline source is merge request' do
-    before do
-      stub_ci_pipeline_yaml_file(YAML.dump(config))
-      subject
-    end
-
-    let(:pipeline) { build_stubbed(:ci_pipeline, project: project) }
-
-    let(:merge_request_pipeline) do
-      build(:ci_pipeline, source: :merge_request_event, project: project)
-    end
-
-    let(:chain) { described_class.new(merge_request_pipeline, command).tap(&:perform!) }
-
-    context "when config contains 'merge_requests' keyword" do
-      let(:config) { { rspec: { script: 'echo', only: ['merge_requests'] } } }
-
-      it 'does not break the chain' do
-        expect(chain).not_to be_break
-      end
-    end
-
-    context "when config contains 'merge_request' keyword" do
-      let(:config) { { rspec: { script: 'echo', only: ['merge_request'] } } }
-
-      it 'does not break the chain' do
-        expect(chain).not_to be_break
-      end
-    end
-  end
-end

spec/models/ci/build_spec.rb

@@ -2221,7 +2221,7 @@ describe Ci::Build do
           { key: 'CI_PAGES_URL', value: project.pages_url, public: true, masked: false },
           { key: 'CI_API_V4_URL', value: 'http://localhost/api/v4', public: true, masked: false },
           { key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s, public: true, masked: false },
-          { key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true, masked: false },
+          { key: 'CI_CONFIG_PATH', value: pipeline.config_path, public: true, masked: false },
           { key: 'CI_PIPELINE_SOURCE', value: pipeline.source, public: true, masked: false },
           { key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message, public: true, masked: false },
           { key: 'CI_COMMIT_TITLE', value: pipeline.git_commit_title, public: true, masked: false },
@@ -2667,11 +2667,17 @@ describe Ci::Build do
       it { is_expected.to include(deployment_variable) }
     end
 
+    context 'when project has default CI config path' do
+      let(:ci_config_path) { { key: 'CI_CONFIG_PATH', value: '.gitlab-ci.yml', public: true, masked: false } }
+
+      it { is_expected.to include(ci_config_path) }
+    end
+
     context 'when project has custom CI config path' do
       let(:ci_config_path) { { key: 'CI_CONFIG_PATH', value: 'custom', public: true, masked: false } }
 
       before do
-        project.update(ci_config_path: 'custom')
+        expect_any_instance_of(Project).to receive(:ci_config_path) { 'custom' }
       end
 
       it { is_expected.to include(ci_config_path) }

spec/requests/api/merge_requests_spec.rb

@@ -1043,9 +1043,7 @@ describe API::MergeRequests do
 
   describe 'POST /projects/:id/merge_requests/:merge_request_iid/pipelines' do
     before do
-      allow_any_instance_of(Ci::Pipeline)
-        .to receive(:ci_yaml_file)
-        .and_return(YAML.dump({
+      stub_ci_pipeline_yaml_file(YAML.dump({
         rspec: {
           script: 'ls',
           only: ['merge_requests']

spec/services/ci/create_pipeline_service_spec.rb

@@ -65,6 +65,7 @@ describe Ci::CreatePipelineService do
         expect(pipeline.iid).not_to be_nil
         expect(pipeline.repository_source?).to be true
         expect(pipeline.builds.first).to be_kind_of(Ci::Build)
+        expect(pipeline.yaml_errors).not_to be_present
       end
 
       it 'increments the prometheus counter' do
@@ -474,6 +475,66 @@ describe Ci::CreatePipelineService do
       end
     end
 
+    context 'config evaluation' do
+      context 'when config is in a file in repository' do
+        before do
+          content = YAML.dump(rspec: { script: 'echo' })
+          stub_ci_pipeline_yaml_file(content)
+        end
+
+        it 'pull it from the repository' do
+          pipeline = execute_service
+          expect(pipeline).to be_repository_source
+          expect(pipeline.builds.map(&:name)).to eq ['rspec']
+        end
+      end
+
+      context 'when config is from Auto-DevOps' do
+        before do
+          stub_ci_pipeline_yaml_file(nil)
+          allow_any_instance_of(Project).to receive(:auto_devops_enabled?).and_return(true)
+        end
+
+        it 'pull it from Auto-DevOps' do
+          pipeline = execute_service
+          expect(pipeline).to be_auto_devops_source
+          expect(pipeline.builds.map(&:name)).to eq %w[test code_quality build]
+        end
+      end
+
+      context 'when config is not found' do
+        before do
+          stub_ci_pipeline_yaml_file(nil)
+        end
+
+        it 'attaches errors to the pipeline' do
+          pipeline = execute_service
+
+          expect(pipeline.errors.full_messages).to eq ['Missing .gitlab-ci.yml file']
+          expect(pipeline).not_to be_persisted
+        end
+      end
+
+      context 'when an unexpected error is raised' do
+        before do
+          expect(Gitlab::Ci::YamlProcessor).to receive(:new)
+            .and_raise(RuntimeError, 'undefined failure')
+        end
+
+        it 'saves error in pipeline' do
+          pipeline = execute_service
+
+          expect(pipeline.yaml_errors).to include('Undefined error')
+        end
+
+        it 'logs error' do
+          expect(Gitlab::Sentry).to receive(:track_acceptable_exception).and_call_original
+
+          execute_service
+        end
+      end
+    end
+
     context 'when yaml is invalid' do
       let(:ci_yaml) { 'invalid: file: fiile' }
       let(:message) { 'Message' }
@@ -539,6 +600,25 @@ describe Ci::CreatePipelineService do
       end
     end
 
+    context 'when an unexpected error is raised' do
+      before do
+        expect(Gitlab::Ci::YamlProcessor).to receive(:new)
+          .and_raise(RuntimeError, 'undefined failure')
+      end
+
+      it 'saves error in pipeline' do
+        pipeline = execute_service
+
+        expect(pipeline.yaml_errors).to include('Undefined error')
+      end
+
+      it 'logs error' do
+        expect(Gitlab::Sentry).to receive(:track_acceptable_exception).and_call_original
+
+        execute_service
+      end
+    end
+
     context 'when commit contains a [ci skip] directive' do
       let(:message) { "some message[ci skip]" }
 

spec/support/helpers/stub_gitlab_calls.rb

@@ -18,8 +18,13 @@ module StubGitlabCalls
     stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
   end
 
-  def stub_ci_pipeline_yaml_file(ci_yaml)
-    allow_any_instance_of(Ci::Pipeline).to receive(:ci_yaml_file) { ci_yaml }
+  def stub_ci_pipeline_yaml_file(ci_yaml_content)
+    allow_any_instance_of(Repository).to receive(:gitlab_ci_yml_for).and_return(ci_yaml_content)
+
+    # Ensure we don't hit auto-devops when config not found in repository
+    unless ci_yaml_content
+      allow_any_instance_of(Project).to receive(:auto_devops_enabled?).and_return(false)
+    end
   end
 
   def stub_pipeline_modified_paths(pipeline, modified_paths)