Add ability to use YAML string create CI pipelines

Adds new parameter source, command attribute and config source enum in
order to create pipelines on-demand.

app/models/ci/pipeline.rb

@@ -349,6 +349,10 @@ module Ci
       success.group(:project_id).select('max(id) as id')
     end
 
+    def self.last_finished_for_ref_id(ci_ref_id)
+      where(ci_ref_id: ci_ref_id).ci_sources.finished.order(id: :desc).select(:id).take
+    end
+
     def self.truncate_sha(sha)
       sha[0...8]
     end

app/models/ci/pipeline_enums.rb

@@ -45,7 +45,8 @@ module Ci
         webide_source: 3,
         remote_source: 4,
         external_project_source: 5,
-        bridge_source: 6
+        bridge_source: 6,
+        parameter_source: 7
       }
     end
 

app/models/ci/ref.rb

@@ -43,7 +43,7 @@ module Ci
     end
 
     def last_finished_pipeline_id
-      Ci::Pipeline.where(ci_ref_id: self.id).finished.order(id: :desc).select(:id).take&.id
+      Ci::Pipeline.last_finished_for_ref_id(self.id)&.id
     end
 
     def update_status_by!(pipeline)

app/services/ci/create_pipeline_service.rb

@@ -23,6 +23,24 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::Activity,
                 Gitlab::Ci::Pipeline::Chain::Limit::JobActivity].freeze
 
+    # Create a new pipeline in the specified project.
+    #
+    # @param [Symbol] source                             What event (Ci::Pipeline.sources) triggers the pipeline
+    #                                                    creation.
+    # @param [Boolean] ignore_skip_ci                    Whether skipping a pipeline creation when `[skip ci]` comment
+    #                                                    is present in the commit body
+    # @param [Boolean] save_on_errors                    Whether persisting an invalid pipeline when it encounters an
+    #                                                    error during creation (e.g. invalid yaml)
+    # @param [Ci::TriggerRequest] trigger_request        The pipeline trigger triggers the pipeline creation.
+    # @param [Ci::PipelineSchedule] schedule             The pipeline schedule triggers the pipeline creation.
+    # @param [MergeRequest] merge_request                The merge request triggers the pipeline creation.
+    # @param [ExternalPullRequest] external_pull_request The external pull request triggers the pipeline creation.
+    # @param [Ci::Bridge] bridge                         The bridge job that triggers the downstream pipeline creation.
+    # @param [String] content                            The content of .gitlab-ci.yml to override the default config
+    #                                                    contents (e.g. .gitlab-ci.yml in repostiry). Mainly used for
+    #                                                    generating a dangling pipeline.
+    #
+    # @return [Ci::Pipeline]                             The created Ci::Pipeline object.
     # rubocop: disable Metrics/ParameterLists
     def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, merge_request: nil, external_pull_request: nil, bridge: nil, **options, &block)
       @pipeline = Ci::Pipeline.new
@@ -122,13 +140,8 @@ module Ci
       end
     end
 
-    def extra_options(options = {})
-      # In Ruby 2.4, even when options is empty, f(**options) doesn't work when f
-      # doesn't have any parameters. We reproduce the Ruby 2.5 behavior by
-      # checking explicitly that no arguments are given.
-      raise ArgumentError if options.any?
-
-      {} # overridden in EE
+    def extra_options(content: nil)
+      { content: content }
     end
   end
 end

changelogs/unreleased/add-static-source-to-ci-pipeline-218685.yml

+---
+title: Ability to use an arbitrary YAML blob to create CI pipelines
+merge_request: 34706
+author:
+type: added

ee/app/services/ee/ci/create_pipeline_service.rb

@@ -6,10 +6,8 @@ module EE
       extend ::Gitlab::Utils::Override
 
       override :extra_options
-      def extra_options(mirror_update: false)
-        {
-          allow_mirror_update: mirror_update
-        }
+      def extra_options(mirror_update: false, **options)
+        options.merge(allow_mirror_update: mirror_update)
       end
     end
   end

lib/gitlab/ci/pipeline/chain/command.rb

@@ -10,7 +10,7 @@ module Gitlab
           :trigger_request, :schedule, :merge_request, :external_pull_request,
           :ignore_skip_ci, :save_incompleted,
           :seeds_block, :variables_attributes, :push_options,
-          :chat_data, :allow_mirror_update, :bridge,
+          :chat_data, :allow_mirror_update, :bridge, :content,
           # These attributes are set by Chains during processing:
           :config_content, :config_processor, :stage_seeds
         ) do

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -9,6 +9,7 @@ module Gitlab
             include Chain::Helpers
 
             SOURCES = [
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter,
               Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge,
               Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
               Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,

lib/gitlab/ci/pipeline/chain/config/content/parameter.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Parameter < Source
+              def content
+                strong_memoize(:content) do
+                  next unless command.content.present?
+
+                  command.content
+                end
+              end
+
+              def source
+                :parameter_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/source.rb

@@ -11,6 +11,8 @@ module Gitlab
 
               DEFAULT_YAML_FILE = '.gitlab-ci.yml'
 
+              attr_reader :command
+
               def initialize(pipeline, command)
                 @pipeline = pipeline
                 @command = command

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

@@ -5,7 +5,8 @@ require 'spec_helper'
 RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do
   let(:project) { create(:project, ci_config_path: ci_config_path) }
   let(:pipeline) { build(:ci_pipeline, project: project) }
-  let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
+  let(:content) { nil }
+  let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project, content: content) }
 
   subject { described_class.new(pipeline, command) }
 
@@ -141,6 +142,25 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Config::Content do
       end
     end
 
+    context 'when config is passed as a parameter' do
+      let(:ci_config_path) { nil }
+      let(:content) do
+        <<~EOY
+          ---
+          stages:
+          - dast
+        EOY
+      end
+
+      it 'uses the parameter content' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'parameter_source'
+        expect(pipeline.pipeline_config.content).to eq(content)
+        expect(command.config_content).to eq(content)
+      end
+    end
+
     context 'when config is not defined anywhere' do
       let(:ci_config_path) { nil }
 

spec/models/ci/pipeline_spec.rb

@@ -1947,6 +1947,23 @@ RSpec.describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '.last_finished_for_ref_id' do
+    let(:project) { create(:project, :repository) }
+    let(:branch) { project.default_branch }
+    let(:ref) { project.ci_refs.take }
+    let(:config_source) { Ci::PipelineEnums.config_sources[:parameter_source] }
+    let!(:pipeline1) { create(:ci_pipeline, :success, project: project, ref: branch) }
+    let!(:pipeline2) { create(:ci_pipeline, :success, project: project, ref: branch) }
+    let!(:pipeline3) { create(:ci_pipeline, :failed, project: project, ref: branch) }
+    let!(:pipeline4) { create(:ci_pipeline, :success, project: project, ref: branch) }
+    let!(:pipeline5) { create(:ci_pipeline, :success, project: project, ref: branch, config_source: config_source) }
+
+    it 'returns the expected pipeline' do
+      result = described_class.last_finished_for_ref_id(ref.id)
+      expect(result).to eq(pipeline4)
+    end
+  end
+
   describe '.internal_sources' do
     subject { described_class.internal_sources }
 

spec/models/ci/ref_spec.rb

@@ -62,6 +62,35 @@ RSpec.describe Ci::Ref do
     end
   end
 
+  describe '#last_finished_pipeline_id' do
+    let(:pipeline_status) { :running }
+    let(:config_source) { Ci::PipelineEnums.config_sources[:repository_source] }
+    let(:pipeline) { create(:ci_pipeline, pipeline_status, config_source: config_source) }
+    let(:ci_ref) { pipeline.ci_ref }
+
+    context 'when there are no finished pipelines' do
+      it 'returns nil' do
+        expect(ci_ref.last_finished_pipeline_id).to be_nil
+      end
+    end
+
+    context 'when there are finished pipelines' do
+      let(:pipeline_status) { :success }
+
+      it 'returns the pipeline id' do
+        expect(ci_ref.last_finished_pipeline_id).to eq(pipeline.id)
+      end
+
+      context 'when the pipeline is not a ci_source' do
+        let(:config_source) { Ci::PipelineEnums.config_sources[:parameter_source] }
+
+        it 'returns nil' do
+          expect(ci_ref.last_finished_pipeline_id).to be_nil
+        end
+      end
+    end
+  end
+
   describe '#update_status_by!' do
     subject { ci_ref.update_status_by!(pipeline) }
 

spec/services/ci/create_pipeline_service/parameter_content_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:admin) }
+  let(:service) { described_class.new(project, user, { ref: 'refs/heads/master' }) }
+  let(:content) do
+    <<~EOY
+      ---
+      stages:
+        - dast
+
+      variables:
+        DAST_VERSION: 1
+        SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
+      dast:
+        stage: dast
+        image:
+          name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION"
+        variables:
+          GIT_STRATEGY: none
+        script:
+          - /analyze
+    EOY
+  end
+
+  describe '#execute' do
+    subject { service.execute(:web, content: content) }
+
+    context 'parameter config content' do
+      it 'creates a pipeline' do
+        expect(subject).to be_persisted
+      end
+
+      it 'creates builds with the correct names' do
+        expect(subject.builds.pluck(:name)).to match_array %w[dast]
+      end
+
+      it 'creates stages with the correct names' do
+        expect(subject.stages.pluck(:name)).to match_array %w[dast]
+      end
+
+      it 'sets the correct config source' do
+        expect(subject.config_source).to eq 'parameter_source'
+      end
+    end
+  end
+end