Introduce a root level CI YAML

Add a layer of CI configuration on top of the .gitlab-ci.yml
which uses `include:` feature to include a file from repository,
from a remote URL or from a template (Auto-DevOps)

app/models/ci/pipeline.rb

@@ -204,7 +204,7 @@ module Ci
     end
 
     scope :internal, -> { where(source: internal_sources) }
-    scope :ci_sources, -> { where(config_source: ci_sources_values) }
+    scope :ci_sources, -> { where(config_source: ::Ci::PipelineEnums.ci_config_sources_values) }
     scope :for_user, -> (user) { where(user: user) }
     scope :for_sha, -> (sha) { where(sha: sha) }
     scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) }
@@ -315,10 +315,6 @@ module Ci
       sources.reject { |source| source == "external" }.values
     end
 
-    def self.ci_sources_values
-      config_sources.values_at(:repository_source, :auto_devops_source, :unknown_source)
-    end
-
     def self.bridgeable_statuses
       ::Ci::Pipeline::AVAILABLE_STATUSES - %w[created preparing pending]
     end

app/models/ci/pipeline_enums.rb

@@ -35,9 +35,20 @@ module Ci
       {
         unknown_source: nil,
         repository_source: 1,
-        auto_devops_source: 2
+        auto_devops_source: 2,
+        remote_source: 4,
+        external_project_source: 5
       }
     end
+
+    def self.ci_config_sources_values
+      config_sources.values_at(
+        :unknown_source,
+        :repository_source,
+        :auto_devops_source,
+        :remote_source,
+        :external_project_source)
+    end
   end
 end
 

changelogs/unreleased/add-root-ci-config-including-user-defined-config.yml

+---
+title: Allow CI config path to point to a URL or file in a different repository
+merge_request: 20179
+author:
+type: added

doc/user/project/pipelines/settings.md

@@ -67,20 +67,37 @@ For information about setting a maximum artifact size for a project, see
 
 ## Custom CI configuration path
 
-> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/12509) in GitLab 9.4.
+> - [Support for external `.gitlab-ci.yml` locations](https://gitlab.com/gitlab-org/gitlab/issues/14376) introduced in GitLab 12.6.
 
 By default we look for the `.gitlab-ci.yml` file in the project's root
-directory. If you require a different location **within** the repository,
-you can set a custom path that will be used to look up the configuration file,
-this path should be **relative** to the root.
+directory. If needed, you can specify an alternate path and file name, including locations outside the project.
 
-Here are some valid examples:
+Hosting the configuration file in a separate project will allow stricter control of the
+configuration file. You can limit access to the project hosting the configuration to only people
+with proper authorization, and users can use the configuration for their pipelines,
+without being able to modify it.
 
-- `.gitlab-ci.yml`
+If the CI configuration will stay within the repository, but in a
+location different than the default,
+the path must be relative to the root directory. Examples of valid paths and file names:
+
+- `.gitlab-ci.yml` (default)
 - `.my-custom-file.yml`
 - `my/path/.gitlab-ci.yml`
 - `my/path/.my-custom-file.yml`
 
+If the CI configuration will be hosted in a different project within GitLab, the path must be relative
+to the root directory in the other project, with the group and project name added to the end:
+
+- `.gitlab-ci.yml@mygroup/another-project`
+- `my/path/.my-custom-file.yml@mygroup/another-project`
+
+If the CI configuration will be hosted on an external site, different than the GitLab instance,
+the URL link must end with `.yml`:
+
+- `http://example.com/generate/ci/config.yml`
+
 The path can be customized at a project level. To customize the path:
 
 1. Go to the project's **Settings > CI / CD**.

ee/spec/models/ci/pipeline_spec.rb

@@ -22,6 +22,29 @@ describe Ci::Pipeline do
     end
   end
 
+  describe '.ci_sources' do
+    subject { described_class.ci_sources }
+
+    let(:all_config_sources) { described_class.config_sources }
+
+    before do
+      all_config_sources.each do |source, _value|
+        create(:ci_pipeline, config_source: source)
+      end
+    end
+
+    it 'contains pipelines having CI only config sources' do
+      expect(subject.map(&:config_source)).to contain_exactly(
+        'auto_devops_source',
+        'external_project_source',
+        'remote_source',
+        'repository_source',
+        'unknown_source'
+      )
+      expect(subject.size).to be < all_config_sources.size
+    end
+  end
+
   describe '#with_vulnerabilities scope' do
     let!(:pipeline_1) { create(:ci_pipeline, project: project) }
     let!(:pipeline_2) { create(:ci_pipeline, project: project) }

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -8,21 +8,28 @@ module Gitlab
           class Content < Chain::Base
             include Chain::Helpers
 
-            def perform!
-              return if @command.config_content
-
-              if content = content_from_repo
-                @command.config_content = content
-                @pipeline.config_source = :repository_source
-                # TODO: we should persist ci_config_path
-                # @pipeline.config_path = ci_config_path
-              elsif content = content_from_auto_devops
-                @command.config_content = content
-                @pipeline.config_source = :auto_devops_source
-              end
+            SOURCES = [
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Repository,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Remote,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops
+            ].freeze
+
+            LEGACY_SOURCES = [
+              Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository,
+              Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops
+            ].freeze
 
-              unless @command.config_content
-                return error("Missing #{ci_config_path} file")
+            def perform!
+              if config = find_config
+                # TODO: we should persist config_content
+                # @pipeline.config_content = config.content
+                @command.config_content = config.content
+                @pipeline.config_source = config.source
+              else
+                error('Missing CI config file')
               end
             end
 
@@ -32,24 +39,21 @@ module Gitlab
 
             private
 
-            def content_from_repo
-              return unless project
-              return unless @pipeline.sha
-              return unless ci_config_path
+            def find_config
+              sources.each do |source|
+                config = source.new(@pipeline, @command)
+                return config if config.exists?
+              end
 
-              project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
-            rescue GRPC::NotFound, GRPC::Internal
               nil
             end
 
-            def content_from_auto_devops
-              return unless project&.auto_devops_enabled?
-
-              Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
+            def sources
+              if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
+                SOURCES
+              else
+                LEGACY_SOURCES
               end
-
-            def ci_config_path
-              project.ci_config_path.presence || '.gitlab-ci.yml'
             end
           end
         end

lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class AutoDevops < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project&.auto_devops_enabled?
+
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  YAML.dump('include' => [{ 'template' => template.full_name }])
+                end
+              end
+
+              def source
+                :auto_devops_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/external_project.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class ExternalProject < Source
+              def content
+                strong_memoize(:content) do
+                  next unless external_project_path?
+
+                  path_file, path_project = ci_config_path.split('@', 2)
+                  YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }])
+                end
+              end
+
+              def source
+                :external_project_source
+              end
+
+              private
+
+              # Example: path/to/.gitlab-ci.yml@another-group/another-project
+              def external_project_path?
+                ci_config_path =~ /\A.+(yml|yaml)@.+\z/
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class LegacyAutoDevops < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project&.auto_devops_enabled?
+
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  template.content
+                end
+              end
+
+              def source
+                :auto_devops_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class LegacyRepository < Source
+              def content
+                strong_memoize(:content) do
+                  next unless project
+                  next unless @pipeline.sha
+                  next unless ci_config_path
+
+                  project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
+                rescue GRPC::NotFound, GRPC::Internal
+                  nil
+                end
+              end
+
+              def source
+                :repository_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/remote.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Remote < Source
+              def content
+                strong_memoize(:content) do
+                  next unless ci_config_path =~ URI.regexp(%w[http https])
+
+                  YAML.dump('include' => [{ 'remote' => ci_config_path }])
+                end
+              end
+
+              def source
+                :remote_source
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/repository.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Repository < Source
+              def content
+                strong_memoize(:content) do
+                  next unless file_in_repository?
+
+                  YAML.dump('include' => [{ 'local' => ci_config_path }])
+                end
+              end
+
+              def source
+                :repository_source
+              end
+
+              private
+
+              def file_in_repository?
+                return unless project
+                return unless @pipeline.sha
+
+                project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present?
+              rescue GRPC::NotFound, GRPC::Internal
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/runtime.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Runtime < Source
+              def content
+                @command.config_content
+              end
+
+              def source
+                # The only case when this source is used is when the config content
+                # is passed in as parameter to Ci::CreatePipelineService.
+                # This would only occur with parent/child pipelines which is being
+                # implemented.
+                # TODO: change source to return :runtime_source
+                # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041
+
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/chain/config/content/source.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content
+            class Source
+              include Gitlab::Utils::StrongMemoize
+
+              DEFAULT_YAML_FILE = '.gitlab-ci.yml'
+
+              def initialize(pipeline, command)
+                @pipeline = pipeline
+                @command = command
+              end
+
+              def exists?
+                strong_memoize(:exists) do
+                  content.present?
+                end
+              end
+
+              def content
+                raise NotImplementedError
+              end
+
+              def source
+                raise NotImplementedError
+              end
+
+              def project
+                @project ||= @pipeline.project
+              end
+
+              def ci_config_path
+                @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/features/projects/pipelines/pipelines_spec.rb

@@ -706,7 +706,7 @@ describe 'Pipelines', :js do
             click_on 'Run Pipeline'
           end
 
-          it { expect(page).to have_content('Missing .gitlab-ci.yml file') }
+          it { expect(page).to have_content('Missing CI config file') }
           it 'creates a pipeline after first request failed and a valid gitlab-ci.yml file is available when trying again' do
             click_button project.default_branch
 

spec/lib/gitlab/chat/command_spec.rb

@@ -44,7 +44,7 @@ describe Gitlab::Chat::Command do
     let(:pipeline) { command.create_pipeline }
 
     before do
-      stub_repository_ci_yaml_file(sha: project.commit.id)
+      stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
 
       project.add_developer(chat_name.user)
     end

spec/lib/gitlab/ci/pipeline/chain/build_spec.rb

@@ -30,7 +30,7 @@ describe Gitlab::Ci::Pipeline::Chain::Build do
   let(:step) { described_class.new(pipeline, command) }
 
   before do
-    stub_repository_ci_yaml_file(sha: anything)
+    stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
   end
 
   it 'never breaks the chain' do

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Chain::Config::Content do
+  let(:project) { create(:project, ci_config_path: ci_config_path) }
+  let(:pipeline) { build(:ci_pipeline, project: project) }
+  let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
+
+  subject { described_class.new(pipeline, command) }
+
+  describe '#perform!' do
+    context 'when feature flag is disabled' do
+      before do
+        stub_feature_flags(ci_root_config_content: false)
+      end
+
+      context 'when config is defined in a custom path in the repository' do
+        let(:ci_config_path) { 'path/to/config.yml' }
+
+        before do
+          expect(project.repository)
+            .to receive(:gitlab_ci_yml_for)
+            .with(pipeline.sha, ci_config_path)
+            .and_return('the-content')
+        end
+
+        it 'returns the content of the YAML file' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'repository_source'
+          expect(command.config_content).to eq('the-content')
+        end
+      end
+
+      context 'when config is defined remotely' do
+        let(:ci_config_path) { 'http://example.com/path/to/ci/config.yml' }
+
+        it 'does not support URLs and default to AutoDevops' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+          expect(command.config_content).to eq(template.content)
+        end
+      end
+
+      context 'when config is defined in a separate repository' do
+        let(:ci_config_path) { 'path/to/.gitlab-ci.yml@another-group/another-repo' }
+
+        it 'does not support YAML from external repository and default to AutoDevops' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+          expect(command.config_content).to eq(template.content)
+        end
+      end
+
+      context 'when config is defined in the default .gitlab-ci.yml' do
+        let(:ci_config_path) { nil }
+
+        before do
+          expect(project.repository)
+            .to receive(:gitlab_ci_yml_for)
+            .with(pipeline.sha, '.gitlab-ci.yml')
+            .and_return('the-content')
+        end
+
+        it 'returns the content of the canonical config file' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'repository_source'
+          expect(command.config_content).to eq('the-content')
+        end
+      end
+
+      context 'when config is the Auto-Devops template' do
+        let(:ci_config_path) { nil }
+
+        before do
+          expect(project).to receive(:auto_devops_enabled?).and_return(true)
+        end
+
+        it 'returns the content of AutoDevops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+          expect(command.config_content).to eq(template.content)
+        end
+      end
+
+      context 'when config is not defined anywhere' do
+        let(:ci_config_path) { nil }
+
+        before do
+          expect(project).to receive(:auto_devops_enabled?).and_return(false)
+        end
+
+        it 'builds root config including the auto-devops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq('unknown_source')
+          expect(command.config_content).to be_nil
+          expect(pipeline.errors.full_messages).to include('Missing CI config file')
+        end
+      end
+    end
+
+    context 'when config is defined in a custom path in the repository' do
+      let(:ci_config_path) { 'path/to/config.yml' }
+
+      before do
+        expect(project.repository)
+          .to receive(:gitlab_ci_yml_for)
+          .with(pipeline.sha, ci_config_path)
+          .and_return('the-content')
+      end
+
+      it 'builds root config including the local custom file' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'repository_source'
+        expect(command.config_content).to eq(<<~EOY)
+          ---
+          include:
+          - local: #{ci_config_path}
+        EOY
+      end
+    end
+
+    context 'when config is defined remotely' do
+      let(:ci_config_path) { 'http://example.com/path/to/ci/config.yml' }
+
+      it 'builds root config including the remote config' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'remote_source'
+        expect(command.config_content).to eq(<<~EOY)
+          ---
+          include:
+          - remote: #{ci_config_path}
+        EOY
+      end
+    end
+
+    context 'when config is defined in a separate repository' do
+      let(:ci_config_path) { 'path/to/.gitlab-ci.yml@another-group/another-repo' }
+
+      it 'builds root config including the path to another repository' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'external_project_source'
+        expect(command.config_content).to eq(<<~EOY)
+          ---
+          include:
+          - project: another-group/another-repo
+            file: path/to/.gitlab-ci.yml
+        EOY
+      end
+    end
+
+    context 'when config is defined in the default .gitlab-ci.yml' do
+      let(:ci_config_path) { nil }
+
+      before do
+        expect(project.repository)
+          .to receive(:gitlab_ci_yml_for)
+          .with(pipeline.sha, '.gitlab-ci.yml')
+          .and_return('the-content')
+      end
+
+      it 'builds root config including the canonical CI config file' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'repository_source'
+        expect(command.config_content).to eq(<<~EOY)
+          ---
+          include:
+          - local: ".gitlab-ci.yml"
+        EOY
+      end
+    end
+
+    context 'when config is the Auto-Devops template' do
+      let(:ci_config_path) { nil }
+
+      before do
+        expect(project).to receive(:auto_devops_enabled?).and_return(true)
+      end
+
+      it 'builds root config including the auto-devops template' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'auto_devops_source'
+        expect(command.config_content).to eq(<<~EOY)
+          ---
+          include:
+          - template: Auto-DevOps.gitlab-ci.yml
+        EOY
+      end
+    end
+
+    context 'when config is not defined anywhere' do
+      let(:ci_config_path) { nil }
+
+      before do
+        expect(project).to receive(:auto_devops_enabled?).and_return(false)
+      end
+
+      it 'builds root config including the auto-devops template' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq('unknown_source')
+        expect(command.config_content).to be_nil
+        expect(pipeline.errors.full_messages).to include('Missing CI config file')
+      end
+    end
+  end
+end

spec/requests/api/pipelines_spec.rb

@@ -384,7 +384,7 @@ describe API::Pipelines do
             post api("/projects/#{project.id}/pipeline", user), params: { ref: project.default_branch }
 
             expect(response).to have_gitlab_http_status(400)
-            expect(json_response['message']['base'].first).to eq 'Missing .gitlab-ci.yml file'
+            expect(json_response['message']['base'].first).to eq 'Missing CI config file'
             expect(json_response).not_to be_an Array
           end
         end

spec/services/ci/create_pipeline_service_spec.rb

@@ -10,7 +10,7 @@ describe Ci::CreatePipelineService do
   let(:ref_name) { 'refs/heads/master' }
 
   before do
-    stub_repository_ci_yaml_file(sha: anything)
+    stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
   end
 
   describe '#execute' do
@@ -510,7 +510,7 @@ describe Ci::CreatePipelineService do
         it 'attaches errors to the pipeline' do
           pipeline = execute_service
 
-          expect(pipeline.errors.full_messages).to eq ['Missing .gitlab-ci.yml file']
+          expect(pipeline.errors.full_messages).to eq ['Missing CI config file']
           expect(pipeline).not_to be_persisted
         end
       end

spec/support/helpers/stub_gitlab_calls.rb

@@ -19,24 +19,28 @@ module StubGitlabCalls
   end
 
   def stub_ci_pipeline_yaml_file(ci_yaml_content)
-    allow_any_instance_of(Repository).to receive(:gitlab_ci_yml_for).and_return(ci_yaml_content)
+    allow_any_instance_of(Repository)
+      .to receive(:gitlab_ci_yml_for)
+      .and_return(ci_yaml_content)
 
     # Ensure we don't hit auto-devops when config not found in repository
     unless ci_yaml_content
       allow_any_instance_of(Project).to receive(:auto_devops_enabled?).and_return(false)
     end
+
+    # Stub the first call to `include:[local: .gitlab-ci.yml]` when
+    # evaluating the CI root config content.
+    if Feature.enabled?(:ci_root_config_content, default_enabled: true)
+      allow_any_instance_of(Gitlab::Ci::Config::External::File::Local)
+        .to receive(:content)
+        .and_return(ci_yaml_content)
+    end
   end
 
   def stub_pipeline_modified_paths(pipeline, modified_paths)
     allow(pipeline).to receive(:modified_paths).and_return(modified_paths)
   end
 
-  def stub_repository_ci_yaml_file(sha:, path: '.gitlab-ci.yml')
-    allow_any_instance_of(Repository)
-      .to receive(:gitlab_ci_yml_for).with(sha, path)
-      .and_return(gitlab_ci_yaml)
-  end
-
   def stub_ci_builds_disabled
     allow_any_instance_of(Project).to receive(:builds_enabled?).and_return(false)
   end