Implement including multiple files from a project

This feature is behind a FF "ci_include_multiple_files_from_project"

config/feature_flags/development/ci_include_multiple_files_from_project.yml

+---
+name: ci_include_multiple_files_from_project
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45991
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/271560
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/README.md

@@ -438,6 +438,42 @@ All [nested includes](#nested-includes) are executed in the scope of the target
 This means you can use local (relative to target project), project, remote,
 or template includes.
 
+##### Multiple files from a project
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26793) in GitLab 13.6.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. **(CORE ONLY)**
+
+You can include multiple files from the same project:
+
+```yaml
+include:
+  - project: 'my-group/my-project'
+    ref: master
+    file:
+      - '/templates/.builds.yml'
+      - '/templates/.tests.yml'
+```
+
+Including multiple files from the same project is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_include_multiple_files_from_project)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_include_multiple_files_from_project)
+```
+
 #### `include:remote`
 
 `include:remote` can be used to include a file from a different location,

lib/gitlab/ci/config/external/mapper.rb

@@ -33,6 +33,7 @@ module Gitlab
             locations
               .compact
               .map(&method(:normalize_location))
+              .flat_map(&method(:expand_project_files))
               .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
           end
@@ -52,6 +53,15 @@ module Gitlab
             end
           end
 
+          def expand_project_files(location)
+            return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: false)
+            return location unless location[:project]
+
+            Array.wrap(location[:file]).map do |file|
+              location.merge(file: file)
+            end
+          end
+
           def normalize_location_string(location)
             if ::Gitlab::UrlSanitizer.valid?(location)
               { remote: location }

qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb

+# frozen_string_literal: true
+
+require 'faker'
+
+module QA
+  RSpec.describe 'Verify', :runner, :requires_admin, :skip_live_env do
+    describe "Include multiple files from a project" do
+      let(:feature_flag) { :ci_include_multiple_files_from_project }
+      let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
+      let(:expected_text) { Faker::Lorem.sentence }
+      let(:unexpected_text) { Faker::Lorem.sentence }
+
+      let(:project) do
+        Resource::Project.fabricate_via_api! do |project|
+          project.name = 'project-with-pipeline-1'
+        end
+      end
+
+      let(:other_project) do
+        Resource::Project.fabricate_via_api! do |project|
+          project.name = 'project-with-pipeline-2'
+        end
+      end
+
+      let!(:runner) do
+        Resource::Runner.fabricate! do |runner|
+          runner.project = project
+          runner.name = executor
+          runner.tags = [executor]
+        end
+      end
+
+      before do
+        Runtime::Feature.enable(feature_flag)
+        Flow::Login.sign_in
+        add_included_files
+        add_main_ci_file
+        project.visit!
+        view_the_last_pipeline
+      end
+
+      after do
+        Runtime::Feature.disable(feature_flag)
+        runner.remove_via_api!
+      end
+
+      it 'runs the pipeline with composed config', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1082' do
+        Page::Project::Pipeline::Show.perform do |pipeline|
+          aggregate_failures 'pipeline has all expected jobs' do
+            expect(pipeline).to have_job('build')
+            expect(pipeline).to have_job('test')
+            expect(pipeline).to have_job('deploy')
+          end
+
+          pipeline.click_job('test')
+        end
+
+        Page::Project::Job::Show.perform do |job|
+          aggregate_failures 'main CI is not overridden' do
+            expect(job.output).to have_no_content("#{unexpected_text}")
+            expect(job.output).to have_content("#{expected_text}")
+          end
+        end
+      end
+
+      private
+
+      def add_main_ci_file
+        Resource::Repository::Commit.fabricate_via_api! do |commit|
+          commit.project = project
+          commit.commit_message = 'Add config file'
+          commit.add_files([main_ci_file])
+        end
+      end
+
+      def add_included_files
+        Resource::Repository::Commit.fabricate_via_api! do |commit|
+          commit.project = other_project
+          commit.commit_message = 'Add files'
+          commit.add_files([included_file_1, included_file_2])
+        end
+      end
+
+      def view_the_last_pipeline
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success)
+        Page::Project::Pipeline::Index.perform(&:click_on_latest_pipeline)
+      end
+
+      def main_ci_file
+        {
+          file_path: '.gitlab-ci.yml',
+          content: <<~YAML
+            include:
+              - project: #{other_project.full_path}
+                file:
+                  - file1.yml
+                  - file2.yml
+
+            build:
+              stage: build
+              tags: ["#{executor}"]
+              script: echo 'build'
+
+            test:
+              stage: test
+              tags: ["#{executor}"]
+              script: echo "#{expected_text}"
+          YAML
+        }
+      end
+
+      def included_file_1
+        {
+          file_path: 'file1.yml',
+          content: <<~YAML
+            test:
+              stage: test
+              tags: ["#{executor}"]
+              script: echo "#{unexpected_text}"
+          YAML
+        }
+      end
+
+      def included_file_2
+        {
+          file_path: 'file2.yml',
+          content: <<~YAML
+            deploy:
+              stage: deploy
+              tags: ["#{executor}"]
+              script: echo 'deploy'
+          YAML
+        }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -100,6 +100,42 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
           expect { subject }.to raise_error(described_class::AmbigiousSpecificationError)
         end
       end
+
+      context "when the key is a project's file" do
+        let(:values) do
+          { include: { project: project.full_path, file: local_file },
+            image: 'ruby:2.7' }
+        end
+
+        it 'returns File instances' do
+          expect(subject).to contain_exactly(
+            an_instance_of(Gitlab::Ci::Config::External::File::Project))
+        end
+      end
+
+      context "when the key is project's files" do
+        let(:values) do
+          { include: { project: project.full_path, file: [local_file, 'another_file_path.yml'] },
+            image: 'ruby:2.7' }
+        end
+
+        it 'returns two File instances' do
+          expect(subject).to contain_exactly(
+            an_instance_of(Gitlab::Ci::Config::External::File::Project),
+            an_instance_of(Gitlab::Ci::Config::External::File::Project))
+        end
+
+        context 'when FF ci_include_multiple_files_from_project is disabled' do
+          before do
+            stub_feature_flags(ci_include_multiple_files_from_project: false)
+          end
+
+          it 'returns a File instance' do
+            expect(subject).to contain_exactly(
+              an_instance_of(Gitlab::Ci::Config::External::File::Project))
+          end
+        end
+      end
     end
 
     context "when 'include' is defined as an array" do
@@ -161,6 +197,16 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
       it 'raises an exception' do
         expect { subject }.to raise_error(described_class::DuplicateIncludesError)
       end
+
+      context 'when including multiple files from a project' do
+        let(:values) do
+          { include: { project: project.full_path, file: [local_file, local_file] } }
+        end
+
+        it 'raises an exception' do
+          expect { subject }.to raise_error(described_class::DuplicateIncludesError)
+        end
+      end
     end
 
     context "when too many 'includes' are defined" do
@@ -179,6 +225,16 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
       it 'raises an exception' do
         expect { subject }.to raise_error(described_class::TooManyIncludesError)
       end
+
+      context 'when including multiple files from a project' do
+        let(:values) do
+          { include: { project: project.full_path, file: [local_file, 'another_file_path.yml'] } }
+        end
+
+        it 'raises an exception' do
+          expect { subject }.to raise_error(described_class::TooManyIncludesError)
+        end
+      end
     end
   end
 end

spec/lib/gitlab/ci/config/external/processor_spec.rb

@@ -302,5 +302,82 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         end
       end
     end
+
+    context 'when a valid project file is defined' do
+      let(:values) do
+        {
+          include: { project: another_project.full_path, file: '/templates/my-build.yml' },
+          image: 'ruby:2.7'
+        }
+      end
+
+      before do
+        another_project.add_developer(user)
+
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do
+            <<~HEREDOC
+              my_build:
+                script: echo Hello World
+            HEREDOC
+          end
+        end
+      end
+
+      it 'appends the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :my_build])
+      end
+    end
+
+    context 'when valid project files are defined in a single include' do
+      let(:values) do
+        {
+          include: {
+            project: another_project.full_path,
+            file: ['/templates/my-build.yml', '/templates/my-test.yml']
+          },
+          image: 'ruby:2.7'
+        }
+      end
+
+      before do
+        another_project.add_developer(user)
+
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-build.yml') do
+            <<~HEREDOC
+              my_build:
+                script: echo Hello World
+            HEREDOC
+          end
+
+          allow(repository).to receive(:blob_data_at).with(another_project.commit.id, '/templates/my-test.yml') do
+            <<~HEREDOC
+              my_test:
+                script: echo Hello World
+            HEREDOC
+          end
+        end
+      end
+
+      it 'appends the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :my_build, :my_test])
+      end
+
+      context 'when FF ci_include_multiple_files_from_project is disabled' do
+        before do
+          stub_feature_flags(ci_include_multiple_files_from_project: false)
+        end
+
+        it 'raises an error' do
+          expect { processor.perform }.to raise_error(
+            described_class::IncludeError,
+            'Included file `["/templates/my-build.yml", "/templates/my-test.yml"]` needs to be a string'
+          )
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config_spec.rb

@@ -246,6 +246,14 @@ RSpec.describe Gitlab::Ci::Config do
     let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
     let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
 
+    let(:local_file_content) do
+      File.read(Rails.root.join(local_location))
+    end
+
+    let(:local_location_hash) do
+      YAML.safe_load(local_file_content).deep_symbolize_keys
+    end
+
     let(:remote_file_content) do
       <<~HEREDOC
       variables:
@@ -256,8 +264,8 @@ RSpec.describe Gitlab::Ci::Config do
       HEREDOC
     end
 
-    let(:local_file_content) do
-      File.read(Rails.root.join(local_location))
+    let(:remote_file_hash) do
+      YAML.safe_load(remote_file_content).deep_symbolize_keys
     end
 
     let(:gitlab_ci_yml) do
@@ -283,22 +291,11 @@ RSpec.describe Gitlab::Ci::Config do
 
     context "when gitlab_ci_yml has valid 'include' defined" do
       it 'returns a composed hash' do
-        before_script_values = [
-          "apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs", "ruby -v",
-          "which ruby",
-          "bundle install --jobs $(nproc)  \"${FLAGS[@]}\""
-        ]
-        variables = {
-          POSTGRES_USER: "user",
-          POSTGRES_PASSWORD: "testing-password",
-          POSTGRES_ENABLED: "true",
-          POSTGRES_DB: "$CI_ENVIRONMENT_SLUG"
-        }
         composed_hash = {
-          before_script: before_script_values,
+          before_script: local_location_hash[:before_script],
           image: "ruby:2.7",
           rspec: { script: ["bundle exec rspec"] },
-          variables: variables
+          variables: remote_file_hash[:variables]
         }
 
         expect(config.to_hash).to eq(composed_hash)
@@ -575,5 +572,56 @@ RSpec.describe Gitlab::Ci::Config do
         )
       end
     end
+
+    context "when including multiple files from a project" do
+      let(:other_file_location) { 'my_builds.yml' }
+
+      let(:other_file_content) do
+        <<~HEREDOC
+        build:
+          stage: build
+          script: echo hello
+
+        rspec:
+          stage: test
+          script: bundle exec rspec
+        HEREDOC
+      end
+
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include:
+          - project: #{project.full_path}
+            file:
+              - #{local_location}
+              - #{other_file_location}
+
+        image: ruby:2.7
+        HEREDOC
+      end
+
+      before do
+        project.add_developer(user)
+
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:blob_data_at).with(an_instance_of(String), local_location)
+                                                     .and_return(local_file_content)
+
+          allow(repository).to receive(:blob_data_at).with(an_instance_of(String), other_file_location)
+                                                     .and_return(other_file_content)
+        end
+      end
+
+      it 'returns a composed hash' do
+        composed_hash = {
+          before_script: local_location_hash[:before_script],
+          image: "ruby:2.7",
+          build: { stage: "build", script: "echo hello" },
+          rspec: { stage: "test", script: "bundle exec rspec" }
+        }
+
+        expect(config.to_hash).to eq(composed_hash)
+      end
+    end
   end
 end

spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb

@@ -279,6 +279,40 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do
           end
         end
       end
+
+      context 'when specifying multiple files' do
+        let(:config) do
+          <<~YAML
+          test:
+            script: rspec
+          deploy:
+            variables:
+              CROSS: downstream
+            stage: deploy
+            trigger:
+              include:
+                - project: my-namespace/my-project
+                  file:
+                    - 'path/to/child1.yml'
+                    - 'path/to/child2.yml'
+          YAML
+        end
+
+        it_behaves_like 'successful creation' do
+          let(:expected_bridge_options) do
+            {
+              'trigger' => {
+                'include' => [
+                  {
+                    'file' => ["path/to/child1.yml", "path/to/child2.yml"],
+                    'project' => 'my-namespace/my-project'
+                  }
+                ]
+              }
+            }
+          end
+        end
+      end
     end
   end