Merge branch 'support-variables-include-section' into 'master'

Support for variables in include section of gitlab-ci.yml

See merge request gitlab-org/gitlab!50188

config/feature_flags/development/variables_in_include_section_ci.yml

+---
+name: variables_in_include_section_ci
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50188/
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/294294
+milestone: '13.8'
+type: development
+group: group::compliance
+default_enabled: false

lib/gitlab/ci/config.rb

@@ -98,7 +98,8 @@ module Gitlab
           project: project,
           sha: sha || project&.repository&.root_ref_sha,
           user: user,
-          parent_pipeline: parent_pipeline)
+          parent_pipeline: parent_pipeline,
+          variables: project&.predefined_variables&.to_runner_variables)
       end
 
       def track_and_raise_for_dev_exception(error)

lib/gitlab/ci/config/external/context.rb

@@ -7,14 +7,15 @@ module Gitlab
         class Context
           TimeoutError = Class.new(StandardError)
 
-          attr_reader :project, :sha, :user, :parent_pipeline
+          attr_reader :project, :sha, :user, :parent_pipeline, :variables
           attr_reader :expandset, :execution_deadline
 
-          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil)
+          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: [])
             @project = project
             @sha = sha
             @user = user
             @parent_pipeline = parent_pipeline
+            @variables = variables
             @expandset = Set.new
             @execution_deadline = 0
 

lib/gitlab/ci/config/external/file/local.rb

@@ -41,7 +41,8 @@ module Gitlab
                 project: context.project,
                 sha: context.sha,
                 user: context.user,
-                parent_pipeline: context.parent_pipeline
+                parent_pipeline: context.parent_pipeline,
+                variables: context.variables
               }
             end
           end

lib/gitlab/ci/config/external/file/project.rb

@@ -72,7 +72,8 @@ module Gitlab
                 project: project,
                 sha: sha,
                 user: context.user,
-                parent_pipeline: context.parent_pipeline
+                parent_pipeline: context.parent_pipeline,
+                variables: context.variables
               }
             end
           end

lib/gitlab/ci/config/external/mapper.rb

@@ -34,6 +34,7 @@ module Gitlab
               .compact
               .map(&method(:normalize_location))
               .flat_map(&method(:expand_project_files))
+              .map(&method(:expand_variables))
               .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
           end
@@ -47,7 +48,8 @@ module Gitlab
           # convert location if String to canonical form
           def normalize_location(location)
             if location.is_a?(String)
-              normalize_location_string(location)
+              expanded_location = expand_variables(location)
+              normalize_location_string(expanded_location)
             else
               location.deep_symbolize_keys
             end
@@ -96,6 +98,33 @@ module Gitlab
 
             matching.first
           end
+
+          def expand_variables(data)
+            return data unless ::Feature.enabled?(:variables_in_include_section_ci)
+
+            if data.is_a?(String)
+              expand(data)
+            else
+              transform(data)
+            end
+          end
+
+          def transform(data)
+            data.transform_values do |values|
+              case values
+              when Array
+                values.map { |value| expand(value.to_s) }
+              when String
+                expand(values)
+              else
+                values
+              end
+            end
+          end
+
+          def expand(data)
+            ExpandVariables.expand(data, context.variables)
+          end
         end
       end
     end

spec/helpers/blob_helper_spec.rb

@@ -94,7 +94,7 @@ RSpec.describe BlobHelper do
   context 'viewer related' do
     include FakeBlobHelpers
 
-    let(:project) { build(:project, lfs_enabled: true) }
+    let_it_be(:project) { create(:project, lfs_enabled: true) }
 
     before do
       allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)

spec/lib/gitlab/ci/config/external/file/local_spec.rb

@@ -16,7 +16,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do
       project: project,
       sha: sha,
       user: user,
-      parent_pipeline: parent_pipeline
+      parent_pipeline: parent_pipeline,
+      variables: project.predefined_variables.to_runner_variables
     }
   end
 
@@ -131,7 +132,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Local do
         user: user,
         project: project,
         sha: sha,
-        parent_pipeline: parent_pipeline)
+        parent_pipeline: parent_pipeline,
+        variables: project.predefined_variables.to_runner_variables)
     end
   end
 

spec/lib/gitlab/ci/config/external/file/project_spec.rb

@@ -16,7 +16,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
       project: context_project,
       sha: '12345',
       user: context_user,
-      parent_pipeline: parent_pipeline
+      parent_pipeline: parent_pipeline,
+      variables: project.predefined_variables.to_runner_variables
     }
   end
 
@@ -165,7 +166,8 @@ RSpec.describe Gitlab::Ci::Config::External::File::Project do
         user: user,
         project: project,
         sha: project.commit('master').id,
-        parent_pipeline: parent_pipeline)
+        parent_pipeline: parent_pipeline,
+        variables: project.predefined_variables.to_runner_variables)
     end
   end
 

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
   let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' }
   let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
   let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' }
-  let(:context_params) { { project: project, sha: '123456', user: user } }
+  let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables.to_runner_variables } }
   let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) }
 
   let(:file_content) do
@@ -236,5 +236,118 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
         end
       end
     end
+
+    context "when 'include' section uses project variable" do
+      let(:full_local_file_path) { '$CI_PROJECT_PATH' + local_file }
+
+      context 'when local file is included as a single string' do
+        let(:values) do
+          { include: full_local_file_path }
+        end
+
+        it 'expands the variable', :aggregate_failures do
+          expect(subject[0].location).to eq(project.full_path + local_file)
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Local))
+        end
+      end
+
+      context 'when remote file is included as a single string' do
+        let(:remote_url) { "#{Gitlab.config.gitlab.url}/radio/.gitlab-ci.yml" }
+
+        let(:values) do
+          { include: '$CI_SERVER_URL/radio/.gitlab-ci.yml' }
+        end
+
+        it 'expands the variable', :aggregate_failures do
+          expect(subject[0].location).to eq(remote_url)
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote))
+        end
+      end
+
+      context 'defined as an array' do
+        let(:values) do
+          { include: [full_local_file_path, remote_url],
+            image: 'ruby:2.7' }
+        end
+
+        it 'expands the variable' do
+          expect(subject[0].location).to eq(project.full_path + local_file)
+          expect(subject[1].location).to eq(remote_url)
+        end
+      end
+
+      context 'defined as an array of hashes' do
+        let(:values) do
+          { include: [{ local: full_local_file_path }, { remote: remote_url }],
+            image: 'ruby:2.7' }
+        end
+
+        it 'expands the variable' do
+          expect(subject[0].location).to eq(project.full_path + local_file)
+          expect(subject[1].location).to eq(remote_url)
+        end
+      end
+
+      context 'local file hash' do
+        let(:values) do
+          { include: { 'local' => full_local_file_path } }
+        end
+
+        it 'expands the variable' do
+          expect(subject[0].location).to eq(project.full_path + local_file)
+        end
+      end
+
+      context 'project name' do
+        let(:values) do
+          { include: { project: '$CI_PROJECT_PATH', file: local_file },
+            image: 'ruby:2.7' }
+        end
+
+        it 'expands the variable', :aggregate_failures do
+          expect(subject[0].project_name).to eq(project.full_path)
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Project))
+        end
+      end
+
+      context 'with multiple files' do
+        let(:values) do
+          { include: { project: project.full_path, file: [full_local_file_path, 'another_file_path.yml'] },
+            image: 'ruby:2.7' }
+        end
+
+        it 'expands the variable' do
+          expect(subject[0].location).to eq(project.full_path + local_file)
+          expect(subject[1].location).to eq('another_file_path.yml')
+        end
+      end
+
+      context 'when include variable has an unsupported type for variable expansion' do
+        let(:values) do
+          { include: { project: project.id, file: local_file },
+            image: 'ruby:2.7' }
+        end
+
+        it 'does not invoke expansion for the variable', :aggregate_failures do
+          expect(ExpandVariables).not_to receive(:expand).with(project.id, context_params[:variables])
+
+          expect { subject }.to raise_error(described_class::AmbigiousSpecificationError)
+        end
+      end
+
+      context 'when feature flag is turned off' do
+        let(:values) do
+          { include: full_local_file_path }
+        end
+
+        before do
+          stub_feature_flags(variables_in_include_section_ci: false)
+        end
+
+        it 'does not expand the variables' do
+          expect(subject[0].location).to eq('$CI_PROJECT_PATH' + local_file)
+        end
+      end
+    end
   end
 end