Merge branch '337633-pipeline-variables-to-include' into 'master'

Add support of pipeline variables to include

See merge request gitlab-org/gitlab!72968

doc/ci/yaml/index.md

@@ -422,22 +422,36 @@ configurations. Local configurations in the `.gitlab-ci.yml` file override inclu
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/284883) in GitLab 13.8.
 > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/294294) in GitLab 13.9.
 > - [Support for project, group, and instance variables added](https://gitlab.com/gitlab-org/gitlab/-/issues/219065) in GitLab 14.2.
+> - [Support for pipeline variables added](https://gitlab.com/gitlab-org/gitlab/-/issues/337633) in GitLab 14.5.
 
 In `include` sections in your `.gitlab-ci.yml` file, you can use:
 
-- `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md) in GitLab 14.2
-  and later.
-  - When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full
-    ref path, like `refs/heads/branch-name`. In other cases, this variable returns only
-    the branch name, like `branch-name`.
-
-    To use `CI_COMMIT_REF_NAME` in `include:rules`, you might need to use `if: $CI_COMMIT_REF_NAME =~ /main/`
-    (not `== main`). [An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/337633)
-    to align `CI_COMMIT_REF_NAME` behavior in all cases.
 - [Project variables](../variables/index.md#add-a-cicd-variable-to-a-project)
 - [Group variables](../variables/index.md#add-a-cicd-variable-to-a-group)
 - [Instance variables](../variables/index.md#add-a-cicd-variable-to-an-instance)
-- Project [predefined variables](../variables/predefined_variables.md).
+- Project [predefined variables](../variables/predefined_variables.md)
+- In GitLab 14.2 and later, the `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md).
+
+  When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full
+  ref path, like `refs/heads/branch-name`. In `include:rules`, you might need to use
+  `if: $CI_COMMIT_REF_NAME =~ /main/` (not `== main`). This behavior is resolved in GitLab 14.5.
+
+In GitLab 14.5 and later, you can also use:
+
+- [Trigger variables](../triggers/index.md#making-use-of-trigger-variables).
+- [Scheduled pipeline variables](../pipelines/schedules.md#using-variables).
+- [Manual pipeline run variables](../variables/index.md#override-a-variable-when-running-a-pipeline-manually).
+- Pipeline [predefined variables](../variables/predefined_variables.md).
+
+  YAML files are parsed before the pipeline is created, so the following pipeline predefined variables
+  are **not** available:
+
+  - `CI_PIPELINE_ID`
+  - `CI_PIPELINE_URL`
+  - `CI_PIPELINE_IID`
+  - `CI_PIPELINE_CREATED_AT`
+
+For example:
 
 ```yaml
 include:
@@ -448,9 +462,6 @@ include:
 For an example of how you can include these predefined variables, and the variables' impact on CI/CD jobs,
 see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos).
 
-There is a [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337633)
-that proposes expanding this feature to support more variables.
-
 #### `rules` with `include`
 
 > - Introduced in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default.
@@ -468,6 +479,9 @@ include:
   - local: builds.yml
     rules:
       - if: '$INCLUDE_BUILDS == "true"'
+  - local: deploys.yml
+    rules:
+      - if: $CI_COMMIT_BRANCH == "main"
 
 test:
   stage: test

ee/spec/lib/ee/gitlab/ci/config_spec.rb

@@ -40,14 +40,16 @@ RSpec.describe Gitlab::Ci::Config do
   describe 'with security orchestration policy' do
     let(:source) { 'push' }
 
-    let_it_be(:ref) { 'refs/heads/master' }
+    let_it_be(:ref) { 'master' }
     let_it_be_with_refind(:project) { create(:project, :repository) }
 
     let_it_be(:policies_repository) { create(:project, :repository) }
     let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, project: project, security_policy_management_project: policies_repository) }
     let_it_be(:policy_yaml) { build(:orchestration_policy_yaml, scan_execution_policy: [build(:scan_execution_policy)]) }
 
-    subject(:config) { described_class.new(ci_yml, source_ref_path: ref, project: project, source: source) }
+    let(:pipeline) { build(:ci_pipeline, project: project, ref: ref) }
+
+    subject(:config) { described_class.new(ci_yml, pipeline: pipeline, project: project, source: source) }
 
     before do
       allow_next_instance_of(Repository) do |repository|
@@ -74,7 +76,7 @@ RSpec.describe Gitlab::Ci::Config do
       end
 
       context 'when policy is not applicable on branch from the pipeline' do
-        let_it_be(:ref) { 'refs/heads/production' }
+        let_it_be(:ref) { 'production' }
 
         context 'when DAST profiles are not found' do
           it 'adds a job with error message' do

lib/gitlab/ci/config.rb

@@ -19,11 +19,12 @@ module Gitlab
 
       attr_reader :root, :context, :source_ref_path, :source
 
-      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil)
-        @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path)
+      def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil)
+        @source_ref_path = pipeline&.source_ref_path
+
+        @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
         @context.set_deadline(TIMEOUT_SECONDS)
 
-        @source_ref_path = source_ref_path
         @source = source
 
         @config = expand_config(config)
@@ -108,16 +109,16 @@ module Gitlab
         end
       end
 
-      def build_context(project:, sha:, user:, parent_pipeline:, ref:)
+      def build_context(project:, pipeline:, sha:, user:, parent_pipeline:)
         Config::External::Context.new(
           project: project,
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: build_variables(project: project, ref: ref))
+          variables: build_variables(project: project, pipeline: pipeline))
       end
 
-      def build_variables(project:, ref:)
+      def build_variables(project:, pipeline:)
         Gitlab::Ci::Variables::Collection.new.tap do |variables|
           break variables unless project
 
@@ -126,18 +127,12 @@ module Gitlab
           #
           # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
           variables.concat(project.predefined_variables)
-          variables.concat(pipeline_predefined_variables(ref: ref))
-          variables.concat(project.ci_instance_variables_for(ref: ref))
-          variables.concat(project.group.ci_variables_for(ref, project)) if project.group
-          variables.concat(project.ci_variables_for(ref: ref))
-        end
-      end
-
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables
-      # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines.
-      def pipeline_predefined_variables(ref:)
-        Gitlab::Ci::Variables::Collection.new.tap do |v|
-          v.append(key: 'CI_COMMIT_REF_NAME', value: ref)
+          variables.concat(pipeline.predefined_variables) if pipeline
+          variables.concat(project.ci_instance_variables_for(ref: source_ref_path))
+          variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group
+          variables.concat(project.ci_variables_for(ref: source_ref_path))
+          variables.concat(pipeline.variables) if pipeline
+          variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule
         end
       end
 

lib/gitlab/ci/pipeline/chain/config/process.rb

@@ -14,7 +14,7 @@ module Gitlab
               result = ::Gitlab::Ci::YamlProcessor.new(
                 @command.config_content, {
                   project: project,
-                  source_ref_path: @pipeline.source_ref_path,
+                  pipeline: @pipeline,
                   sha: @pipeline.sha,
                   source: @pipeline.source,
                   user: current_user,

spec/lib/gitlab/ci/config_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe Gitlab::Ci::Config do
   end
 
   let(:config) do
-    described_class.new(yml, project: nil, sha: nil, user: nil)
+    described_class.new(yml, project: nil, pipeline: nil, sha: nil, user: nil)
   end
 
   context 'when config is valid' do
@@ -286,9 +286,12 @@ RSpec.describe Gitlab::Ci::Config do
   end
 
   context "when using 'include' directive" do
-    let(:group) { create(:group) }
+    let_it_be(:group) { create(:group) }
+
     let(:project) { create(:project, :repository, group: group) }
     let(:main_project) { create(:project, :repository, :public, group: group) }
+    let(:pipeline) { build(:ci_pipeline, project: project) }
+
     let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
     let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
 
@@ -327,7 +330,7 @@ RSpec.describe Gitlab::Ci::Config do
     end
 
     let(:config) do
-      described_class.new(gitlab_ci_yml, project: project, sha: '12345', user: user)
+      described_class.new(gitlab_ci_yml, project: project, pipeline: pipeline, sha: '12345', user: user)
     end
 
     before do
@@ -724,7 +727,7 @@ RSpec.describe Gitlab::Ci::Config do
       end
     end
 
-    context "when an 'include' has rules" do
+    context "when an 'include' has rules with a project variable" do
       let(:gitlab_ci_yml) do
         <<~HEREDOC
         include:
@@ -751,5 +754,30 @@ RSpec.describe Gitlab::Ci::Config do
         end
       end
     end
+
+    context "when an 'include' has rules with a pipeline variable" do
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include:
+          - local: #{local_location}
+            rules:
+              - if: $CI_COMMIT_SHA == "#{project.commit.sha}"
+        HEREDOC
+      end
+
+      context 'when a pipeline is passed' do
+        it 'includes the file' do
+          expect(config.to_hash).to include(local_location_hash)
+        end
+      end
+
+      context 'when a pipeline is not passed' do
+        let(:pipeline) { nil }
+
+        it 'does not include the file' do
+          expect(config.to_hash).not_to include(local_location_hash)
+        end
+      end
+    end
   end
 end

spec/services/ci/create_pipeline_service/include_spec.rb

@@ -7,9 +7,11 @@ RSpec.describe Ci::CreatePipelineService do
     let_it_be(:project) { create(:project, :repository) }
     let_it_be(:user)    { project.owner }
 
-    let(:ref)      { 'refs/heads/master' }
-    let(:source)   { :push }
-    let(:service)  { described_class.new(project, user, { ref: ref }) }
+    let(:ref)                  { 'refs/heads/master' }
+    let(:variables_attributes) { [{ key: 'MYVAR', secret_value: 'hello' }] }
+    let(:source)               { :push }
+
+    let(:service)  { described_class.new(project, user, { ref: ref, variables_attributes: variables_attributes }) }
     let(:pipeline) { service.execute(source).payload }
 
     let(:file_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
@@ -24,6 +26,20 @@ RSpec.describe Ci::CreatePipelineService do
         .and_return(File.read(Rails.root.join(file_location)))
     end
 
+    shared_examples 'not including the file' do
+      it 'does not include the job in the file' do
+        expect(pipeline).to be_created_successfully
+        expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
+      end
+    end
+
+    shared_examples 'including the file' do
+      it 'includes the job in the file' do
+        expect(pipeline).to be_created_successfully
+        expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
+      end
+    end
+
     context 'with a local file' do
       let(:config) do
         <<~EOY
@@ -33,13 +49,10 @@ RSpec.describe Ci::CreatePipelineService do
         EOY
       end
 
-      it 'includes the job in the file' do
-        expect(pipeline).to be_created_successfully
-        expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
-      end
+      it_behaves_like 'including the file'
     end
 
-    context 'with a local file with rules' do
+    context 'with a local file with rules with a project variable' do
       let(:config) do
         <<~EOY
         include:
@@ -54,19 +67,63 @@ RSpec.describe Ci::CreatePipelineService do
       context 'when the rules matches' do
         let(:project_id) { project.id }
 
-        it 'includes the job in the file' do
-          expect(pipeline).to be_created_successfully
-          expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
-        end
+        it_behaves_like 'including the file'
       end
 
       context 'when the rules does not match' do
         let(:project_id) { non_existing_record_id }
 
-        it 'does not include the job in the file' do
-          expect(pipeline).to be_created_successfully
-          expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
-        end
+        it_behaves_like 'not including the file'
+      end
+    end
+
+    context 'with a local file with rules with a predefined pipeline variable' do
+      let(:config) do
+        <<~EOY
+        include:
+          - local: #{file_location}
+            rules:
+              - if: $CI_PIPELINE_SOURCE == "#{pipeline_source}"
+        job:
+          script: exit 0
+        EOY
+      end
+
+      context 'when the rules matches' do
+        let(:pipeline_source) { 'push' }
+
+        it_behaves_like 'including the file'
+      end
+
+      context 'when the rules does not match' do
+        let(:pipeline_source) { 'web' }
+
+        it_behaves_like 'not including the file'
+      end
+    end
+
+    context 'with a local file with rules with a run pipeline variable' do
+      let(:config) do
+        <<~EOY
+        include:
+          - local: #{file_location}
+            rules:
+              - if: $MYVAR == "#{my_var}"
+        job:
+          script: exit 0
+        EOY
+      end
+
+      context 'when the rules matches' do
+        let(:my_var) { 'hello' }
+
+        it_behaves_like 'including the file'
+      end
+
+      context 'when the rules does not match' do
+        let(:my_var) { 'mello' }
+
+        it_behaves_like 'not including the file'
       end
     end
   end