Merge branch '209864-variables-in-pipeline-rules' into 'master'

Implement variables for pipeline job rules

See merge request gitlab-org/gitlab!48752

config/feature_flags/development/ci_rules_variables.yml

+---
+name: ci_rules_variables
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48752
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/289803
+milestone: '13.7'
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/README.md

@@ -989,6 +989,7 @@ The job attributes you can use with `rules` are:
 - [`when`](#when): If not defined, defaults to `when: on_success`.
   - If used as `when: delayed`, `start_in` is also required.
 - [`allow_failure`](#allow_failure): If not defined, defaults to `allow_failure: false`.
+- [`variables`](#rulesvariables): If not defined, uses the [variables defined elsewhere](#variables).
 
 If a rule evaluates to true, and `when` has any value except `never`, the job is included in the pipeline.
 
@@ -1410,6 +1411,56 @@ job:
 
 In this example, if the first rule matches, then the job has `when: manual` and `allow_failure: true`.
 
+#### `rules:variables`
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209864) in GitLab 13.7.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-rulesvariables). **(CORE ONLY)**
+
+WARNING:
+This feature might not be available to you. Check the **version history** note above for details.
+
+You can use [`variables`](#variables) in `rules:` to define variables for specific conditions.
+
+For example:
+
+```yaml
+job:
+  variables:
+    DEPLOY_VARIABLE: "default-deploy"
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /master/
+      variables:                              # Override DEPLOY_VARIABLE defined
+        DEPLOY_VARIABLE: "deploy-production"  # at the job level.
+    - if: $CI_COMMIT_REF_NAME =~ /feature/
+      variables:
+        IS_A_FEATURE: "true"                  # Define a new variable.
+  script:
+    - echo "Run script with $DEPLOY_VARIABLE as an argument"
+    - echo "Run another script if $IS_A_FEATURE exists"
+```
+
+##### Enable or disable rules:variables **(CORE ONLY)**
+
+rules:variables is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_rules_variables)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_rules_variables)
+```
+
 #### Complex rule clauses
 
 To conjoin `if`, `changes`, and `exists` clauses with an `AND`, use them in the

lib/gitlab/ci/build/rules.rb

@@ -6,18 +6,31 @@ module Gitlab
       class Rules
         include ::Gitlab::Utils::StrongMemoize
 
-        Result = Struct.new(:when, :start_in, :allow_failure) do
-          def build_attributes
+        Result = Struct.new(:when, :start_in, :allow_failure, :variables) do
+          def build_attributes(seed_attributes = {})
             {
               when: self.when,
               options: { start_in: start_in }.compact,
-              allow_failure: allow_failure
+              allow_failure: allow_failure,
+              yaml_variables: yaml_variables(seed_attributes[:yaml_variables])
             }.compact
           end
 
           def pass?
             self.when != 'never'
           end
+
+          private
+
+          def yaml_variables(seed_variables)
+            return unless variables && seed_variables
+
+            indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] }
+
+            variables.each_with_object(indexed_seed_variables) do |var, hash|
+              hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true }
+            end.values
+          end
         end
 
         def initialize(rule_hashes, default_when:)
@@ -32,7 +45,8 @@ module Gitlab
             Result.new(
               matched_rule.attributes[:when] || @default_when,
               matched_rule.attributes[:start_in],
-              matched_rule.attributes[:allow_failure]
+              matched_rule.attributes[:allow_failure],
+              matched_rule.attributes[:variables]
             )
           else
             Result.new('never')

lib/gitlab/ci/config/entry/rules/rule.rb

@@ -6,14 +6,18 @@ module Gitlab
       module Entry
         class Rules::Rule < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Configurable
           include ::Gitlab::Config::Entry::Attributable
 
           CLAUSES        = %i[if changes exists].freeze
-          ALLOWED_KEYS   = %i[if changes exists when start_in allow_failure].freeze
+          ALLOWED_KEYS   = %i[if changes exists when start_in allow_failure variables].freeze
           ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze
 
           attributes :if, :changes, :exists, :when, :start_in, :allow_failure
 
+          entry :variables, Entry::Variables,
+            description: 'Environment variables to define for rule conditions.'
+
           validations do
             validates :config, presence: true
             validates :config, type: { with: Hash }

lib/gitlab/ci/features.rb

@@ -66,6 +66,10 @@ module Gitlab
       def self.allow_failure_with_exit_codes_enabled?
         ::Feature.enabled?(:ci_allow_failure_with_exit_codes)
       end
+
+      def self.rules_variables_enabled?(project)
+        ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false)
+      end
     end
   end
 end

lib/gitlab/ci/pipeline/seed/build.rb

@@ -156,10 +156,12 @@ module Gitlab
 
           def rules_attributes
             strong_memoize(:rules_attributes) do
-              if @using_rules
-                rules_result.build_attributes
+              next {} unless @using_rules
+
+              if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project)
+                rules_result.build_attributes(@seed_attributes)
               else
-                {}
+                rules_result.build_attributes
               end
             end
           end

spec/lib/gitlab/ci/build/rules/rule/clause_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause do
+  describe '.fabricate' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:value) { 'some value' }
+
+    subject { described_class.fabricate(type, value) }
+
+    context 'when type is valid' do
+      where(:type, :result) do
+        'changes' | Gitlab::Ci::Build::Rules::Rule::Clause::Changes
+        'exists'  | Gitlab::Ci::Build::Rules::Rule::Clause::Exists
+        'if'      | Gitlab::Ci::Build::Rules::Rule::Clause::If
+      end
+
+      with_them do
+        it { is_expected.to be_instance_of(result) }
+      end
+    end
+
+    context 'when type is invalid' do
+      let(:type) { 'when' }
+
+      it { is_expected.to be_nil }
+
+      context "when type is 'variables'" do
+        let(:type) { 'variables' }
+
+        it { is_expected.to be_nil }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/build/rules_spec.rb

@@ -104,7 +104,7 @@ RSpec.describe Gitlab::Ci::Build::Rules do
     context 'with one rule without any clauses' do
       let(:rule_list) { [{ when: 'manual', allow_failure: true }] }
 
-      it { is_expected.to eq(described_class::Result.new('manual', nil, true)) }
+      it { is_expected.to eq(described_class::Result.new('manual', nil, true, nil)) }
     end
 
     context 'with one matching rule' do
@@ -171,7 +171,7 @@ RSpec.describe Gitlab::Ci::Build::Rules do
       context 'with matching rule' do
         let(:rule_list) { [{ if: '$VAR == null', allow_failure: true }] }
 
-        it { is_expected.to eq(described_class::Result.new('on_success', nil, true)) }
+        it { is_expected.to eq(described_class::Result.new('on_success', nil, true, nil)) }
       end
 
       context 'with non-matching rule' do
@@ -180,25 +180,61 @@ RSpec.describe Gitlab::Ci::Build::Rules do
         it { is_expected.to eq(described_class::Result.new('never')) }
       end
     end
+
+    context 'with variables' do
+      context 'with matching rule' do
+        let(:rule_list) { [{ if: '$VAR == null', variables: { MY_VAR: 'my var' } }] }
+
+        it { is_expected.to eq(described_class::Result.new('on_success', nil, nil, { MY_VAR: 'my var' })) }
+      end
+    end
   end
 
   describe 'Gitlab::Ci::Build::Rules::Result' do
     let(:when_value) { 'on_success' }
     let(:start_in) { nil }
     let(:allow_failure) { nil }
+    let(:variables) { nil }
 
     subject(:result) do
-      Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure)
+      Gitlab::Ci::Build::Rules::Result.new(when_value, start_in, allow_failure, variables)
     end
 
     describe '#build_attributes' do
+      let(:seed_attributes) { {} }
+
       subject(:build_attributes) do
-        result.build_attributes
+        result.build_attributes(seed_attributes)
       end
 
       it 'compacts nil values' do
         is_expected.to eq(options: {}, when: 'on_success')
       end
+
+      context 'when there are variables in rules' do
+        let(:variables) { { VAR1: 'new var 1', VAR3: 'var 3' } }
+
+        context 'when there are seed variables' do
+          let(:seed_attributes) do
+            { yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true },
+                               { key: 'VAR2', value: 'var 2', public: true }] }
+          end
+
+          it 'returns yaml_variables with override' do
+            is_expected.to include(
+              yaml_variables: [{ key: 'VAR1', value: 'new var 1', public: true },
+                               { key: 'VAR2', value: 'var 2', public: true },
+                               { key: 'VAR3', value: 'var 3', public: true }]
+            )
+          end
+        end
+
+        context 'when there is not seed variables' do
+          it 'does not return yaml_variables' do
+            is_expected.not_to have_key(:yaml_variables)
+          end
+        end
+      end
     end
 
     describe '#pass?' do
@@ -206,7 +242,7 @@ RSpec.describe Gitlab::Ci::Build::Rules do
         let!(:when_value) { 'never' }
 
         it 'returns false' do
-          expect(subject.pass?).to eq(false)
+          expect(result.pass?).to eq(false)
         end
       end
 
@@ -214,7 +250,7 @@ RSpec.describe Gitlab::Ci::Build::Rules do
         let!(:when_value) { 'on_success' }
 
         it 'returns true' do
-          expect(subject.pass?).to eq(true)
+          expect(result.pass?).to eq(true)
         end
       end
     end

spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb

@@ -339,6 +339,22 @@ RSpec.describe Gitlab::Ci::Config::Entry::Rules::Rule do
           end
         end
       end
+
+      context 'with an invalid variables' do
+        let(:config) do
+          { if: '$THIS == "that"', variables: 'hello' }
+        end
+
+        before do
+          subject.compose!
+        end
+
+        it { is_expected.not_to be_valid }
+
+        it 'returns an error about invalid variables:' do
+          expect(subject.errors).to include(/variables config should be a hash of key value pairs/)
+        end
+      end
     end
 
     context 'allow_failure: validation' do

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -71,6 +71,33 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
       end
     end
 
+    context 'with job:rules:[variables:]' do
+      let(:attributes) do
+        { name: 'rspec',
+          ref: 'master',
+          yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true },
+                           { key: 'VAR2', value: 'var 2', public: true }],
+          rules: [{ if: '$VAR == null', variables: { VAR1: 'new var 1', VAR3: 'var 3' } }] }
+      end
+
+      it do
+        is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'new var 1', public: true },
+                                                { key: 'VAR2', value: 'var 2', public: true },
+                                                { key: 'VAR3', value: 'var 3', public: true }])
+      end
+
+      context 'when FF ci_rules_variables is disabled' do
+        before do
+          stub_feature_flags(ci_rules_variables: false)
+        end
+
+        it do
+          is_expected.to include(yaml_variables: [{ key: 'VAR1', value: 'var 1', public: true },
+                                                  { key: 'VAR2', value: 'var 2', public: true }])
+        end
+      end
+    end
+
     context 'with cache:key' do
       let(:attributes) do
         {

spec/services/ci/create_pipeline_service/rules_spec.rb

@@ -160,6 +160,81 @@ RSpec.describe Ci::CreatePipelineService do
         end
       end
     end
+
+    context 'if:' do
+      context 'variables:' do
+        let(:config) do
+          <<-EOY
+          job:
+            script: "echo job1"
+            variables:
+              VAR1: my var 1
+              VAR2: my var 2
+            rules:
+              - if: $CI_COMMIT_REF_NAME =~ /master/
+                variables:
+                  VAR1: overridden var 1
+              - if: $CI_COMMIT_REF_NAME =~ /feature/
+                variables:
+                  VAR2: overridden var 2
+                  VAR3: new var 3
+              - when: on_success
+          EOY
+        end
+
+        let(:job) { pipeline.builds.find_by(name: 'job') }
+
+        context 'when matching to the first rule' do
+          let(:ref) { 'refs/heads/master' }
+
+          it 'overrides VAR1' do
+            variables = job.scoped_variables_hash
+
+            expect(variables['VAR1']).to eq('overridden var 1')
+            expect(variables['VAR2']).to eq('my var 2')
+            expect(variables['VAR3']).to be_nil
+          end
+
+          context 'when FF ci_rules_variables is disabled' do
+            before do
+              stub_feature_flags(ci_rules_variables: false)
+            end
+
+            it 'does not affect variables' do
+              variables = job.scoped_variables_hash
+
+              expect(variables['VAR1']).to eq('my var 1')
+              expect(variables['VAR2']).to eq('my var 2')
+              expect(variables['VAR3']).to be_nil
+            end
+          end
+        end
+
+        context 'when matching to the second rule' do
+          let(:ref) { 'refs/heads/feature' }
+
+          it 'overrides VAR2 and adds VAR3' do
+            variables = job.scoped_variables_hash
+
+            expect(variables['VAR1']).to eq('my var 1')
+            expect(variables['VAR2']).to eq('overridden var 2')
+            expect(variables['VAR3']).to eq('new var 3')
+          end
+        end
+
+        context 'when no match' do
+          let(:ref) { 'refs/heads/wip' }
+
+          it 'does not affect vars' do
+            variables = job.scoped_variables_hash
+
+            expect(variables['VAR1']).to eq('my var 1')
+            expect(variables['VAR2']).to eq('my var 2')
+            expect(variables['VAR3']).to be_nil
+          end
+        end
+      end
+    end
   end
 
   context 'when workflow:rules are used' do