Support variables in rules:changes

Add variable expansion to the rules:changes entries

config/feature_flags/development/ci_variable_expansion_in_rules_changes.yml

+---
+name: ci_variable_expansion_in_rules_changes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45037
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/267192
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/README.md

@@ -1347,6 +1347,53 @@ Tag pipelines, scheduled pipelines, and so on do **not** have a Git `push` event
 associated with them. A `rules: changes` job is **always** added to those pipeline
 if there is no `if:` statement that limits the job to branch or merge request pipelines.
 
+##### Variables in `rules:changes`
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34272) in GitLab 13.6.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-variables-support-in-ruleschanges). **(CORE ONLY)**
+
+CAUTION: **Warning:**
+This feature might not be available to you. Check the **version history** note above for details.
+
+Environment variables can be used in `rules:changes` expressions to determine when
+to add jobs to a pipeline:
+
+```yaml
+docker build:
+  variables:
+    DOCKERFILES_DIR: 'path/to/files/'
+  script: docker build -t my-image:$CI_COMMIT_REF_SLUG .
+  rules:
+    - changes:
+        - $DOCKERFILES_DIR/*
+```
+
+The `$` character can be used for both variables and paths. For example, if the
+`$DOCKERFILES_DIR` variable exists, its value is used. If it does not exist, the
+`$` is interpreted as being part of a path.
+
+###### Enable or disable variables support in `rules:changes` **(CORE ONLY)**
+
+Variables support in `rules:changes` is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_variable_expansion_in_rules_changes)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_variable_expansion_in_rules_changes)
+```
+
 #### `rules:exists`
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4.

lib/expand_variables.rb

 # frozen_string_literal: true
 
 module ExpandVariables
+  VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze
+
   class << self
     def expand(value, variables)
       variables_hash = nil
 
-      value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do
+      value.gsub(VARIABLES_REGEXP) do
         variables_hash ||= transform_variables(variables)
         variables_hash[Regexp.last_match(1) || Regexp.last_match(2)]
       end
     end
 
+    def expand_existing(value, variables)
+      variables_hash = nil
+
+      value.gsub(VARIABLES_REGEXP) do
+        variables_hash ||= transform_variables(variables)
+        variables_hash.fetch(
+          Regexp.last_match(1) || Regexp.last_match(2),
+          Regexp.last_match(0)
+        )
+      end
+    end
+
     private
 
     def transform_variables(variables)

lib/gitlab/ci/build/rules/rule/clause/changes.rb

@@ -11,12 +11,22 @@ module Gitlab
         def satisfied_by?(pipeline, context)
           return true if pipeline.modified_paths.nil?
 
+          expanded_globs = expand_globs(pipeline, context)
           pipeline.modified_paths.any? do |path|
-            @globs.any? do |glob|
+            expanded_globs.any? do |glob|
               File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
             end
           end
         end
+
+        def expand_globs(pipeline, context)
+          return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project)
+          return @globs unless context
+
+          @globs.map do |glob|
+            ExpandVariables.expand_existing(glob, context.variables)
+          end
+        end
       end
     end
   end

spec/lib/expand_variables_spec.rb

@@ -3,8 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe ExpandVariables do
-  describe '#expand' do
-    context 'table tests' do
+  shared_examples 'common variable expansion' do |expander|
     using RSpec::Parameterized::TableSyntax
 
     where do
@@ -14,11 +13,6 @@ RSpec.describe ExpandVariables do
           result: 'key',
           variables: []
         },
-          "missing variable": {
-            value: 'key$variable',
-            result: 'key',
-            variables: []
-          },
         "simple expansion": {
           value: 'key$variable',
           result: 'keyvalue',
@@ -56,13 +50,6 @@ RSpec.describe ExpandVariables do
             { key: 'variable2', value: 'result' }
           ]
         },
-          "complex expansions with missing variable": {
-            value: 'key${variable}${variable2}',
-            result: 'keyvalue',
-            variables: [
-              { key: 'variable', value: 'value' }
-            ]
-          },
         "out-of-order expansion": {
           value: 'key$variable2$variable',
           result: 'keyresultvalue',
@@ -99,12 +86,44 @@ RSpec.describe ExpandVariables do
       }
     end
 
+    with_them do
+      subject { expander.call(value, variables) }
+
+      it { is_expected.to eq(result) }
+    end
+  end
+
+  describe '#expand' do
+    context 'table tests' do
+      it_behaves_like 'common variable expansion', described_class.method(:expand)
+
+      context 'with missing variables' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "missing variable": {
+              value: 'key$variable',
+              result: 'key',
+              variables: []
+            },
+            "complex expansions with missing variable": {
+              value: 'key${variable}${variable2}',
+              result: 'keyvalue',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            }
+          }
+        end
+
         with_them do
           subject { ExpandVariables.expand(value, variables) }
 
           it { is_expected.to eq(result) }
         end
       end
+    end
 
     context 'lazily inits variables' do
       let(:variables) { -> { [{ key: 'variable', value: 'result' }] } }
@@ -132,4 +151,70 @@ RSpec.describe ExpandVariables do
       end
     end
   end
+
+  describe '#expand_existing' do
+    context 'table tests' do
+      it_behaves_like 'common variable expansion', described_class.method(:expand_existing)
+
+      context 'with missing variables' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "missing variable": {
+              value: 'key$variable',
+              result: 'key$variable',
+              variables: []
+            },
+            "complex expansions with missing variable": {
+              value: 'key${variable}${variable2}',
+              result: 'keyvalue${variable2}',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            },
+            "complex expansions with missing variable for Windows": {
+              value: 'key%variable%%variable2%',
+              result: 'keyvalue%variable2%',
+              variables: [
+                { key: 'variable', value: 'value' }
+              ]
+            }
+          }
+        end
+
+        with_them do
+          subject { ExpandVariables.expand_existing(value, variables) }
+
+          it { is_expected.to eq(result) }
+        end
+      end
+    end
+
+    context 'lazily inits variables' do
+      let(:variables) { -> { [{ key: 'variable', value: 'result' }] } }
+
+      subject { described_class.expand_existing(value, variables) }
+
+      context 'when expanding variable' do
+        let(:value) { 'key$variable$variable2' }
+
+        it 'calls block at most once' do
+          expect(variables).to receive(:call).once.and_call_original
+
+          is_expected.to eq('keyresult$variable2')
+        end
+      end
+
+      context 'when no expansion is needed' do
+        let(:value) { 'key' }
+
+        it 'does not call block' do
+          expect(variables).not_to receive(:call)
+
+          is_expected.to eq('key')
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb

@@ -13,5 +13,41 @@ RSpec.describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do
 
       subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
     end
+
+    context 'when using variable expansion' do
+      let(:pipeline) { build(:ci_pipeline) }
+      let(:modified_paths) { ['helm/test.txt'] }
+      let(:globs) { ['$HELM_DIR/**/*'] }
+      let(:context) { double('context') }
+      let(:variables) { [] }
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, context) }
+
+      before do
+        allow(pipeline).to receive(:modified_paths).and_return(modified_paths)
+        allow(context).to receive(:variables).and_return(variables)
+      end
+
+      context 'when context is nil' do
+        let(:context) {}
+
+        it { is_expected.to be_falsey }
+      end
+
+      context 'when context has the specified variables' do
+        let(:variables) do
+          [{ key: "HELM_DIR", value: "helm", public: true }]
+        end
+
+        it { is_expected.to be_truthy }
+      end
+
+      context 'when variable expansion does not match' do
+        let(:globs) { ['path/with/$in/it/*'] }
+        let(:modified_paths) { ['path/with/$in/it/file.txt'] }
+
+        it { is_expected.to be_truthy }
+      end
+    end
   end
 end

spec/services/ci/create_pipeline_service_spec.rb

@@ -1870,6 +1870,12 @@ RSpec.describe Ci::CreatePipelineService do
                 - changes:
                   - README.md
                   allow_failure: true
+
+            README:
+              script: "I use variables for changes!"
+              rules:
+                - changes:
+                  - $CI_JOB_NAME*
           EOY
         end
 
@@ -1879,10 +1885,10 @@ RSpec.describe Ci::CreatePipelineService do
               .to receive(:modified_paths).and_return(%w[README.md])
           end
 
-          it 'creates two jobs' do
+          it 'creates five jobs' do
             expect(pipeline).to be_persisted
             expect(build_names)
-              .to contain_exactly('regular-job', 'rules-job', 'delayed-job', 'negligible-job')
+              .to contain_exactly('regular-job', 'rules-job', 'delayed-job', 'negligible-job', 'README')
           end
 
           it 'sets when: for all jobs' do