Merge branch '276515-include-rules' into 'master'

Add rules support for CI pipeline include

See merge request gitlab-org/gitlab!67409

config/feature_flags/development/ci_include_rules.yml

+---
+name: ci_include_rules
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67409
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337507
+milestone: '14.2'
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/index.md

@@ -436,6 +436,32 @@ include:
 For an example of how you can include these predefined variables, and the variables' impact on CI/CD jobs,
 see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos).
 
+There is a [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337633)
+that proposes expanding this feature to support more variables.
+
+#### `rules` with `include`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276515) in GitLab 14.2.
+
+NOTE:
+On self-managed GitLab, by default this feature is not available. To make it available,
+ask an administrator to [enable the `ci_include_rules` flag](../../administration/feature_flags.md).
+On GitLab.com, this feature is not available. The feature is not ready for production use.
+
+You can use [`rules`](#rules) with `include` to conditionally include other configuration files.
+You can only use `rules:if` in `include` with [certain variables](#variables-with-include).
+
+```yaml
+include:
+  - local: builds.yml
+    rules:
+      - if: '$INCLUDE_BUILDS == "true"'
+
+test:
+  stage: test
+  script: exit 0
+```
+
 #### `include:local`
 
 Use `include:local` to include a file that is in the same repository as the `.gitlab-ci.yml` file.

lib/gitlab/ci/config/entry/include.rb

@@ -9,8 +9,10 @@ module Gitlab
         #
         class Include < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze
+          ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze
 
           validations do
             validates :config, hash_or_string: true
@@ -27,6 +29,20 @@ module Gitlab
                 errors.add(:config, "must specify the file where to fetch the config from")
               end
             end
+
+            with_options allow_nil: true do
+              validates :rules, array_of_hashes: true
+            end
+          end
+
+          entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules,
+              description: 'List of evaluable Rules to determine file inclusion.',
+              inherit: false
+
+          attributes :rules
+
+          def skip_config_hash_validation?
+            true
           end
         end
       end

lib/gitlab/ci/config/entry/include/rules.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Include
+          class Rules < ::Gitlab::Config::Entry::ComposableArray
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations do
+              validates :config, presence: true
+              validates :config, type: Array
+            end
+
+            def value
+              @config
+            end
+
+            def composable_class
+              Entry::Include::Rules::Rule
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/include/rules/rule.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Include
+          class Rules::Rule < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
+
+            ALLOWED_KEYS = %i[if].freeze
+
+            attributes :if
+
+            validations do
+              validates :config, presence: true
+              validates :config, type: { with: Hash }
+              validates :config, allowed_keys: ALLOWED_KEYS
+
+              with_options allow_nil: true do
+                validates :if, expression: true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/external/mapper.rb

@@ -33,6 +33,7 @@ module Gitlab
             locations
               .compact
               .map(&method(:normalize_location))
+              .filter_map(&method(:verify_rules))
               .flat_map(&method(:expand_project_files))
               .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
@@ -56,6 +57,15 @@ module Gitlab
             end
           end
 
+          def verify_rules(location)
+            # Behaves like there is no `rules`
+            return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml)
+
+            return unless Rules.new(location[:rules]).evaluate(context).pass?
+
+            location
+          end
+
           def expand_project_files(location)
             return location unless location[:project]
 

lib/gitlab/ci/config/external/rules.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Rules
+          def initialize(rule_hashes)
+            @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes)
+          end
+
+          def evaluate(context)
+            Result.new(@rule_list.nil? || match_rule(context))
+          end
+
+          private
+
+          def match_rule(context)
+            @rule_list.find { |rule| rule.matches?(nil, context) }
+          end
+
+          Result = Struct.new(:result) do
+            def pass?
+              !!result
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/include/rules/rule_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules::Rule do
+  let(:factory) do
+    Gitlab::Config::Entry::Factory.new(described_class)
+      .value(config)
+  end
+
+  subject(:entry) { factory.create! }
+
+  describe '.new' do
+    shared_examples 'an invalid config' do |error_message|
+      it { is_expected.not_to be_valid }
+
+      it 'has errors' do
+        expect(entry.errors).to include(error_message)
+      end
+    end
+
+    context 'when specifying an if: clause' do
+      let(:config) { { if: '$THIS || $THAT' } }
+
+      it { is_expected.to be_valid }
+    end
+
+    context 'using a list of multiple expressions' do
+      let(:config) { { if: ['$MY_VAR == "this"', '$YOUR_VAR == "that"'] } }
+
+      it_behaves_like 'an invalid config', /invalid expression syntax/
+    end
+
+    context 'when specifying an invalid if: clause expression' do
+      let(:config) { { if: ['$MY_VAR =='] } }
+
+      it_behaves_like 'an invalid config', /invalid expression syntax/
+    end
+
+    context 'when specifying an if: clause expression with an invalid token' do
+      let(:config) { { if: ['$MY_VAR == 123'] } }
+
+      it_behaves_like 'an invalid config', /invalid expression syntax/
+    end
+
+    context 'when using invalid regex in an if: clause' do
+      let(:config) { { if: ['$MY_VAR =~ /some ( thing/'] } }
+
+      it_behaves_like 'an invalid config', /invalid expression syntax/
+    end
+
+    context 'when using an if: clause with lookahead regex character "?"' do
+      let(:config) { { if: '$CI_COMMIT_REF =~ /^(?!master).+/' } }
+
+      context 'when allow_unsafe_ruby_regexp is disabled' do
+        it_behaves_like 'an invalid config', /invalid expression syntax/
+      end
+    end
+
+    context 'when specifying unknown policy' do
+      let(:config) { { invalid: :something } }
+
+      it_behaves_like 'an invalid config', /unknown keys: invalid/
+    end
+
+    context 'when clause is empty' do
+      let(:config) { {} }
+
+      it_behaves_like 'an invalid config', /can't be blank/
+    end
+
+    context 'when policy strategy does not match' do
+      let(:config) { 'string strategy' }
+
+      it_behaves_like 'an invalid config', /should be a hash/
+    end
+  end
+
+  describe '#value' do
+    subject(:value) { entry.value }
+
+    context 'when specifying an if: clause' do
+      let(:config) { { if: '$THIS || $THAT' } }
+
+      it 'returns the config' do
+        expect(subject).to eq(if: '$THIS || $THAT')
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/include/rules_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::Entry::Include::Rules do
+  let(:factory) do
+    Gitlab::Config::Entry::Factory.new(described_class)
+      .value(config)
+  end
+
+  subject(:entry) { factory.create! }
+
+  describe '.new' do
+    shared_examples 'a valid config' do
+      it { is_expected.to be_valid }
+
+      context 'when composed' do
+        before do
+          entry.compose!
+        end
+
+        it { is_expected.to be_valid }
+      end
+    end
+
+    shared_examples 'an invalid config' do |error_message|
+      it { is_expected.not_to be_valid }
+
+      it 'has errors' do
+        expect(entry.errors).to include(error_message)
+      end
+    end
+
+    context 'with an "if"' do
+      let(:config) do
+        [{ if: '$THIS == "that"' }]
+      end
+
+      it_behaves_like 'a valid config'
+    end
+
+    context 'with a "changes"' do
+      let(:config) do
+        [{ changes: ['filename.txt'] }]
+      end
+
+      context 'when composed' do
+        before do
+          entry.compose!
+        end
+
+        it_behaves_like 'an invalid config', /contains unknown keys: changes/
+      end
+    end
+
+    context 'with a list of two rules' do
+      let(:config) do
+        [
+          { if: '$THIS == "that"' },
+          { if: '$SKIP' }
+        ]
+      end
+
+      it_behaves_like 'a valid config'
+    end
+
+    context 'without an array' do
+      let(:config) do
+        { if: '$SKIP' }
+      end
+
+      it_behaves_like 'an invalid config', /should be a array/
+    end
+  end
+
+  describe '#value' do
+    subject(:value) { entry.value }
+
+    context 'with an "if"' do
+      let(:config) do
+        [{ if: '$THIS == "that"' }]
+      end
+
+      it { is_expected.to eq(config) }
+    end
+
+    context 'with a list of two rules' do
+      let(:config) do
+        [
+          { if: '$THIS == "that"' },
+          { if: '$SKIP' }
+        ]
+      end
+
+      it { is_expected.to eq(config) }
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/include_spec.rb

 # frozen_string_literal: true
 
-require 'spec_helper'
+require 'fast_spec_helper'
 
 RSpec.describe ::Gitlab::Ci::Config::Entry::Include do
   subject(:include_entry) { described_class.new(config) }
@@ -86,6 +86,22 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Include do
           end
         end
       end
+
+      context 'when using with "rules"' do
+        let(:config) { { local: 'test.yml', rules: [{ if: '$VARIABLE' }] } }
+
+        it { is_expected.to be_valid }
+
+        context 'when rules is not an array of hashes' do
+          let(:config) { { local: 'test.yml', rules: ['$VARIABLE'] } }
+
+          it { is_expected.not_to be_valid }
+
+          it 'has specific error' do
+            expect(include_entry.errors).to include('include rules should be an array of hashes')
+          end
+        end
+      end
     end
 
     context 'when value is something else' do
@@ -94,4 +110,26 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Include do
       it { is_expected.not_to be_valid }
     end
   end
+
+  describe '#value' do
+    subject(:value) { include_entry.value }
+
+    context 'when config is a string' do
+      let(:config) { 'test.yml' }
+
+      it { is_expected.to eq('test.yml') }
+    end
+
+    context 'when config is a hash' do
+      let(:config) { { local: 'test.yml' } }
+
+      it { is_expected.to eq(local: 'test.yml') }
+    end
+
+    context 'when config has "rules"' do
+      let(:config) { { local: 'test.yml', rules: [{ if: '$VARIABLE' }] } }
+
+      it { is_expected.to eq(local: 'test.yml', rules: [{ if: '$VARIABLE' }]) }
+    end
+  end
 end

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -11,7 +11,7 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
   let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' }
   let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
   let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' }
-  let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables.to_runner_variables } }
+  let(:context_params) { { project: project, sha: '123456', user: user, variables: project.predefined_variables } }
   let(:context) { Gitlab::Ci::Config::External::Context.new(**context_params) }
 
   let(:file_content) do
@@ -348,5 +348,52 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
         expect(subject.map(&:location)).to contain_exactly('myfolder/file1.yml', 'myfolder/file2.yml')
       end
     end
+
+    context "when 'include' has rules" do
+      let(:values) do
+        { include: [{ remote: remote_url },
+                    { local: local_file, rules: [{ if: "$CI_PROJECT_ID == '#{project_id}'" }] }],
+          image: 'ruby:2.7' }
+      end
+
+      context 'when the rules matches' do
+        let(:project_id) { project.id }
+
+        it 'includes the file' do
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote),
+                                             an_instance_of(Gitlab::Ci::Config::External::File::Local))
+        end
+
+        context 'when the FF ci_include_rules is disabled' do
+          before do
+            stub_feature_flags(ci_include_rules: false)
+          end
+
+          it 'includes the file' do
+            expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote),
+                                               an_instance_of(Gitlab::Ci::Config::External::File::Local))
+          end
+        end
+      end
+
+      context 'when the rules does not match' do
+        let(:project_id) { non_existing_record_id }
+
+        it 'does not include the file' do
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote))
+        end
+
+        context 'when the FF ci_include_rules is disabled' do
+          before do
+            stub_feature_flags(ci_include_rules: false)
+          end
+
+          it 'includes the file' do
+            expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Remote),
+                                               an_instance_of(Gitlab::Ci::Config::External::File::Local))
+          end
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/external/rules_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::External::Rules do
+  let(:rule_hashes) {}
+
+  subject(:rules) { described_class.new(rule_hashes) }
+
+  describe '#evaluate' do
+    let(:context) { double(variables: {}) }
+
+    subject(:result) { rules.evaluate(context).pass? }
+
+    context 'when there is no rule' do
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when there is a rule' do
+      let(:rule_hashes) { [{ if: '$MY_VAR == "hello"' }] }
+
+      context 'when the rule matches' do
+        let(:context) { double(variables: { MY_VAR: 'hello' }) }
+
+        it { is_expected.to eq(true) }
+      end
+
+      context 'when the rule does not match' do
+        let(:context) { double(variables: { MY_VAR: 'invalid' }) }
+
+        it { is_expected.to eq(false) }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config_spec.rb

@@ -723,5 +723,33 @@ RSpec.describe Gitlab::Ci::Config do
         expect(config.to_hash).to eq(composed_hash)
       end
     end
+
+    context "when an 'include' has rules" do
+      let(:gitlab_ci_yml) do
+        <<~HEREDOC
+        include:
+          - local: #{local_location}
+            rules:
+              - if: $CI_PROJECT_ID == "#{project_id}"
+        image: ruby:2.7
+        HEREDOC
+      end
+
+      context 'when the rules condition is satisfied' do
+        let(:project_id) { project.id }
+
+        it 'includes the file' do
+          expect(config.to_hash).to include(local_location_hash)
+        end
+      end
+
+      context 'when the rules condition is satisfied' do
+        let(:project_id) { non_existing_record_id }
+
+        it 'does not include the file' do
+          expect(config.to_hash).not_to include(local_location_hash)
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -1152,6 +1152,10 @@ module Gitlab
             end
 
             it { is_expected.to be_valid }
+
+            it 'adds the job from the included file' do
+              expect(subject.builds.map { |build| build[:name] }).to contain_exactly('job1', 'rspec')
+            end
           end
 
           context "when the included internal file is not present" do

spec/services/ci/create_pipeline_service/include_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService do
+  context 'include:' do
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:user)    { project.owner }
+
+    let(:ref)      { 'refs/heads/master' }
+    let(:source)   { :push }
+    let(:service)  { described_class.new(project, user, { ref: ref }) }
+    let(:pipeline) { service.execute(source).payload }
+
+    let(:file_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
+
+    before do
+      allow(project.repository)
+        .to receive(:blob_data_at).with(project.commit.id, '.gitlab-ci.yml')
+        .and_return(config)
+
+      allow(project.repository)
+        .to receive(:blob_data_at).with(project.commit.id, file_location)
+        .and_return(File.read(Rails.root.join(file_location)))
+    end
+
+    context 'with a local file' do
+      let(:config) do
+        <<~EOY
+        include: #{file_location}
+        job:
+          script: exit 0
+        EOY
+      end
+
+      it 'includes the job in the file' do
+        expect(pipeline).to be_created_successfully
+        expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
+      end
+    end
+
+    context 'with a local file with rules' do
+      let(:config) do
+        <<~EOY
+        include:
+          - local: #{file_location}
+            rules:
+              - if: $CI_PROJECT_ID == "#{project_id}"
+        job:
+          script: exit 0
+        EOY
+      end
+
+      context 'when the rules matches' do
+        let(:project_id) { project.id }
+
+        it 'includes the job in the file' do
+          expect(pipeline).to be_created_successfully
+          expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
+        end
+
+        context 'when the FF ci_include_rules is disabled' do
+          before do
+            stub_feature_flags(ci_include_rules: false)
+          end
+
+          it 'includes the job in the file' do
+            expect(pipeline).to be_created_successfully
+            expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
+          end
+        end
+      end
+
+      context 'when the rules does not match' do
+        let(:project_id) { non_existing_record_id }
+
+        it 'does not include the job in the file' do
+          expect(pipeline).to be_created_successfully
+          expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
+        end
+
+        context 'when the FF ci_include_rules is disabled' do
+          before do
+            stub_feature_flags(ci_include_rules: false)
+          end
+
+          it 'includes the job in the file' do
+            expect(pipeline).to be_created_successfully
+            expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
+          end
+        end
+      end
+    end
+  end
+end