Merge branch 'job_file_matching' into 'master'

Add file matching rule to flexible CI rules

See merge request gitlab-org/gitlab!16574

app/models/ci/pipeline.rb

@@ -771,6 +771,18 @@ module Ci
       end
     end
 
+    def all_worktree_paths
+      strong_memoize(:all_worktree_paths) do
+        project.repository.ls_files(sha)
+      end
+    end
+
+    def top_level_worktree_paths
+      strong_memoize(:top_level_worktree_paths) do
+        project.repository.tree(sha).blobs.map(&:path)
+      end
+    end
+
     def default_branch?
       ref == project.default_branch
     end

changelogs/unreleased/job_file_matching.yml

+---
+title: Add file matching rule to flexible CI rules
+merge_request: 16574
+author:
+type: added

lib/gitlab/ci/build/rules/rule/clause/exists.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Exists < Rules::Rule::Clause
+        # The maximum number of patterned glob comparisons that will be
+        # performed before the rule assumes that it has a match
+        MAX_PATTERN_COMPARISONS = 10_000
+
+        def initialize(globs)
+          globs = Array(globs)
+
+          @top_level_only = globs.all?(&method(:top_level_glob?))
+          @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
+        end
+
+        def satisfied_by?(pipeline, seed)
+          paths = worktree_paths(pipeline)
+
+          exact_matches?(paths) || pattern_matches?(paths)
+        end
+
+        private
+
+        def worktree_paths(pipeline)
+          if @top_level_only
+            pipeline.top_level_worktree_paths
+          else
+            pipeline.all_worktree_paths
+          end
+        end
+
+        def exact_matches?(paths)
+          @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } }
+        end
+
+        def pattern_matches?(paths)
+          comparisons = 0
+          @pattern_globs.any? do |glob|
+            paths.any? do |path|
+              comparisons += 1
+              comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path)
+            end
+          end
+        end
+
+        def pattern_match?(glob, path)
+          File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+        end
+
+        # matches glob patterns that only match files in the top level directory
+        def top_level_glob?(glob)
+          !glob.include?('/') && !glob.include?('**')
+        end
+
+        # matches glob patterns that have no metacharacters for File#fnmatch?
+        def exact_glob?(glob)
+          !glob.include?('*') && !glob.include?('?') && !glob.include?('[') && !glob.include?('{')
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/rules/rule.rb

@@ -8,11 +8,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          CLAUSES      = %i[if changes].freeze
-          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          CLAUSES      = %i[if changes exists].freeze
+          ALLOWED_KEYS = %i[if changes exists when start_in].freeze
           ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
 
-          attributes :if, :changes, :when, :start_in
+          attributes :if, :changes, :exists, :when, :start_in
 
           validations do
             validates :config, presence: true
@@ -24,7 +24,7 @@ module Gitlab
 
             with_options allow_nil: true do
               validates :if, expression: true
-              validates :changes, array_of_strings: true
+              validates :changes, :exists, array_of_strings: true, length: { maximum: 50 }
               validates :when, allowed_values: { in: ALLOWED_WHEN }
             end
           end

spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do
+  describe '#satisfied_by?' do
+    it_behaves_like 'a glob matching rule' do
+      let(:pipeline) { build(:ci_pipeline) }
+
+      before do
+        allow(pipeline).to receive(:modified_paths).and_return(files.keys)
+      end
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
+    end
+  end
+end

spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do
+  describe '#satisfied_by?' do
+    it_behaves_like 'a glob matching rule' do
+      let(:project) { create(:project, :custom_repo, files: files) }
+      let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb

@@ -103,6 +103,52 @@ describe Gitlab::Ci::Config::Entry::Rules::Rule do
       end
     end
 
+    context 'when using a long list as an invalid changes: clause' do
+      let(:config) { { changes: ['app/'] * 51 } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/changes is too long \(maximum is 50 characters\)/)
+      end
+    end
+
+    context 'when using a exists: clause' do
+      let(:config) { { exists: %w[app/ lib/ spec/ other/* paths/**/*.rb] } }
+
+      it { is_expected.to be_valid }
+    end
+
+    context 'when using a string as an invalid exists: clause' do
+      let(:config) { { exists: 'a regular string' } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'reports an error about invalid policy' do
+        expect(subject.errors).to include(/should be an array of strings/)
+      end
+    end
+
+    context 'when using a list as an invalid exists: clause' do
+      let(:config) { { exists: [1, 2] } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/exists should be an array of strings/)
+      end
+    end
+
+    context 'when using a long list as an invalid exists: clause' do
+      let(:config) { { exists: ['app/'] * 51 } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/exists is too long \(maximum is 50 characters\)/)
+      end
+    end
+
     context 'specifying a delayed job' do
       let(:config) { { if: '$THIS || $THAT', when: 'delayed', start_in: '15 minutes' } }
 
@@ -198,6 +244,12 @@ describe Gitlab::Ci::Config::Entry::Rules::Rule do
         expect(entry.value).to eq(config)
       end
     end
+
+    context 'when using a exists: clause' do
+      let(:config) { { exists: %w[app/ lib/ spec/ other/* paths/**/*.rb] } }
+
+      it { is_expected.to eq(config) }
+    end
   end
 
   describe '.default' do

spec/models/ci/pipeline_spec.rb

@@ -1755,6 +1755,30 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '#all_worktree_paths' do
+    let(:files) { { 'main.go' => '', 'mocks/mocks.go' => '' } }
+    let(:project) { create(:project, :custom_repo, files: files) }
+    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+    it 'returns all file paths cached' do
+      expect(project.repository).to receive(:ls_files).with(pipeline.sha).once.and_call_original
+      expect(pipeline.all_worktree_paths).to eq(files.keys)
+      expect(pipeline.all_worktree_paths).to eq(files.keys)
+    end
+  end
+
+  describe '#top_level_worktree_paths' do
+    let(:files) { { 'main.go' => '', 'mocks/mocks.go' => '' } }
+    let(:project) { create(:project, :custom_repo, files: files) }
+    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+    it 'returns top-level file paths cached' do
+      expect(project.repository).to receive(:tree).with(pipeline.sha).once.and_call_original
+      expect(pipeline.top_level_worktree_paths).to eq(['main.go'])
+      expect(pipeline.top_level_worktree_paths).to eq(['main.go'])
+    end
+  end
+
   describe '#has_kubernetes_active?' do
     context 'when kubernetes is active' do
       context 'when user configured kubernetes from CI/CD > Clusters' do

spec/services/ci/create_pipeline_service/rules_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  context 'rules' do
+    let(:user)        { create(:admin) }
+    let(:ref)         { 'refs/heads/master' }
+    let(:source)      { :push }
+    let(:service)     { described_class.new(project, user, { ref: ref }) }
+    let(:pipeline)    { service.execute(source) }
+    let(:build_names) { pipeline.builds.pluck(:name) }
+
+    before do
+      stub_ci_pipeline_yaml_file(config)
+      allow_any_instance_of(Ci::BuildScheduleWorker).to receive(:perform).and_return(true)
+    end
+
+    context 'exists:' do
+      let(:config) do
+        <<-EOY
+        regular-job:
+          script: 'echo Hello, World!'
+
+        rules-job:
+          script: "echo hello world, $CI_COMMIT_REF_NAME"
+          rules:
+            - exists:
+              - README.md
+              when: manual
+            - exists:
+              - app.rb
+              when: on_success
+
+        delayed-job:
+          script: "echo See you later, World!"
+          rules:
+            - exists:
+              - README.md
+              when: delayed
+              start_in: 4 hours
+        EOY
+      end
+      let(:regular_job) { pipeline.builds.find_by(name: 'regular-job') }
+      let(:rules_job)   { pipeline.builds.find_by(name: 'rules-job') }
+      let(:delayed_job) { pipeline.builds.find_by(name: 'delayed-job') }
+
+      context 'with matches' do
+        let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) }
+
+        it 'creates two jobs' do
+          expect(pipeline).to be_persisted
+          expect(build_names).to contain_exactly('regular-job', 'rules-job', 'delayed-job')
+        end
+
+        it 'sets when: for all jobs' do
+          expect(regular_job.when).to eq('on_success')
+          expect(rules_job.when).to eq('manual')
+          expect(delayed_job.when).to eq('delayed')
+          expect(delayed_job.options[:start_in]).to eq('4 hours')
+        end
+      end
+
+      context 'with matches on the second rule' do
+        let(:project) { create(:project, :custom_repo, files: { 'app.rb' => '' }) }
+
+        it 'includes both jobs' do
+          expect(pipeline).to be_persisted
+          expect(build_names).to contain_exactly('regular-job', 'rules-job')
+        end
+
+        it 'sets when: for the created rules job based on the second clause' do
+          expect(regular_job.when).to eq('on_success')
+          expect(rules_job.when).to eq('on_success')
+        end
+      end
+
+      context 'without matches' do
+        let(:project) { create(:project, :custom_repo, files: { 'useless_script.rb' => '' }) }
+
+        it 'only persists the job without rules' do
+          expect(pipeline).to be_persisted
+          expect(regular_job).to be_persisted
+          expect(rules_job).to be_nil
+          expect(delayed_job).to be_nil
+        end
+
+        it 'sets when: for the created job' do
+          expect(regular_job.when).to eq('on_success')
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/lib/gitlab/ci/build/rules/rule/clause/clause_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'a glob matching rule' do
+  using RSpec::Parameterized::TableSyntax
+
+  where(:case_name, :globs, :files, :satisfied) do
+    'exact top-level match'      | ['Dockerfile']               | { 'Dockerfile' => '', 'Gemfile' => '' }            | true
+    'exact top-level no match'   | ['Dockerfile']               | { 'Gemfile' => '' }                                | false
+    'pattern top-level match'    | ['Docker*']                  | { 'Dockerfile' => '', 'Gemfile' => '' }            | true
+    'pattern top-level no match' | ['Docker*']                  | { 'Gemfile' => '' }                                | false
+    'exact nested match'         | ['project/build.properties'] | { 'project/build.properties' => '' }               | true
+    'exact nested no match'      | ['project/build.properties'] | { 'project/README.md' => '' }                      | false
+    'pattern nested match'       | ['src/**/*.go']              | { 'src/gitlab.com/goproject/goproject.go' => '' }  | true
+    'pattern nested no match'    | ['src/**/*.go']              | { 'src/gitlab.com/goproject/README.md' => '' }     | false
+    'ext top-level match'        | ['*.go']                     | { 'main.go' => '', 'cmd/goproject/main.go' => '' } | true
+    'ext nested no match'        | ['*.go']                     | { 'cmd/goproject/main.go' => '' }                  | false
+    'ext slash no match'         | ['/*.go']                    | { 'main.go' => '', 'cmd/goproject/main.go' => '' } | false
+  end
+
+  with_them do
+    it { is_expected.to eq(satisfied) }
+  end
+end