Create CI cache keys based on commit ids

Allow sharing CI files cache across branches to speed
up pipeline execution time for many users.

Adds `key:files: []` to the CI config file.
Works by selecting the latest commit that changed any
of given files and uses it as the key.

changelogs/unreleased/18986-allow-to-use-commit-sha-in-cache-key.yml

+---
+title: Build CI cache key from commit SHAs that changed given files
+merge_request: 19392
+author:
+type: added

doc/ci/yaml/README.md

@@ -1535,6 +1535,50 @@ cache:
     - binaries/
 ```
 
+##### `cache:key:files`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18986) in GitLab v12.5.
+
+If `cache:key:files` is added, the cache `key` will use the SHA of the most recent commit
+that changed either of the given files. If neither file was changed in any commits, the key will be `default`.
+A maximum of two files are allowed.
+
+```yaml
+cache:
+  key:
+    files:
+      - Gemfile.lock
+      - package.json
+  paths:
+    - vendor/ruby
+    - node_modules
+```
+
+##### `cache:key:prefix`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/18986) in GitLab v12.5.
+
+The `prefix` parameter adds extra functionality to `key:files` by allowing the key to
+be composed of the given `prefix` combined with the SHA of the most recent commit
+that changed either of the files. For example, adding a `prefix` of `rspec`, will
+cause keys to look like: `rspec-feef9576d21ee9b6a32e30c5c79d0a0ceb68d1e5`. If neither
+file was changed in any commits, the prefix is added to `default`, so the key in the
+example would be `rspec-default`.
+
+`prefix` follows the same restrictions as `key`, so it can use any of the
+[predefined variables](../variables/README.md). Similarly, the `/` character or the
+equivalent URI-encoded `%2F`, or a value made only of `.` or `%2E`, is not allowed.
+
+```yaml
+cache:
+  key:
+    files:
+      - Gemfile.lock
+    prefix: ${CI_JOB_NAME}
+  paths:
+    - vendor/ruby
+```
+
 #### `cache:untracked`
 
 Set `untracked: true` to cache all files that are untracked in your Git

lib/gitlab/ci/config/entry/files.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents an array of file paths.
+        #
+        class Files < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, array_of_strings: true
+            validates :config, length: {
+              minimum: 1,
+              maximum: 2,
+              too_short: 'requires at least %{count} item',
+              too_long: 'has too many items (maximum is %{count})'
+            }
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/key.rb

@@ -7,11 +7,48 @@ module Gitlab
         ##
         # Entry that represents a key.
         #
-        class Key < ::Gitlab::Config::Entry::Node
-          include ::Gitlab::Config::Entry::Validatable
+        class Key < ::Gitlab::Config::Entry::Simplifiable
+          strategy :SimpleKey, if: -> (config) { config.is_a?(String) || config.is_a?(Symbol) }
+          strategy :ComplexKey, if: -> (config) { config.is_a?(Hash) }
 
-          validations do
-            validates :config, key: true
+          class SimpleKey < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations do
+              validates :config, key: true
+            end
+
+            def self.default
+              'default'
+            end
+
+            def value
+              super.to_s
+            end
+          end
+
+          class ComplexKey < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Attributable
+            include ::Gitlab::Config::Entry::Configurable
+
+            ALLOWED_KEYS = %i[files prefix].freeze
+            REQUIRED_KEYS = %i[files].freeze
+
+            validations do
+              validates :config, allowed_keys: ALLOWED_KEYS
+              validates :config, required_keys: REQUIRED_KEYS
+            end
+
+            entry :files, Entry::Files,
+              description: 'Files that should be used to build the key'
+            entry :prefix, Entry::Prefix,
+              description: 'Prefix that is added to the final cache key'
+          end
+
+          class UnknownStrategy < ::Gitlab::Config::Entry::Node
+            def errors
+              ["#{location} should be a hash, a string or a symbol"]
+            end
           end
 
           def self.default

lib/gitlab/ci/config/entry/prefix.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a key prefix.
+        #
+        class Prefix < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, key: true
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/seed/build.rb

@@ -29,6 +29,8 @@ module Gitlab
               .fabricate(attributes.delete(:except))
             @rules = Gitlab::Ci::Build::Rules
               .new(attributes.delete(:rules))
+            @cache = Seed::Build::Cache
+              .new(pipeline, attributes.delete(:cache))
           end
 
           def name
@@ -59,6 +61,7 @@ module Gitlab
             @seed_attributes
               .deep_merge(pipeline_attributes)
               .deep_merge(rules_attributes)
+              .deep_merge(cache_attributes)
           end
 
           def bridge?
@@ -150,6 +153,12 @@ module Gitlab
               @using_rules ? @rules.evaluate(@pipeline, self).build_attributes : {}
             end
           end
+
+          def cache_attributes
+            strong_memoize(:cache_attributes) do
+              @cache.build_attributes
+            end
+          end
         end
       end
     end

lib/gitlab/ci/pipeline/seed/build/cache.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Build
+          class Cache
+            def initialize(pipeline, cache)
+              @pipeline = pipeline
+              local_cache = cache.to_h.deep_dup
+              @key = local_cache.delete(:key)
+              @paths = local_cache.delete(:paths)
+              @policy = local_cache.delete(:policy)
+              @untracked = local_cache.delete(:untracked)
+
+              raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any?
+            end
+
+            def build_attributes
+              {
+                options: {
+                  cache: {
+                    key: key_string,
+                    paths: @paths,
+                    policy: @policy,
+                    untracked: @untracked
+                  }.compact.presence
+                }.compact
+              }
+            end
+
+            private
+
+            def key_string
+              key_from_string || key_from_files
+            end
+
+            def key_from_string
+              @key.to_s if @key.is_a?(String) || @key.is_a?(Symbol)
+            end
+
+            def key_from_files
+              return unless @key.is_a?(Hash)
+
+              [@key[:prefix], files_digest].select(&:present?).join('-')
+            end
+
+            def files_digest
+              hash_of_the_latest_changes || 'default'
+            end
+
+            def hash_of_the_latest_changes
+              return unless Feature.enabled?(:ci_file_based_cache, @pipeline.project, default_enabled: true)
+
+              ids = files.map { |path| last_commit_id_for_path(path) }
+              ids = ids.compact.sort.uniq
+
+              Digest::SHA1.hexdigest(ids.join('-')) if ids.any?
+            end
+
+            def files
+              @key[:files]
+                .to_a
+                .select(&:present?)
+                .uniq
+            end
+
+            def last_commit_id_for_path(path)
+              @pipeline.project.repository.last_commit_id_for_path(@pipeline.sha, path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/yaml_processor.rb

@@ -43,11 +43,11 @@ module Gitlab
           needs_attributes: job.dig(:needs, :job),
           interruptible: job[:interruptible],
           rules: job[:rules],
+          cache: job[:cache],
           options: {
             image: job[:image],
             services: job[:services],
             artifacts: job[:artifacts],
-            cache: job[:cache],
             dependencies: job[:dependencies],
             job_timeout: job[:timeout],
             before_script: job[:before_script],

spec/lib/gitlab/ci/config/entry/cache_spec.rb

@@ -12,22 +12,53 @@ describe Gitlab::Ci::Config::Entry::Cache do
 
     context 'when entry config value is correct' do
       let(:policy) { nil }
+      let(:key) { 'some key' }
 
       let(:config) do
-        { key: 'some key',
+        { key: key,
           untracked: true,
           paths: ['some/path/'],
           policy: policy }
       end
 
       describe '#value' do
-        it 'returns hash value' do
-          expect(entry.value).to eq(key: 'some key', untracked: true, paths: ['some/path/'], policy: 'pull-push')
+        shared_examples 'hash key value' do
+          it 'returns hash value' do
+            expect(entry.value).to eq(key: key, untracked: true, paths: ['some/path/'], policy: 'pull-push')
+          end
+        end
+
+        it_behaves_like 'hash key value'
+
+        context 'with files' do
+          let(:key) { { files: ['a-file', 'other-file'] } }
+
+          it_behaves_like 'hash key value'
+        end
+
+        context 'with files and prefix' do
+          let(:key) { { files: ['a-file', 'other-file'], prefix: 'prefix-value' } }
+
+          it_behaves_like 'hash key value'
+        end
+
+        context 'with prefix' do
+          let(:key) { { prefix: 'prefix-value' } }
+
+          it 'key is nil' do
+            expect(entry.value).to match(a_hash_including(key: nil))
+          end
         end
       end
 
       describe '#valid?' do
         it { is_expected.to be_valid }
+
+        context 'with files' do
+          let(:key) { { files: ['a-file', 'other-file'] } }
+
+          it { is_expected.to be_valid }
+        end
       end
 
       context 'policy is pull-push' do
@@ -87,10 +118,44 @@ describe Gitlab::Ci::Config::Entry::Cache do
         end
 
         context 'when descendants are invalid' do
-          let(:config) { { key: 1 } }
+          context 'with invalid keys' do
+            let(:config) { { key: 1 } }
 
-          it 'reports error with descendants' do
-            is_expected.to include 'key config should be a string or symbol'
+            it 'reports error with descendants' do
+              is_expected.to include 'key should be a hash, a string or a symbol'
+            end
+          end
+
+          context 'with empty key' do
+            let(:config) { { key: {} } }
+
+            it 'reports error with descendants' do
+              is_expected.to include 'key config missing required keys: files'
+            end
+          end
+
+          context 'with invalid files' do
+            let(:config) { { key: { files: 'a-file' } } }
+
+            it 'reports error with descendants' do
+              is_expected.to include 'key:files config should be an array of strings'
+            end
+          end
+
+          context 'with prefix without files' do
+            let(:config) { { key: { prefix: 'a-prefix' } } }
+
+            it 'reports error with descendants' do
+              is_expected.to include 'key config missing required keys: files'
+            end
+          end
+
+          context 'when there is an unknown key present' do
+            let(:config) { { key: { unknown: 'a-file' } } }
+
+            it 'reports error with descendants' do
+              is_expected.to include 'key config contains unknown keys: unknown'
+            end
           end
         end
 

spec/lib/gitlab/ci/config/entry/files_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Files do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validations' do
+    context 'when entry config value is valid' do
+      let(:config) { ['some/file', 'some/path/'] }
+
+      describe '#value' do
+        it 'returns key value' do
+          expect(entry.value).to eq config
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    describe '#errors' do
+      context 'when entry value is not an array' do
+        let(:config) { 'string' }
+
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'files config should be an array of strings'
+        end
+      end
+
+      context 'when entry value is not an array of strings' do
+        let(:config) { [1] }
+
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'files config should be an array of strings'
+        end
+      end
+
+      context 'when entry value contains more than two values' do
+        let(:config) { %w[file1 file2 file3] }
+
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'files config has too many items (maximum is 2)'
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/key_spec.rb

@@ -6,38 +6,38 @@ describe Gitlab::Ci::Config::Entry::Key do
   let(:entry) { described_class.new(config) }
 
   describe 'validations' do
-    shared_examples 'key with slash' do
-      it 'is invalid' do
-        expect(entry).not_to be_valid
-      end
+    it_behaves_like 'key entry validations', 'simple key'
 
-      it 'reports errors with config value' do
-        expect(entry.errors).to include 'key config cannot contain the "/" character'
-      end
-    end
+    context 'when entry config value is correct' do
+      context 'when key is a hash' do
+        let(:config) { { files: ['test'], prefix: 'something' } }
 
-    shared_examples 'key with only dots' do
-      it 'is invalid' do
-        expect(entry).not_to be_valid
-      end
+        describe '#value' do
+          it 'returns key value' do
+            expect(entry.value).to match(config)
+          end
+        end
 
-      it 'reports errors with config value' do
-        expect(entry.errors).to include 'key config cannot be "." or ".."'
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
       end
-    end
 
-    context 'when entry config value is correct' do
-      let(:config) { 'test' }
+      context 'when key is a symbol' do
+        let(:config) { :key }
 
-      describe '#value' do
-        it 'returns key value' do
-          expect(entry.value).to eq 'test'
+        describe '#value' do
+          it 'returns key value' do
+            expect(entry.value).to eq(config.to_s)
+          end
         end
-      end
 
-      describe '#valid?' do
-        it 'is valid' do
-          expect(entry).to be_valid
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
         end
       end
     end
@@ -47,53 +47,11 @@ describe Gitlab::Ci::Config::Entry::Key do
 
       describe '#errors' do
         it 'saves errors' do
-          expect(entry.errors)
-            .to include 'key config should be a string or symbol'
+          expect(entry.errors.first)
+            .to match /should be a hash, a string or a symbol/
         end
       end
     end
-
-    context 'when entry value contains slash' do
-      let(:config) { 'key/with/some/slashes' }
-
-      it_behaves_like 'key with slash'
-    end
-
-    context 'when entry value contains URI encoded slash (%2F)' do
-      let(:config) { 'key%2Fwith%2Fsome%2Fslashes' }
-
-      it_behaves_like 'key with slash'
-    end
-
-    context 'when entry value is a dot' do
-      let(:config) { '.' }
-
-      it_behaves_like 'key with only dots'
-    end
-
-    context 'when entry value is two dots' do
-      let(:config) { '..' }
-
-      it_behaves_like 'key with only dots'
-    end
-
-    context 'when entry value is a URI encoded dot (%2E)' do
-      let(:config) { '%2e' }
-
-      it_behaves_like 'key with only dots'
-    end
-
-    context 'when entry value is two URI encoded dots (%2E)' do
-      let(:config) { '%2E%2e' }
-
-      it_behaves_like 'key with only dots'
-    end
-
-    context 'when entry value is one dot and one URI encoded dot' do
-      let(:config) { '.%2e' }
-
-      it_behaves_like 'key with only dots'
-    end
   end
 
   describe '.default' do

spec/lib/gitlab/ci/config/entry/prefix_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Prefix do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validations' do
+    it_behaves_like 'key entry validations', :prefix
+
+    context 'when entry value is not correct' do
+      let(:config) { ['incorrect'] }
+
+      describe '#errors' do
+        it 'saves errors' do
+          expect(entry.errors)
+            .to include 'prefix config should be a string or symbol'
+        end
+      end
+    end
+  end
+
+  describe '.default' do
+    it 'returns default key' do
+      expect(described_class.default).to be_nil
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/seed/build/cache_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Seed::Build::Cache do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:head_sha) { project.repository.head_commit.id }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: head_sha) }
+
+  let(:processor) { described_class.new(pipeline, config) }
+
+  describe '#build_attributes' do
+    subject { processor.build_attributes }
+
+    context 'with cache:key' do
+      let(:config) do
+        {
+          key: 'a-key',
+          paths: ['vendor/ruby']
+        }
+      end
+
+      it { is_expected.to include(options: { cache: config }) }
+    end
+
+    context 'with cache:key as a symbol' do
+      let(:config) do
+        {
+          key: :a_key,
+          paths: ['vendor/ruby']
+        }
+      end
+
+      it { is_expected.to include(options: { cache: config.merge(key: "a_key") }) }
+    end
+
+    context 'with cache:key:files' do
+      shared_examples 'default key' do
+        let(:config) do
+          { key: { files: files } }
+        end
+
+        it 'uses default key' do
+          expected = { options: { cache: { key: 'default' } } }
+
+          is_expected.to include(expected)
+        end
+      end
+
+      shared_examples 'version and gemfile files' do
+        let(:config) do
+          {
+            key: {
+              files: files
+            },
+            paths: ['vendor/ruby']
+          }
+        end
+
+        it 'builds a string key' do
+          expected = {
+            options: {
+              cache: {
+                key: '703ecc8fef1635427a1f86a8a1a308831c122392',
+                paths: ['vendor/ruby']
+              }
+            }
+          }
+
+          is_expected.to include(expected)
+        end
+      end
+
+      context 'with existing files' do
+        let(:files) { ['VERSION', 'Gemfile.zip'] }
+
+        it_behaves_like 'version and gemfile files'
+      end
+
+      context 'with files starting with ./' do
+        let(:files) { ['Gemfile.zip', './VERSION'] }
+
+        it_behaves_like 'version and gemfile files'
+      end
+
+      context 'with feature flag disabled' do
+        let(:files) { ['VERSION', 'Gemfile.zip'] }
+
+        before do
+          stub_feature_flags(ci_file_based_cache: false)
+        end
+
+        it_behaves_like 'default key'
+      end
+
+      context 'with files ending with /' do
+        let(:files) { ['Gemfile.zip/'] }
+
+        it_behaves_like 'default key'
+      end
+
+      context 'with new line in filenames' do
+        let(:files) { ["Gemfile.zip\nVERSION"] }
+
+        it_behaves_like 'default key'
+      end
+
+      context 'with missing files' do
+        let(:files) { ['project-gemfile.lock', ''] }
+
+        it_behaves_like 'default key'
+      end
+
+      context 'with directories' do
+        shared_examples 'foo/bar directory key' do
+          let(:config) do
+            {
+              key: {
+                files: files
+              }
+            }
+          end
+
+          it 'builds a string key' do
+            expected = {
+              options: {
+                cache: { key: '74bf43fb1090f161bdd4e265802775dbda2f03d1' }
+              }
+            }
+
+            is_expected.to include(expected)
+          end
+        end
+
+        context 'with directory' do
+          let(:files) { ['foo/bar'] }
+
+          it_behaves_like 'foo/bar directory key'
+        end
+
+        context 'with directory ending in slash' do
+          let(:files) { ['foo/bar/'] }
+
+          it_behaves_like 'foo/bar directory key'
+        end
+
+        context 'with directories ending in slash star' do
+          let(:files) { ['foo/bar/*'] }
+
+          it_behaves_like 'foo/bar directory key'
+        end
+      end
+    end
+
+    context 'with cache:key:prefix' do
+      context 'without files' do
+        let(:config) do
+          {
+            key: {
+              prefix: 'a-prefix'
+            },
+            paths: ['vendor/ruby']
+          }
+        end
+
+        it 'adds prefix to default key' do
+          expected = {
+            options: {
+              cache: {
+                key: 'a-prefix-default',
+                paths: ['vendor/ruby']
+              }
+            }
+          }
+
+          is_expected.to include(expected)
+        end
+      end
+
+      context 'with existing files' do
+        let(:config) do
+          {
+            key: {
+              files: ['VERSION', 'Gemfile.zip'],
+              prefix: 'a-prefix'
+            },
+            paths: ['vendor/ruby']
+          }
+        end
+
+        it 'adds prefix key' do
+          expected = {
+            options: {
+              cache: {
+                key: 'a-prefix-703ecc8fef1635427a1f86a8a1a308831c122392',
+                paths: ['vendor/ruby']
+              }
+            }
+          }
+
+          is_expected.to include(expected)
+        end
+      end
+
+      context 'with missing files' do
+        let(:config) do
+          {
+            key: {
+              files: ['project-gemfile.lock', ''],
+              prefix: 'a-prefix'
+            },
+            paths: ['vendor/ruby']
+          }
+        end
+
+        it 'adds prefix to default key' do
+          expected = {
+            options: {
+              cache: {
+                key: 'a-prefix-default',
+                paths: ['vendor/ruby']
+              }
+            }
+          }
+
+          is_expected.to include(expected)
+        end
+      end
+    end
+
+    context 'with all cache option keys' do
+      let(:config) do
+        {
+          key: 'a-key',
+          paths: ['vendor/ruby'],
+          untracked: true,
+          policy: 'push'
+        }
+      end
+
+      it { is_expected.to include(options: { cache: config }) }
+    end
+
+    context 'with unknown cache option keys' do
+      let(:config) do
+        {
+          key: 'a-key',
+          unknown_key: true
+        }
+      end
+
+      it { expect { subject }.to raise_error(ArgumentError, /unknown_key/) }
+    end
+
+    context 'with empty config' do
+      let(:config) { {} }
+
+      it { is_expected.to include(options: {}) }
+    end
+  end
+end

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -4,7 +4,8 @@ require 'spec_helper'
 
 describe Gitlab::Ci::Pipeline::Seed::Build do
   let(:project) { create(:project, :repository) }
-  let(:pipeline) { create(:ci_empty_pipeline, project: project) }
+  let(:head_sha) { project.repository.head_commit.id }
+  let(:pipeline) { create(:ci_empty_pipeline, project: project, sha: head_sha) }
   let(:attributes) { { name: 'rspec', ref: 'master' } }
   let(:previous_stages) { [] }
 
@@ -69,6 +70,101 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
         it { is_expected.to include(when: 'never') }
       end
     end
+
+    context 'with cache:key' do
+      let(:attributes) do
+        {
+          name: 'rspec',
+          ref: 'master',
+          cache: {
+            key: 'a-value'
+          }
+        }
+      end
+
+      it { is_expected.to include(options: { cache: { key: 'a-value' } }) }
+    end
+
+    context 'with cache:key:files' do
+      let(:attributes) do
+        {
+          name: 'rspec',
+          ref: 'master',
+          cache: {
+            key: {
+              files: ['VERSION']
+            }
+          }
+        }
+      end
+
+      it 'includes cache options' do
+        cache_options = {
+          options: {
+            cache: {
+              key: 'f155568ad0933d8358f66b846133614f76dd0ca4'
+            }
+          }
+        }
+
+        is_expected.to include(cache_options)
+      end
+    end
+
+    context 'with cache:key:prefix' do
+      let(:attributes) do
+        {
+          name: 'rspec',
+          ref: 'master',
+          cache: {
+            key: {
+              prefix: 'something'
+            }
+          }
+        }
+      end
+
+      it { is_expected.to include(options: { cache: { key: 'something-default' } }) }
+    end
+
+    context 'with cache:key:files and prefix' do
+      let(:attributes) do
+        {
+          name: 'rspec',
+          ref: 'master',
+          cache: {
+            key: {
+              files: ['VERSION'],
+              prefix: 'something'
+            }
+          }
+        }
+      end
+
+      it 'includes cache options' do
+        cache_options = {
+          options: {
+            cache: {
+              key: 'something-f155568ad0933d8358f66b846133614f76dd0ca4'
+            }
+          }
+        }
+
+        is_expected.to include(cache_options)
+      end
+    end
+
+    context 'with empty cache' do
+      let(:attributes) do
+        {
+          name: 'rspec',
+          ref: 'master',
+          cache: {}
+        }
+      end
+
+      it { is_expected.to include(options: {}) }
+    end
   end
 
   describe '#bridge?' do

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -950,7 +950,7 @@ module Gitlab
           config_processor = Gitlab::Ci::YamlProcessor.new(config)
 
           expect(config_processor.stage_builds_attributes("test").size).to eq(1)
-          expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq(
+          expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq(
             paths: ["logs/", "binaries/"],
             untracked: true,
             key: 'key',
@@ -962,7 +962,7 @@ module Gitlab
           config = YAML.dump(
             {
               default: {
-                cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'key' }
+                cache: { paths: ["logs/", "binaries/"], untracked: true, key: { files: ['file'] } }
               },
               rspec: {
                 script: "rspec"
@@ -972,33 +972,79 @@ module Gitlab
           config_processor = Gitlab::Ci::YamlProcessor.new(config)
 
           expect(config_processor.stage_builds_attributes("test").size).to eq(1)
-          expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq(
+          expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq(
             paths: ["logs/", "binaries/"],
             untracked: true,
-            key: 'key',
+            key: { files: ['file'] },
             policy: 'pull-push'
           )
         end
 
-        it "returns cache when defined in a job" do
+        it 'returns cache key when defined in a job' do
           config = YAML.dump({
                                rspec: {
-                                 cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'key' },
-                                 script: "rspec"
+                                 cache: { paths: ['logs/', 'binaries/'], untracked: true, key: 'key' },
+                                 script: 'rspec'
                                }
                              })
 
           config_processor = Gitlab::Ci::YamlProcessor.new(config)
 
-          expect(config_processor.stage_builds_attributes("test").size).to eq(1)
-          expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq(
-            paths: ["logs/", "binaries/"],
+          expect(config_processor.stage_builds_attributes('test').size).to eq(1)
+          expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq(
+            paths: ['logs/', 'binaries/'],
             untracked: true,
             key: 'key',
             policy: 'pull-push'
           )
         end
 
+        it 'returns cache files' do
+          config = YAML.dump(
+            rspec: {
+              cache: {
+                paths: ['logs/', 'binaries/'],
+                untracked: true,
+                key: { files: ['file'] }
+              },
+              script: 'rspec'
+            }
+          )
+
+          config_processor = Gitlab::Ci::YamlProcessor.new(config)
+
+          expect(config_processor.stage_builds_attributes('test').size).to eq(1)
+          expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq(
+            paths: ['logs/', 'binaries/'],
+            untracked: true,
+            key: { files: ['file'] },
+            policy: 'pull-push'
+          )
+        end
+
+        it 'returns cache files with prefix' do
+          config = YAML.dump(
+            rspec: {
+              cache: {
+                paths: ['logs/', 'binaries/'],
+                untracked: true,
+                key: { files: ['file'], prefix: 'prefix' }
+              },
+              script: 'rspec'
+            }
+          )
+
+          config_processor = Gitlab::Ci::YamlProcessor.new(config)
+
+          expect(config_processor.stage_builds_attributes('test').size).to eq(1)
+          expect(config_processor.stage_builds_attributes('test').first[:cache]).to eq(
+            paths: ['logs/', 'binaries/'],
+            untracked: true,
+            key: { files: ['file'], prefix: 'prefix' },
+            policy: 'pull-push'
+          )
+        end
+
         it "overwrite cache when defined for a job and globally" do
           config = YAML.dump({
                                cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'global' },
@@ -1011,7 +1057,7 @@ module Gitlab
           config_processor = Gitlab::Ci::YamlProcessor.new(config)
 
           expect(config_processor.stage_builds_attributes("test").size).to eq(1)
-          expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq(
+          expect(config_processor.stage_builds_attributes("test").first[:cache]).to eq(
             paths: ["test/"],
             untracked: false,
             key: 'local',
@@ -1862,14 +1908,42 @@ module Gitlab
           config = YAML.dump({ cache: { key: 1 }, rspec: { script: "test" } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "cache:key config should be a string or symbol")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "cache:key should be a hash, a string or a symbol")
         end
 
         it "returns errors if job cache:key is not an a string" do
           config = YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: 1 } } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:cache:key config should be a string or symbol")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:cache:key should be a hash, a string or a symbol")
+        end
+
+        it 'returns errors if job cache:key:files is not an array of strings' do
+          config = YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { files: [1] } } } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'jobs:rspec:cache:key:files config should be an array of strings')
+        end
+
+        it 'returns errors if job cache:key:files is an empty array' do
+          config = YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { files: [] } } } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'jobs:rspec:cache:key:files config requires at least 1 item')
+        end
+
+        it 'returns errors if job defines only cache:key:prefix' do
+          config = YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { prefix: 'prefix-key' } } } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'jobs:rspec:cache:key config missing required keys: files')
+        end
+
+        it 'returns errors if job cache:key:prefix is not an a string' do
+          config = YAML.dump({ types: %w(build test), rspec: { script: "test", cache: { key: { prefix: 1, files: ['file'] } } } })
+          expect do
+            Gitlab::Ci::YamlProcessor.new(config)
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'jobs:rspec:cache:key:prefix config should be a string or symbol')
         end
 
         it "returns errors if job cache:untracked is not an array of strings" do

spec/services/ci/create_pipeline_service/cache_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  context 'cache' do
+    let(:user)     { create(:admin) }
+    let(:ref)      { 'refs/heads/master' }
+    let(:source)   { :push }
+    let(:service)  { described_class.new(project, user, { ref: ref }) }
+    let(:pipeline) { service.execute(source) }
+    let(:job)      { pipeline.builds.find_by(name: 'job') }
+    let(:project)  { create(:project, :custom_repo, files: files) }
+
+    before do
+      stub_ci_pipeline_yaml_file(config)
+    end
+
+    context 'with cache:key' do
+      let(:files) { { 'some-file' => '' } }
+
+      let(:config) do
+        <<~EOY
+        job:
+          script:
+            - ls
+          cache:
+            key: 'a-key'
+            paths: ['logs/', 'binaries/']
+            untracked: true
+        EOY
+      end
+
+      it 'uses the provided key' do
+        expected = {
+          'key'       => 'a-key',
+          'paths'     => ['logs/', 'binaries/'],
+          'policy'    => 'pull-push',
+          'untracked' => true
+        }
+
+        expect(pipeline).to be_persisted
+        expect(job.cache).to match(a_collection_including(expected))
+      end
+    end
+
+    context 'with cache:key:files' do
+      let(:config) do
+        <<~EOY
+        job:
+          script:
+            - ls
+          cache:
+            paths:
+              - logs/
+            key:
+              files:
+                - file.lock
+                - missing-file.lock
+        EOY
+      end
+
+      context 'when file.lock exists' do
+        let(:files) { { 'file.lock' => '' } }
+
+        it 'builds a cache key' do
+          expected = {
+            'key'       => /[a-f0-9]{40}/,
+            'paths'     => ['logs/'],
+            'policy'    => 'pull-push'
+          }
+
+          expect(pipeline).to be_persisted
+          expect(job.cache).to match(a_collection_including(expected))
+        end
+      end
+
+      context 'when file.lock does not exist' do
+        let(:files) { { 'some-file' => '' } }
+
+        it 'uses default cache key' do
+          expected = {
+            'key'       => /default/,
+            'paths'     => ['logs/'],
+            'policy'    => 'pull-push'
+          }
+
+          expect(pipeline).to be_persisted
+          expect(job.cache).to match(a_collection_including(expected))
+        end
+      end
+    end
+
+    context 'with cache:key:files and prefix' do
+      let(:config) do
+        <<~EOY
+        job:
+          script:
+            - ls
+          cache:
+            paths:
+              - logs/
+            key:
+              files:
+                - file.lock
+              prefix: '$ENV_VAR'
+        EOY
+      end
+
+      context 'when file.lock exists' do
+        let(:files) { { 'file.lock' => '' } }
+
+        it 'builds a cache key' do
+          expected = {
+            'key'       => /\$ENV_VAR-[a-f0-9]{40}/,
+            'paths'     => ['logs/'],
+            'policy'    => 'pull-push'
+          }
+
+          expect(pipeline).to be_persisted
+          expect(job.cache).to match(a_collection_including(expected))
+        end
+      end
+
+      context 'when file.lock does not exist' do
+        let(:files) { { 'some-file' => '' } }
+
+        it 'uses default cache key' do
+          expected = {
+            'key'       => /\$ENV_VAR-default/,
+            'paths'     => ['logs/'],
+            'policy'    => 'pull-push'
+          }
+
+          expect(pipeline).to be_persisted
+          expect(job.cache).to match(a_collection_including(expected))
+        end
+      end
+    end
+
+    context 'with too many files' do
+      let(:files) { { 'some-file' => '' } }
+
+      let(:config) do
+        <<~EOY
+        job:
+          script:
+            - ls
+          cache:
+            paths: ['logs/', 'binaries/']
+            untracked: true
+            key:
+              files:
+                - file.lock
+                - other-file.lock
+                - extra-file.lock
+              prefix: 'some-prefix'
+        EOY
+      end
+
+      it 'has errors' do
+        expect(pipeline).to be_persisted
+        expect(pipeline.yaml_errors).to eq("jobs:job:cache:key:files config has too many items (maximum is 2)")
+        expect(job).to be_nil
+      end
+    end
+  end
+end

spec/support/shared_examples/lib/gitlab/ci/config/entry/key_validations_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'key entry validations' do |config_name|
+  shared_examples 'key with slash' do
+    it 'is invalid' do
+      expect(entry).not_to be_valid
+    end
+
+    it 'reports errors with config value' do
+      expect(entry.errors).to include "#{config_name} config cannot contain the \"/\" character"
+    end
+  end
+
+  shared_examples 'key with only dots' do
+    it 'is invalid' do
+      expect(entry).not_to be_valid
+    end
+
+    it 'reports errors with config value' do
+      expect(entry.errors).to include "#{config_name} config cannot be \".\" or \"..\""
+    end
+  end
+
+  context 'when entry value contains slash' do
+    let(:config) { 'key/with/some/slashes' }
+
+    it_behaves_like 'key with slash'
+  end
+
+  context 'when entry value contains URI encoded slash (%2F)' do
+    let(:config) { 'key%2Fwith%2Fsome%2Fslashes' }
+
+    it_behaves_like 'key with slash'
+  end
+
+  context 'when entry value is a dot' do
+    let(:config) { '.' }
+
+    it_behaves_like 'key with only dots'
+  end
+
+  context 'when entry value is two dots' do
+    let(:config) { '..' }
+
+    it_behaves_like 'key with only dots'
+  end
+
+  context 'when entry value is a URI encoded dot (%2E)' do
+    let(:config) { '%2e' }
+
+    it_behaves_like 'key with only dots'
+  end
+
+  context 'when entry value is two URI encoded dots (%2E)' do
+    let(:config) { '%2E%2e' }
+
+    it_behaves_like 'key with only dots'
+  end
+
+  context 'when entry value is one dot and one URI encoded dot' do
+    let(:config) { '.%2e' }
+
+    it_behaves_like 'key with only dots'
+  end
+
+  context 'when key is a string' do
+    let(:config) { 'test' }
+
+    describe '#value' do
+      it 'returns key value' do
+        expect(entry.value).to eq 'test'
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+  end
+end