Merge branch 'ci/cache-key' into 'master'

Added cache:key to .gitlab-ci.yml allowing to fine tune the caching

The `cache:key` allows you to define the affinity mask of caching, allowing to have single cache for all jobs, or cache per-job, or per-branch, or any other way you would need:

1. Cache per-build for all branches:

```
cache:
    key: "$CI_BUILD_NAME"
    untracked: true
```

2. Cache per-branch for all jobs:

```
cache:
    key: "$CI_BUILD_REF"
    untracked: true
```



/cc @DouweM @grzesiek @axil


See merge request !2436

CHANGELOG

@@ -71,6 +71,7 @@ v 8.4.0 (unreleased)
   - Fix: Creator should be added as a master of the project on creation
   - Added X-GitLab-... headers to emails from CI and Email On Push services (Anton Baklanov)
   - Add IP check against DNSBLs at account sign-up
+  - Added cache:key to .gitlab-ci.yml allowing to fine tune the caching
 
 v 8.3.4
   - Use gitlab-workhorse 0.5.4 (fixes API routing bug)

doc/ci/yaml/README.md

@@ -135,10 +135,9 @@ thus allowing to fine tune them.
 ### cache
 
 `cache` is used to specify a list of files and directories which should be
-cached between builds. Caches are stored according to the branch/ref and the
-job name. They are not currently shared between different job names or between
-branches/refs, which means that caching will benefit you if you push subsequent
-commits to an existing feature branch.
+cached between builds.
+
+**By default the caching is enabled per-job and per-branch.**
 
 If `cache` is defined outside the scope of the jobs, it means it is set
 globally and all jobs will use its definition.
@@ -152,6 +151,59 @@ cache:
   - binaries/
 ```
 
+#### cache:key
+
+_**Note:** Introduced in GitLab Runner v1.0.0._
+
+The `key` directive allows you to define the affinity of caching
+between jobs, allowing to have a single cache for all jobs,
+cache per-job, cache per-branch or any other way you deem proper.
+
+This allows you to fine tune caching, allowing you to cache data between different jobs or even different branches.
+The `cache:key` variable can use any of the [predefined variables](../variables/README.md):
+
+Example configurations:
+
+To enable per-job caching:
+
+    ```yaml
+    cache:
+      key: "$CI_BUILD_NAME"
+      untracked: true
+    ```
+
+To enable per-branch caching:
+
+    ```yaml
+    cache:
+      key: "$CI_BUILD_REF_NAME"
+      untracked: true
+    ```
+
+To enable per-job and per-branch caching:
+
+    ```yaml
+    cache:
+      key: "$CI_BUILD_NAME/$CI_BUILD_REF_NAME"
+      untracked: true
+    ```
+
+To enable per-branch and per-stage caching:
+
+    ```yaml
+    cache:
+      key: "$CI_BUILD_STAGE/$CI_BUILD_REF_NAME"
+      untracked: true
+    ```
+
+If you use **Windows Batch** to run your shell scripts you need to replace the `$` with `%`:
+
+    ```yaml
+    cache:
+      key: "%CI_BUILD_STAGE%/%CI_BUILD_REF_NAME%"
+      untracked: true
+    ```
+
 ## Jobs
 
 `.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job

lib/ci/gitlab_ci_yaml_processor.rb

@@ -115,6 +115,10 @@ module Ci
       end
 
       if @cache
+        if @cache[:key] && !validate_string(@cache[:key])
+          raise ValidationError, "cache:key parameter should be a string"
+        end
+
         if @cache[:untracked] && !validate_boolean(@cache[:untracked])
           raise ValidationError, "cache:untracked parameter should be an boolean"
         end
@@ -198,6 +202,10 @@ module Ci
     end
 
     def validate_job_cache!(name, job)
+      if job[:cache][:key] && !validate_string(job[:cache][:key])
+        raise ValidationError, "#{name} job: cache:key parameter should be a string"
+      end
+
       if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
         raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
       end

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -336,7 +336,7 @@ module Ci
     describe "Caches" do
       it "returns cache when defined globally" do
         config = YAML.dump({
-                             cache: { paths: ["logs/", "binaries/"], untracked: true },
+                             cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'key' },
                              rspec: {
                                script: "rspec"
                              }
@@ -348,13 +348,14 @@ module Ci
         expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
           paths: ["logs/", "binaries/"],
           untracked: true,
+          key: 'key',
         )
       end
 
       it "returns cache when defined in a job" do
         config = YAML.dump({
                              rspec: {
-                               cache: { paths: ["logs/", "binaries/"], untracked: true },
+                               cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'key' },
                                script: "rspec"
                              }
                            })
@@ -365,15 +366,16 @@ module Ci
         expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
           paths: ["logs/", "binaries/"],
           untracked: true,
+          key: 'key',
         )
       end
 
       it "overwrite cache when defined for a job and globally" do
         config = YAML.dump({
-                             cache: { paths: ["logs/", "binaries/"], untracked: true },
+                             cache: { paths: ["logs/", "binaries/"], untracked: true, key: 'global' },
                              rspec: {
                                script: "rspec",
-                               cache: { paths: ["test/"], untracked: false },
+                               cache: { paths: ["test/"], untracked: false, key: 'local' },
                              }
                            })
 
@@ -383,6 +385,7 @@ module Ci
         expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
           paths: ["test/"],
           untracked: false,
+          key: 'local',
         )
       end
     end
@@ -615,6 +618,20 @@ module Ci
         end.to raise_error(GitlabCiYamlProcessor::ValidationError, "cache:paths parameter should be an array of strings")
       end
 
+      it "returns errors if cache:key is not a string" do
+        config = YAML.dump({ cache: { key: 1 }, rspec: { script: "test" } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "cache:key parameter should be a string")
+      end
+
+      it "returns errors if job cache:key is not an a string" do
+        config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", cache: { key: 1 } } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: cache:key parameter should be a string")
+      end
+
       it "returns errors if job cache:untracked is not an array of strings" do
         config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", cache: { untracked: "string" } } })
         expect do