Merge branch 'when-syntax' into 'master'

Implement when syntax in .gitlab-ci.yml

See merge request !1606

CHANGELOG

@@ -22,6 +22,7 @@ v 8.1.0 (unreleased)
   - Add first and last to pagination (Zeger-Jan van de Weg)
   - Added Commit Status API
   - Added Builds View
+  - Added when to .gitlab-ci.yml
   - Show CI status on commit page
   - Added CI_BUILD_TAG, _STAGE, _NAME and _TRIGGERED to CI builds
   - Show CI status on Your projects page and Starred projects page

app/models/ci/build.rb

@@ -93,10 +93,7 @@ module Ci
           Ci::WebHookService.new.build_end(build)
         end
 
-        if build.commit.should_create_next_builds?(build)
-          build.commit.create_next_builds(build.ref, build.tag, build.user, build.trigger_request)
-        end
-
+        build.commit.create_next_builds(build)
         project.execute_services(build)
 
         if project.coverage_enabled?

app/models/ci/commit.rb

@@ -91,19 +91,28 @@ module Ci
     def create_builds(ref, tag, user, trigger_request = nil)
       return unless config_processor
       config_processor.stages.any? do |stage|
-        CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request).present?
+        CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request, 'success').present?
       end
     end
 
-    def create_next_builds(ref, tag, user, trigger_request)
+    def create_next_builds(build)
       return unless config_processor
 
-      stages = builds.where(ref: ref, tag: tag, trigger_request: trigger_request).group_by(&:stage)
+      # don't create other builds if this one is retried
+      latest_builds = builds.similar(build).latest
+      return unless latest_builds.exists?(build.id)
 
-      config_processor.stages.any? do |stage|
-        unless stages.include?(stage)
-          CreateBuildsService.new.execute(self, stage, ref, tag, user, trigger_request).present?
-        end
+      # get list of stages after this build
+      next_stages = config_processor.stages.drop_while { |stage| stage != build.stage }
+      next_stages.delete(build.stage)
+
+      # get status for all prior builds
+      prior_builds = latest_builds.reject { |other_build| next_stages.include?(other_build.stage) }
+      status = Ci::Status.get_status(prior_builds)
+
+      # create builds for next stages based
+      next_stages.any? do |stage|
+        CreateBuildsService.new.execute(self, stage, build.ref, build.tag, build.user, build.trigger_request, status).present?
       end
     end
 
@@ -132,24 +141,7 @@ module Ci
         return 'failed'
       end
 
-      @status ||= begin
-        latest = latest_statuses
-        latest.reject! { |status| status.try(&:allow_failure?) }
-
-        if latest.none?
-          'skipped'
-        elsif latest.all?(&:success?)
-          'success'
-        elsif latest.all?(&:pending?)
-          'pending'
-        elsif latest.any?(&:running?) || latest.any?(&:pending?)
-          'running'
-        elsif latest.all?(&:canceled?)
-          'canceled'
-        else
-          'failed'
-        end
-      end
+      @status ||= Ci::Status.get_status(latest_statuses)
     end
 
     def pending?
@@ -219,16 +211,6 @@ module Ci
       update!(committed_at: DateTime.now)
     end
 
-    def should_create_next_builds?(build)
-      # don't create other builds if this one is retried
-      other_builds = builds.similar(build).latest
-      return false unless other_builds.include?(build)
-
-      other_builds.all? do |build|
-        build.success? || build.ignored?
-      end
-    end
-
     private
 
     def save_yaml_error(error)

app/models/commit_status.rb

@@ -28,7 +28,7 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :drop do
-      transition running: :failed
+      transition [:pending, :running] => :failed
     end
 
     event :success do

app/services/ci/create_builds_service.rb

 module Ci
   class CreateBuildsService
-    def execute(commit, stage, ref, tag, user, trigger_request)
+    def execute(commit, stage, ref, tag, user, trigger_request, status)
       builds_attrs = commit.config_processor.builds_for_stage_and_ref(stage, ref, tag)
 
+      # check when to create next build
+      builds_attrs = builds_attrs.select do |build_attrs|
+        case build_attrs[:when]
+        when 'on_success'
+          status == 'success'
+        when 'on_failure'
+          status == 'failed'
+        when 'always'
+          %w(success failed).include?(status)
+        end
+      end
+
       builds_attrs.map do |build_attrs|
         # don't create the same build twice
         unless commit.builds.find_by(ref: ref, tag: tag, trigger_request: trigger_request, name: build_attrs[:name])

doc/ci/yaml/README.md

@@ -140,6 +140,7 @@ job_name:
 | except        | optional | Defines a list of git refs for which build is not created |
 | tags          | optional | Defines a list of tags which are used to select runner |
 | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
+| when          | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
 
 ### script
 `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
@@ -196,6 +197,55 @@ job:
 
 The above specification will make sure that `job` is built by a runner that have `ruby` AND `postgres` tags defined.
 
+### when
+`when` is used to implement jobs that are run in case of failure or despite the failure.
+
+`when` can be set to one of the following values:
+
+1. `on_success` - execute build only when all builds from prior stages succeeded. This is the default.
+1. `on_failure` - execute build only when at least one build from prior stages failed.
+1. `always` - execute build despite the status of builds from prior stages.
+
+```
+stages:
+- build
+- cleanup_build
+- test
+- deploy
+- cleanup
+
+build:
+  stage: build
+  script:
+  - make build
+
+cleanup_build:
+  stage: cleanup_build
+  script:
+  - cleanup build when failed
+  when: on_failure
+
+test:
+  stage: test
+  script:
+  - make test
+
+deploy:
+  stage: deploy
+  script:
+  - make deploy
+
+cleanup:
+  stage: cleanup
+  script:
+  - cleanup after builds
+  when: always
+```
+
+The above script will:
+1. Execute `cleanup_build` only when the `build` failed,
+2. Always execute `cleanup` as the last step in pipeline.
+
 ## Validate the .gitlab-ci.yml
 Each instance of GitLab CI has an embedded debug tool called Lint.
 You can find the link to the Lint in the project's settings page or use short url `/lint`.

lib/ci/gitlab_ci_yaml_processor.rb

@@ -5,7 +5,7 @@ module Ci
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
     ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
-    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage]
+    ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
 
     attr_reader :before_script, :image, :services, :variables
 
@@ -93,6 +93,7 @@ module Ci
         only: job[:only],
         except: job[:except],
         allow_failure: job[:allow_failure] || false,
+        when: job[:when] || 'on_success',
         options: {
           image: job[:image] || @image,
           services: job[:services] || @services
@@ -184,6 +185,10 @@ module Ci
       if job[:allow_failure] && !job[:allow_failure].in?([true, false])
         raise ValidationError, "#{name}: allow_failure parameter should be an boolean"
       end
+
+      if job[:when] && !job[:when].in?(%w(on_success on_failure always))
+        raise ValidationError, "#{name}: when parameter should be on_success, on_failure or always"
+      end
     end
 
     private

lib/ci/status.rb

+module Ci
+  class Status
+    def self.get_status(statuses)
+      statuses.reject! { |status| status.try(&:allow_failure?) }
+
+      if statuses.none?
+        'skipped'
+      elsif statuses.all?(&:success?)
+        'success'
+      elsif statuses.all?(&:pending?)
+        'pending'
+      elsif statuses.any?(&:running?) || statuses.any?(&:pending?)
+        'running'
+      elsif statuses.all?(&:canceled?)
+        'canceled'
+      else
+        'failed'
+      end
+    end
+  end
+end

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -24,7 +24,8 @@ module Ci
           commands: "pwd\nrspec",
           tag_list: [],
           options: {},
-          allow_failure: false
+          allow_failure: false,
+          when: "on_success"
         })
       end
 
@@ -125,7 +126,8 @@ module Ci
             image: "ruby:2.1",
             services: ["mysql"]
           },
-          allow_failure: false
+          allow_failure: false,
+          when: "on_success"
         })
       end
 
@@ -152,7 +154,8 @@ module Ci
             image: "ruby:2.5",
             services: ["postgresql"]
           },
-          allow_failure: false
+          allow_failure: false,
+          when: "on_success"
         })
       end
     end
@@ -174,6 +177,21 @@ module Ci
       end
     end
 
+    describe "When" do
+      %w(on_success on_failure always).each do |when_state|
+        it "returns #{when_state} when defined" do
+          config = YAML.dump({
+                               rspec: { script: "rspec", when: when_state }
+                             })
+
+          config_processor = GitlabCiYamlProcessor.new(config)
+          builds = config_processor.builds_for_stage_and_ref("test", "master")
+          expect(builds.size).to eq(1)
+          expect(builds.first[:when]).to eq(when_state)
+        end
+      end
+    end
+
     describe "Error handling" do
       it "indicates that object is invalid" do
         expect{GitlabCiYamlProcessor.new("invalid_yaml\n!ccdvlf%612334@@@@")}.to raise_error(GitlabCiYamlProcessor::ValidationError)
@@ -311,6 +329,13 @@ module Ci
           GitlabCiYamlProcessor.new(config)
         end.to raise_error(GitlabCiYamlProcessor::ValidationError, "variables should be a map of key-valued strings")
       end
+
+      it "returns errors if job when is not on_success, on_failure or always" do
+        config = YAML.dump({ rspec: { script: "test", when: 1 } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: when parameter should be on_success, on_failure or always")
+      end
     end
   end
 end