Port to EE: Delayed jobs

app/assets/javascripts/lib/utils/datetime_utility.js

@@ -370,3 +370,24 @@ window.gl.utils = {
   getTimeago,
   localTimeAgo,
 };
+
+/**
+ * Formats milliseconds as timestamp (e.g. 01:02:03).
+ * This takes durations longer than a day into account (e.g. two days would be 48:00:00).
+ *
+ * @param milliseconds
+ * @returns {string}
+ */
+export const formatTime = milliseconds => {
+  const remainingSeconds = Math.floor(milliseconds / 1000) % 60;
+  const remainingMinutes = Math.floor(milliseconds / 1000 / 60) % 60;
+  const remainingHours = Math.floor(milliseconds / 1000 / 60 / 60);
+  let formattedTime = '';
+  if (remainingHours < 10) formattedTime += '0';
+  formattedTime += `${remainingHours}:`;
+  if (remainingMinutes < 10) formattedTime += '0';
+  formattedTime += `${remainingMinutes}:`;
+  if (remainingSeconds < 10) formattedTime += '0';
+  formattedTime += remainingSeconds;
+  return formattedTime;
+};

app/assets/javascripts/pipelines/components/pipelines_actions.vue

 <script>
+import { s__, sprintf } from '~/locale';
+import { formatTime } from '~/lib/utils/datetime_utility';
 import eventHub from '../event_hub';
 import icon from '../../vue_shared/components/icon.vue';
 import tooltip from '../../vue_shared/directives/tooltip';
@@ -22,10 +24,24 @@ export default {
     };
   },
   methods: {
-    onClickAction(endpoint) {
+    onClickAction(action) {
+      if (action.scheduled_at) {
+        const confirmationMessage = sprintf(
+          s__(
+            "DelayedJobs|Are you sure you want to run %{jobName} immediately? This job will run automatically after it's timer finishes.",
+          ),
+          { jobName: action.name },
+        );
+        // https://gitlab.com/gitlab-org/gitlab-ce/issues/52156
+        // eslint-disable-next-line no-alert
+        if (!window.confirm(confirmationMessage)) {
+          return;
+        }
+      }
+
       this.isLoading = true;
 
-      eventHub.$emit('postAction', endpoint);
+      eventHub.$emit('postAction', action.path);
     },
 
     isActionDisabled(action) {
@@ -35,6 +51,11 @@ export default {
 
       return !action.playable;
     },
+
+    remainingTime(action) {
+      const remainingMilliseconds = new Date(action.scheduled_at).getTime() - Date.now();
+      return formatTime(Math.max(0, remainingMilliseconds));
+    },
   },
 };
 </script>
@@ -63,17 +84,24 @@ export default {
 
     <ul class="dropdown-menu dropdown-menu-right">
       <li
-        v-for="(action, i) in actions"
-        :key="i"
+        v-for="action in actions"
+        :key="action.path"
       >
         <button
           :class="{ disabled: isActionDisabled(action) }"
           :disabled="isActionDisabled(action)"
           type="button"
           class="js-pipeline-action-link no-btn btn"
-          @click="onClickAction(action.path)"
+          @click="onClickAction(action)"
         >
           {{ action.name }}
+          <span
+            v-if="action.scheduled_at"
+            class="pull-right"
+          >
+            <icon name="clock" />
+            {{ remainingTime(action) }}
+          </span>
         </button>
       </li>
     </ul>

app/assets/javascripts/pipelines/components/pipelines_table_row.vue

@@ -59,6 +59,16 @@ export default {
     };
   },
   computed: {
+    actions() {
+      if (!this.pipeline || !this.pipeline.details) {
+        return [];
+      }
+      const { details } = this.pipeline;
+      return [
+        ...(details.manual_actions || []),
+        ...(details.scheduled_actions || []),
+      ];
+    },
     /**
      * If provided, returns the commit tag.
      * Needed to render the commit component column.
@@ -321,8 +331,8 @@ export default {
     >
       <div class="btn-group table-action-buttons">
         <pipelines-actions-component
-          v-if="pipeline.details.manual_actions.length"
-          :actions="pipeline.details.manual_actions"
+          v-if="actions.length > 0"
+          :actions="actions"
         />
 
         <pipelines-artifacts-component

app/assets/stylesheets/framework/buttons.scss

@@ -360,6 +360,10 @@
   i {
     color: $gl-text-color-secondary;
   }
+
+  svg {
+    fill: $gl-text-color-secondary;
+  }
 }
 
 .clone-dropdown-btn a {

app/assets/stylesheets/framework/icons.scss

@@ -64,6 +64,7 @@
   }
 }
 
+.ci-status-icon-scheduled,
 .ci-status-icon-manual {
   svg {
     fill: $gl-text-color;

app/assets/stylesheets/pages/pipelines.scss

@@ -762,6 +762,7 @@
   }
 
   &.ci-status-icon-canceled,
+  &.ci-status-icon-scheduled,
   &.ci-status-icon-disabled,
   &.ci-status-icon-not-found,
   &.ci-status-icon-manual {

app/assets/stylesheets/pages/status.scss

@@ -27,6 +27,7 @@
 
   &.ci-canceled,
   &.ci-disabled,
+  &.ci-scheduled,
   &.ci-manual {
     color: $gl-text-color;
     border-color: $gl-text-color;

app/controllers/projects/jobs_controller.rb

@@ -110,6 +110,13 @@ class Projects::JobsController < Projects::ApplicationController
     redirect_to build_path(@build)
   end
 
+  def unschedule
+    return respond_422 unless @build.scheduled?
+
+    @build.unschedule!
+    redirect_to build_path(@build)
+  end
+
   def status
     render json: BuildSerializer
       .new(project: @project, current_user: @current_user)

app/helpers/ci_status_helper.rb

@@ -20,6 +20,8 @@ module CiStatusHelper
               'passed with warnings'
             when 'manual'
               'waiting for manual action'
+            when 'scheduled'
+              'waiting for delayed job'
             else
               status
             end
@@ -39,6 +41,8 @@ module CiStatusHelper
       s_('CiStatusText|passed')
     when 'manual'
       s_('CiStatusText|blocked')
+    when 'scheduled'
+      s_('CiStatusText|scheduled')
     else
       # All states are already being translated inside the detailed statuses:
       # :running => Gitlab::Ci::Status::Running
@@ -83,6 +87,8 @@ module CiStatusHelper
         'status_skipped'
       when 'manual'
         'status_manual'
+      when 'scheduled'
+        'status_scheduled'
       else
         'status_canceled'
       end

app/helpers/time_helper.rb

@@ -21,9 +21,17 @@ module TimeHelper
     "#{from.to_s(:short)} - #{to.to_s(:short)}"
   end
 
-  def duration_in_numbers(duration)
-    time_format = duration < 1.hour ? "%M:%S" : "%H:%M:%S"
+  def duration_in_numbers(duration_in_seconds, allow_overflow = false)
+    if allow_overflow
+      seconds = duration_in_seconds % 1.minute
+      minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute)
+      hours = duration_in_seconds / 1.hour
 
-    Time.at(duration).utc.strftime(time_format)
+      "%02d:%02d:%02d" % [hours, minutes, seconds]
+    else
+      time_format = duration_in_seconds < 1.hour ? "%M:%S" : "%H:%M:%S"
+
+      Time.at(duration_in_seconds).utc.strftime(time_format)
+    end
   end
 end

app/models/ci/build.rb

@@ -96,7 +96,8 @@ module Ci
     scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
-    scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
+    scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) }
+    scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) }
     scope :ref_protected, -> { where(protected: true) }
     scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) }
 
@@ -163,6 +164,34 @@ module Ci
         transition created: :manual
       end
 
+      event :schedule do
+        transition created: :scheduled
+      end
+
+      event :unschedule do
+        transition scheduled: :manual
+      end
+
+      event :enqueue_scheduled do
+        transition scheduled: :pending, if: ->(build) do
+          build.scheduled_at && build.scheduled_at < Time.now
+        end
+      end
+
+      before_transition scheduled: any do |build|
+        build.scheduled_at = nil
+      end
+
+      before_transition created: :scheduled do |build|
+        build.scheduled_at = build.options_scheduled_at
+      end
+
+      after_transition created: :scheduled do |build|
+        build.run_after_commit do
+          Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id)
+        end
+      end
+
       after_transition any => [:pending] do |build|
         build.run_after_commit do
           BuildQueueWorker.perform_async(id)
@@ -230,11 +259,20 @@ module Ci
     end
 
     def playable?
-      action? && (manual? || retryable?)
+      action? && (manual? || scheduled? || retryable?)
+    end
+
+    def schedulable?
+      Feature.enabled?('ci_enable_scheduled_build', default_enabled: true) &&
+        self.when == 'delayed' && options[:start_in].present?
+    end
+
+    def options_scheduled_at
+      ChronicDuration.parse(options[:start_in])&.seconds&.from_now
     end
 
     def action?
-      self.when == 'manual'
+      %w[manual delayed].include?(self.when)
     end
 
     # rubocop: disable CodeReuse/ServiceClass

app/models/ci/pipeline.rb

@@ -46,6 +46,7 @@ module Ci
     has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
     has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
     has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
+    has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
     has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
 
     has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
@@ -93,7 +94,7 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition [:created, :skipped] => :pending
+        transition [:created, :skipped, :scheduled] => :pending
         transition [:success, :failed, :canceled] => :running
       end
 
@@ -121,6 +122,10 @@ module Ci
         transition any - [:manual] => :manual
       end
 
+      event :delay do
+        transition any - [:scheduled] => :scheduled
+      end
+
       # IMPORTANT
       # Do not add any operations to this state_machine
       # Create a separate worker for each new operation
@@ -557,6 +562,7 @@ module Ci
         when 'canceled' then cancel
         when 'skipped' then skip
         when 'manual' then block
+        when 'scheduled' then delay
         else
           raise HasStatus::UnknownStatusError,
                 "Unknown status `#{latest_builds_status}`"

app/models/ci/stage.rb

@@ -65,6 +65,10 @@ module Ci
       event :block do
         transition any - [:manual] => :manual
       end
+
+      event :delay do
+        transition any - [:scheduled] => :scheduled
+      end
     end
 
     def update_status
@@ -77,6 +81,7 @@ module Ci
         when 'failed' then drop
         when 'canceled' then cancel
         when 'manual' then block
+        when 'scheduled' then delay
         when 'skipped', nil then skip
         else
           raise HasStatus::UnknownStatusError,

app/models/commit_status.rb

@@ -51,7 +51,8 @@ class CommitStatus < ActiveRecord::Base
     stuck_or_timeout_failure: 3,
     runner_system_failure: 4,
     missing_dependency_failure: 5,
-    runner_unsupported: 6
+    runner_unsupported: 6,
+    stale_schedule: 7
   }.merge(EE_FAILURE_REASONS)
 
   ##
@@ -73,7 +74,7 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :enqueue do
-      transition [:created, :skipped, :manual] => :pending
+      transition [:created, :skipped, :manual, :scheduled] => :pending
     end
 
     event :run do
@@ -85,7 +86,7 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :drop do
-      transition [:created, :pending, :running] => :failed
+      transition [:created, :pending, :running, :scheduled] => :failed
     end
 
     event :success do
@@ -93,10 +94,10 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :cancel do
-      transition [:created, :pending, :running, :manual] => :canceled
+      transition [:created, :pending, :running, :manual, :scheduled] => :canceled
     end
 
-    before_transition [:created, :skipped, :manual] => :pending do |commit_status|
+    before_transition [:created, :skipped, :manual, :scheduled] => :pending do |commit_status|
       commit_status.queued_at = Time.now
     end
 

app/models/concerns/has_status.rb

@@ -4,14 +4,15 @@ module HasStatus
   extend ActiveSupport::Concern
 
   DEFAULT_STATUS = 'created'.freeze
-  BLOCKED_STATUS = 'manual'.freeze
-  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual].freeze
-  STARTED_STATUSES = %w[running success failed skipped manual].freeze
+  BLOCKED_STATUS = %w[manual scheduled].freeze
+  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual scheduled].freeze
+  STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze
   ACTIVE_STATUSES = %w[pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
-  ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze
+  ORDERED_STATUSES = %w[failed pending running manual scheduled canceled success skipped created].freeze
   STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
-                    failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
+                    failed: 4, canceled: 5, skipped: 6, manual: 7,
+                    scheduled: 8 }.freeze
 
   UnknownStatusError = Class.new(StandardError)
 
@@ -24,6 +25,7 @@ module HasStatus
       created = scope_relevant.created.select('count(*)').to_sql
       success = scope_relevant.success.select('count(*)').to_sql
       manual = scope_relevant.manual.select('count(*)').to_sql
+      scheduled = scope_relevant.scheduled.select('count(*)').to_sql
       pending = scope_relevant.pending.select('count(*)').to_sql
       running = scope_relevant.running.select('count(*)').to_sql
       skipped = scope_relevant.skipped.select('count(*)').to_sql
@@ -40,6 +42,7 @@ module HasStatus
         WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending'
         WHEN (#{running})+(#{pending})>0 THEN 'running'
         WHEN (#{manual})>0 THEN 'manual'
+        WHEN (#{scheduled})>0 THEN 'scheduled'
         WHEN (#{created})>0 THEN 'running'
         ELSE 'failed'
       END)"
@@ -74,6 +77,7 @@ module HasStatus
       state :canceled, value: 'canceled'
       state :skipped, value: 'skipped'
       state :manual, value: 'manual'
+      state :scheduled, value: 'scheduled'
     end
 
     scope :created, -> { where(status: 'created') }
@@ -85,6 +89,7 @@ module HasStatus
     scope :canceled, -> { where(status: 'canceled')  }
     scope :skipped, -> { where(status: 'skipped')  }
     scope :manual, -> { where(status: 'manual')  }
+    scope :scheduled, -> { where(status: 'scheduled')  }
     scope :alive, -> { where(status: [:created, :pending, :running]) }
     scope :created_or_pending, -> { where(status: [:created, :pending]) }
     scope :running_or_pending, -> { where(status: [:running, :pending]) }
@@ -92,7 +97,7 @@ module HasStatus
     scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) }
 
     scope :cancelable, -> do
-      where(status: [:running, :pending, :created])
+      where(status: [:running, :pending, :created, :scheduled])
     end
   end
 
@@ -109,7 +114,7 @@ module HasStatus
   end
 
   def blocked?
-    BLOCKED_STATUS == status
+    BLOCKED_STATUS.include?(status)
   end
 
   private

app/presenters/ci/build_presenter.rb

@@ -35,6 +35,10 @@ module Ci
       "#{subject.name} - #{detailed_status.status_tooltip}"
     end
 
+    def execute_in
+      scheduled? && scheduled_at && [0, scheduled_at - Time.now].max
+    end
+
     private
 
     def tooltip_for_badge

app/presenters/commit_status_presenter.rb

@@ -7,7 +7,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     stuck_or_timeout_failure: 'There has been a timeout failure or the job got stuck. Check your timeout limits or try again',
     runner_system_failure: 'There has been a runner system failure, please try again',
     missing_dependency_failure: 'There has been a missing dependency failure',
-    runner_unsupported: 'Your runner is outdated, please upgrade your runner'
+    runner_unsupported: 'Your runner is outdated, please upgrade your runner',
+    stale_schedule: 'Delayed job could not be executed by some reason, please try again'
   }.freeze
 
   private_constant :CALLOUT_FAILURE_MESSAGES

app/serializers/build_action_entity.rb

@@ -12,6 +12,11 @@ class BuildActionEntity < Grape::Entity
   end
 
   expose :playable?, as: :playable
+  expose :scheduled_at, if: -> (build) { build.scheduled? }
+
+  expose :unschedule_path, if: -> (build) { build.scheduled? } do |build|
+    unschedule_project_job_path(build.project, build)
+  end
 
   private
 

app/serializers/job_entity.rb

@@ -24,7 +24,12 @@ class JobEntity < Grape::Entity
     path_to(:play_namespace_project_job, build)
   end
 
+  expose :unschedule_path, if: -> (*) { scheduled? } do |build|
+    path_to(:unschedule_namespace_project_job, build)
+  end
+
   expose :playable?, as: :playable
+  expose :scheduled_at, if: -> (*) { scheduled? }
   expose :created_at
   expose :updated_at
   expose :detailed_status, as: :status, with: DetailedStatusEntity
@@ -47,6 +52,10 @@ class JobEntity < Grape::Entity
     build.playable? && can?(request.current_user, :update_build, build)
   end
 
+  def scheduled?
+    build.scheduled?
+  end
+
   def detailed_status
     build.detailed_status(request.current_user)
   end

app/serializers/pipeline_details_entity.rb

@@ -5,6 +5,7 @@ class PipelineDetailsEntity < PipelineEntity
     expose :ordered_stages, as: :stages, using: StageEntity
     expose :artifacts, using: BuildArtifactEntity
     expose :manual_actions, using: BuildActionEntity
+    expose :scheduled_actions, using: BuildActionEntity
   end
 
   expose :triggered_by_pipeline, as: :triggered_by, with: TriggeredPipelineEntity

app/serializers/pipeline_serializer.rb

@@ -15,6 +15,7 @@ class PipelineSerializer < BaseSerializer
         { triggered_by_pipeline: [:project, :user] },
         { triggered_pipelines: [:project, :user] },
         :manual_actions,
+        :scheduled_actions,
         :artifacts,
         {
           pending_builds: :project,

app/services/ci/enqueue_build_service.rb

-# frozen_string_literal: true
-module Ci
-  class EnqueueBuildService < BaseService
-    prepend EE::Ci::EnqueueBuildService
-
-    def execute(build)
-      build.enqueue
-    end
-  end
-end

app/services/ci/process_build_service.rb

+# frozen_string_literal: true
+
+module Ci
+  class ProcessBuildService < BaseService
+    prepend EE::Ci::ProcessBuildService
+
+    def execute(build, current_status)
+      if valid_statuses_for_when(build.when).include?(current_status)
+        if build.schedulable?
+          build.schedule
+        elsif build.action?
+          build.actionize
+        else
+          enqueue(build)
+        end
+
+        true
+      else
+        build.skip
+        false
+      end
+    end
+
+    private
+
+    def enqueue(build)
+      build.enqueue
+    end
+
+    def valid_statuses_for_when(value)
+      case value
+      when 'on_success'
+        %w[success skipped]
+      when 'on_failure'
+        %w[failed]
+      when 'always'
+        %w[success failed skipped]
+      when 'manual'
+        %w[success skipped]
+      when 'delayed'
+        %w[success skipped]
+      else
+        []
+      end
+    end
+  end
+end

app/services/ci/process_pipeline_service.rb

@@ -24,42 +24,18 @@ module Ci
     def process_stage(index)
       current_status = status_for_prior_stages(index)
 
-      return if HasStatus::BLOCKED_STATUS == current_status
+      return if HasStatus::BLOCKED_STATUS.include?(current_status)
 
       if HasStatus::COMPLETED_STATUSES.include?(current_status)
         created_builds_in_stage(index).select do |build|
           Gitlab::OptimisticLocking.retry_lock(build) do |subject|
-            process_build(subject, current_status)
+            Ci::ProcessBuildService.new(project, @user)
+              .execute(build, current_status)
           end
         end
       end
     end
 
-    def process_build(build, current_status)
-      if valid_statuses_for_when(build.when).include?(current_status)
-        build.action? ? build.actionize : enqueue_build(build)
-        true
-      else
-        build.skip
-        false
-      end
-    end
-
-    def valid_statuses_for_when(value)
-      case value
-      when 'on_success'
-        %w[success skipped]
-      when 'on_failure'
-        %w[failed]
-      when 'always'
-        %w[success failed skipped]
-      when 'manual'
-        %w[success skipped]
-      else
-        []
-      end
-    end
-
     # rubocop: disable CodeReuse/ActiveRecord
     def status_for_prior_stages(index)
       pipeline.builds.where('stage_idx < ?', index).latest.status || 'success'
@@ -101,9 +77,5 @@ module Ci
         .update_all(retried: true) if latest_statuses.any?
     end
     # rubocop: enable CodeReuse/ActiveRecord
-
-    def enqueue_build(build)
-      Ci::EnqueueBuildService.new(project, @user).execute(build)
-    end
   end
 end

app/services/ci/run_scheduled_build_service.rb

+# frozen_string_literal: true
+
+module Ci
+  class RunScheduledBuildService < ::BaseService
+    def execute(build)
+      unless can?(current_user, :update_build, build)
+        raise Gitlab::Access::AccessDeniedError
+      end
+
+      build.enqueue_scheduled!
+    end
+  end
+end

app/views/projects/ci/builds/_build.html.haml

@@ -47,7 +47,9 @@
         %span.badge.badge-info triggered
       - if job.try(:allow_failure)
         %span.badge.badge-danger allowed to fail
-      - if job.action?
+      - if job.schedulable?
+        %span.badge.badge-info= s_('DelayedJobs|scheduled')
+      - elsif job.action?
         %span.badge.badge-info manual
 
   - if pipeline_link
@@ -101,6 +103,24 @@
         - if job.active?
           = link_to cancel_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Cancel', class: 'btn btn-build' do
             = icon('remove', class: 'cred')
+        - elsif job.scheduled?
+          .btn-group
+            .btn.btn-default.has-tooltip{ disabled: true,
+              title: job.scheduled_at }
+              = sprite_icon('planning')
+              = duration_in_numbers(job.execute_in, true)
+            - confirmation_message = s_("DelayedJobs|Are you sure you want to run %{job_name} immediately? This job will run automatically after it's timer finishes.") % { job_name: job.name }
+            = link_to play_project_job_path(job.project, job, return_to: request.original_url),
+              method: :post,
+              title: s_('DelayedJobs|Start now'),
+              class: 'btn btn-default btn-build has-tooltip',
+              data: { confirm: confirmation_message } do
+              = sprite_icon('play')
+            = link_to unschedule_project_job_path(job.project, job, return_to: request.original_url),
+              method: :post,
+              title: s_('DelayedJobs|Unschedule'),
+              class: 'btn btn-default btn-build has-tooltip' do
+              = sprite_icon('time-out')
         - elsif allow_retry
           - if job.playable? && !admin && can?(current_user, :update_build, job)
             = link_to play_project_job_path(job.project, job, return_to: request.original_url), method: :post, title: 'Play', class: 'btn btn-build' do

app/views/shared/icons/_icon_status_scheduled.svg

+<svg width="14" height="14" viewBox="0 0 14 14" xmlns="http://www.w3.org/2000/svg"><circle cx="7" cy="7" r="7"/><circle fill="#FFF" cx="7" cy="7" r="6"/><g transform="translate(2.75 2.75)" fill-rule="nonzero"><path d="M4.165 7.81a3.644 3.644 0 1 1 0-7.29 3.644 3.644 0 0 1 0 7.29zm0-1.042a2.603 2.603 0 1 0 0-5.206 2.603 2.603 0 0 0 0 5.206z"/><rect x="3.644" y="2.083" width="1.041" height="2.603" rx=".488"/><rect x="3.644" y="3.644" width="2.083" height="1.041" rx=".488"/></g></svg>
\ No newline at end of file

app/views/shared/icons/_icon_status_scheduled_borderless.svg

+<svg width="22px" height="22px" viewBox="0 0 22 22" version="1.1" xmlns="http://www.w3.org/2000/svg"><path d="M6.16 11.55a5.39 5.39 0 1 1 0-10.78 5.39 5.39 0 0 1 0 10.78zm0-1.54a3.85 3.85 0 1 0 0-7.7 3.85 3.85 0 0 0 0 7.7z"/><rect x="5.39" y="3.08" width="1.54" height="3.85" rx=".767"/><rect x="5.39" y="5.39" width="3.08" height="1.54" rx=".767"/></svg>
\ No newline at end of file

app/workers/all_queues.yml

@@ -72,6 +72,7 @@
 - pipeline_processing:pipeline_update
 - pipeline_processing:stage_update
 - pipeline_processing:update_head_pipeline_for_merge_request
+- pipeline_processing:ci_build_schedule
 
 - repository_check:repository_check_clear
 - repository_check:repository_check_batch

app/workers/ci/build_schedule_worker.rb

+# frozen_string_literal: true
+
+module Ci
+  class BuildScheduleWorker
+    include ApplicationWorker
+    include PipelineQueue
+
+    queue_namespace :pipeline_processing
+
+    def perform(build_id)
+      ::Ci::Build.find_by_id(build_id).try do |build|
+        break unless build.scheduled?
+
+        Ci::RunScheduledBuildService
+          .new(build.project, build.user).execute(build)
+      end
+    end
+  end
+end

app/workers/stuck_ci_jobs_worker.rb

@@ -8,6 +8,7 @@ class StuckCiJobsWorker
 
   BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
+  BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
   BUILD_PENDING_STUCK_TIMEOUT = 1.hour
 
   def perform
@@ -15,9 +16,10 @@ class StuckCiJobsWorker
 
     Rails.logger.info "#{self.class}: Cleaning stuck builds"
 
-    drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT
-    drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT
-    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT
+    drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
+    drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
+    drop :scheduled, BUILD_SCHEDULED_OUTDATED_TIMEOUT, 'scheduled_at IS NOT NULL AND scheduled_at < ?', :stale_schedule
+    drop_stuck :pending, BUILD_PENDING_STUCK_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
 
     remove_lease
   end
@@ -32,25 +34,25 @@ class StuckCiJobsWorker
     Gitlab::ExclusiveLease.cancel(EXCLUSIVE_LEASE_KEY, @uuid)
   end
 
-  def drop(status, timeout)
-    search(status, timeout) do |build|
-      drop_build :outdated, build, status, timeout
+  def drop(status, timeout, condition, reason)
+    search(status, timeout, condition) do |build|
+      drop_build :outdated, build, status, timeout, reason
     end
   end
 
-  def drop_stuck(status, timeout)
-    search(status, timeout) do |build|
+  def drop_stuck(status, timeout, condition, reason)
+    search(status, timeout, condition) do |build|
       break unless build.stuck?
 
-      drop_build :stuck, build, status, timeout
+      drop_build :stuck, build, status, timeout, reason
     end
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
-  def search(status, timeout)
+  def search(status, timeout, condition)
     loop do
       jobs = Ci::Build.where(status: status)
-        .where('ci_builds.updated_at < ?', timeout.ago)
+        .where(condition, timeout.ago)
         .includes(:tags, :runner, project: :namespace)
         .limit(100)
         .to_a
@@ -63,10 +65,10 @@ class StuckCiJobsWorker
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
-  def drop_build(type, build, status, timeout)
-    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout})"
+  def drop_build(type, build, status, timeout, reason)
+    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})"
     Gitlab::OptimisticLocking.retry_lock(build, 3) do |b|
-      b.drop(:stuck_or_timeout_failure)
+      b.drop(reason)
     end
   end
 end

changelogs/unreleased/scheduled-manual-jobs.yml

+---
+title: Allow pipelines to schedule delayed job runs
+merge_request: 21767
+author:
+type: added

config/routes/project.rb

@@ -330,6 +330,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           member do
             get :status
             post :cancel
+            post :unschedule
             post :retry
             post :play
             post :erase

db/migrate/20180924190739_add_scheduled_at_to_ci_builds.rb

+# frozen_string_literal: true
+
+class AddScheduledAtToCiBuilds < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :ci_builds, :scheduled_at, :datetime_with_timezone
+  end
+end

db/migrate/20180924201039_add_partial_index_to_scheduled_at.rb

+# frozen_string_literal: true
+
+class AddPartialIndexToScheduledAt < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs'.freeze
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, :scheduled_at, where: "scheduled_at IS NOT NULL AND type = 'Ci::Build' AND status = 'scheduled'", name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
+  end
+end

db/schema.rb

@@ -429,6 +429,7 @@ ActiveRecord::Schema.define(version: 20181001172651) do
     t.integer "artifacts_metadata_store"
     t.boolean "protected"
     t.integer "failure_reason"
+    t.datetime_with_timezone "scheduled_at"
   end
 
   add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
@@ -442,6 +443,7 @@ ActiveRecord::Schema.define(version: 20181001172651) do
   add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
+  add_index "ci_builds", ["scheduled_at"], name: "partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs", where: "((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text))", using: :btree
   add_index "ci_builds", ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)", using: :btree
   add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree
   add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree

ee/app/services/ee/ci/enqueue_build_service.rb → ee/app/services/ee/ci/process_build_service.rb

 # frozen_string_literal: true
 module EE
   module Ci
-    module EnqueueBuildService
+    module ProcessBuildService
       extend ::Gitlab::Utils::Override
 
-      override :execute
-      def execute(build)
+      override :enqueue
+      def enqueue(build)
         unless allowed_to_deploy?(build)
           return build.drop!(:protected_environment_failure)
         end

ee/spec/services/ci/enqueue_build_service_spec.rb → ee/spec/services/ci/process_build_service_spec.rb

 # frozen_string_literal: true
 require 'spec_helper'
 
-describe Ci::EnqueueBuildService, '#execute' do
+describe Ci::ProcessBuildService, '#execute' do
   let(:user) { create(:user) }
   let(:project) { create(:project, :repository) }
   let(:environment) { create(:environment, project: project, name: 'production') }
   let(:protected_environment) { create(:protected_environment, name: environment.name, project: project) }
-  let(:ci_build) { create(:ci_build, :created, environment: environment.name, user: user) }
+  let(:ci_build) { create(:ci_build, :created, environment: environment.name, user: user, when: :on_success) }
+  let(:current_status) { 'success' }
 
-  subject { described_class.new(project, user).execute(ci_build) }
+  subject { described_class.new(project, user).execute(ci_build, current_status) }
 
   before do
     allow(License).to receive(:feature_available?).and_call_original

lib/api/jobs.rb

@@ -151,7 +151,7 @@ module API
         present build, with: Entities::Job
       end
 
-      desc 'Trigger a manual job' do
+      desc 'Trigger a actionable job (manual, scheduled, etc)' do
         success Entities::Job
         detail 'This feature was added in GitLab 8.11'
       end

lib/gitlab/ci/config/entry/job.rb

@@ -10,7 +10,7 @@ module Gitlab
           include Attributable
 
           ALLOWED_KEYS = %i[tags script only except type image services
-                            allow_failure type stage when artifacts cache
+                            allow_failure type stage when start_in artifacts cache
                             dependencies before_script after_script variables
                             environment coverage retry extends].freeze
 
@@ -28,13 +28,16 @@ module Gitlab
                                                 greater_than_or_equal_to: 0,
                                                 less_than_or_equal_to: 2 }
               validates :when,
-                inclusion: { in: %w[on_success on_failure always manual],
+                inclusion: { in: %w[on_success on_failure always manual delayed],
                              message: 'should be on_success, on_failure, ' \
-                                      'always or manual' }
+                                      'always, manual or delayed' }
 
               validates :dependencies, array_of_strings: true
               validates :extends, type: String
             end
+
+            validates :start_in, duration: { limit: '1 day' }, if: :delayed?
+            validates :start_in, absence: true, unless: :delayed?
           end
 
           entry :before_script, Entry::Script,
@@ -84,7 +87,7 @@ module Gitlab
                   :artifacts, :commands, :environment, :coverage, :retry
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
-                     :retry, :extends
+                     :retry, :extends, :start_in
 
           def compose!(deps = nil)
             super do
@@ -114,6 +117,10 @@ module Gitlab
             self.when == 'manual'
           end
 
+          def delayed?
+            self.when == 'delayed'
+          end
+
           def ignored?
             allow_failure.nil? ? manual_action? : allow_failure
           end

lib/gitlab/ci/config/entry/legacy_validation_helpers.rb

@@ -11,6 +11,15 @@ module Gitlab
             false
           end
 
+          def validate_duration_limit(value, limit)
+            return false unless value.is_a?(String)
+
+            ChronicDuration.parse(value).second.from_now <
+              ChronicDuration.parse(limit).second.from_now
+          rescue ChronicDuration::DurationParseError
+            false
+          end
+
           def validate_array_of_strings(values)
             values.is_a?(Array) && values.all? { |value| validate_string(value) }
           end

lib/gitlab/ci/config/entry/validators.rb

@@ -49,6 +49,12 @@ module Gitlab
               unless validate_duration(value)
                 record.errors.add(attribute, 'should be a duration')
               end
+
+              if options[:limit]
+                unless validate_duration_limit(value, options[:limit])
+                  record.errors.add(attribute, 'should not exceed the limit')
+                end
+              end
             end
           end
 

lib/gitlab/ci/status/build/factory.rb

@@ -5,6 +5,7 @@ module Gitlab
         class Factory < Status::Factory
           def self.extended_statuses
             [[Status::Build::Erased,
+              Status::Build::Scheduled,
               Status::Build::Manual,
               Status::Build::Canceled,
               Status::Build::Created,
@@ -14,6 +15,7 @@ module Gitlab
               Status::Build::Retryable],
              [Status::Build::Failed],
              [Status::Build::FailedAllowed,
+              Status::Build::Unschedule,
               Status::Build::Play,
               Status::Build::Stop],
              [Status::Build::Action],

lib/gitlab/ci/status/build/failed.rb

@@ -10,7 +10,8 @@ module Gitlab
             stuck_or_timeout_failure: 'stuck or timeout failure',
             runner_system_failure: 'runner system failure',
             missing_dependency_failure: 'missing dependency failure',
-            runner_unsupported: 'unsupported runner'
+            runner_unsupported: 'unsupported runner',
+            stale_schedule: 'stale schedule'
           }.freeze
 
           private_constant :REASONS

lib/gitlab/ci/status/build/scheduled.rb

+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Scheduled < Status::Extended
+          def illustration
+            {
+              image: 'illustrations/illustrations_scheduled-job_countdown.svg',
+              size: 'svg-394',
+              title: _("This is a scheduled to run in ") + " #{execute_in}",
+              content: _("This job will automatically run after it's timer finishes. " \
+                         "Often they are used for incremental roll-out deploys " \
+                         "to production environments. When unscheduled it converts " \
+                         "into a manual action.")
+            }
+          end
+
+          def status_tooltip
+            "scheduled manual action (#{execute_in})"
+          end
+
+          def self.matches?(build, user)
+            build.scheduled? && build.scheduled_at
+          end
+
+          private
+
+          include TimeHelper
+
+          def execute_in
+            remaining_seconds = [0, subject.scheduled_at - Time.now].max
+            duration_in_numbers(remaining_seconds, true)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/status/build/unschedule.rb

+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class Unschedule < Status::Extended
+          def label
+            'unschedule action'
+          end
+
+          def has_action?
+            can?(user, :update_build, subject)
+          end
+
+          def action_icon
+            'time-out'
+          end
+
+          def action_title
+            'Unschedule'
+          end
+
+          def action_button_title
+            _('Unschedule job')
+          end
+
+          def action_path
+            unschedule_project_job_path(subject.project, subject)
+          end
+
+          def action_method
+            :post
+          end
+
+          def self.matches?(build, user)
+            build.scheduled?
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/status/pipeline/factory.rb

@@ -5,6 +5,7 @@ module Gitlab
         class Factory < Status::Factory
           def self.extended_statuses
             [[Status::SuccessWarning,
+              Status::Pipeline::Scheduled,
               Status::Pipeline::Blocked]]
           end
 

lib/gitlab/ci/status/pipeline/scheduled.rb

+module Gitlab
+  module Ci
+    module Status
+      module Pipeline
+        class Scheduled < Status::Extended
+          def text
+            s_('CiStatusText|scheduled')
+          end
+
+          def label
+            s_('CiStatusLabel|waiting for delayed job')
+          end
+
+          def self.matches?(pipeline, user)
+            pipeline.scheduled?
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/status/scheduled.rb

+module Gitlab
+  module Ci
+    module Status
+      class Scheduled < Status::Core
+        def text
+          s_('CiStatusText|scheduled')
+        end
+
+        def label
+          s_('CiStatusLabel|scheduled')
+        end
+
+        def icon
+          'status_scheduled'
+        end
+
+        def favicon
+          'favicon_status_scheduled'
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/yaml_processor.rb

@@ -49,7 +49,8 @@ module Gitlab
             script: job[:script],
             after_script: job[:after_script],
             environment: job[:environment],
-            retry: job[:retry]
+            retry: job[:retry],
+            start_in: job[:start_in]
           }.compact }
       end
 

locale/gitlab.pot

@@ -1459,9 +1459,15 @@ msgstr ""
 msgid "CiStatusLabel|pending"
 msgstr ""
 
+msgid "CiStatusLabel|scheduled"
+msgstr ""
+
 msgid "CiStatusLabel|skipped"
 msgstr ""
 
+msgid "CiStatusLabel|waiting for delayed job"
+msgstr ""
+
 msgid "CiStatusLabel|waiting for manual action"
 msgstr ""
 
@@ -1486,6 +1492,9 @@ msgstr ""
 msgid "CiStatusText|pending"
 msgstr ""
 
+msgid "CiStatusText|scheduled"
+msgstr ""
+
 msgid "CiStatusText|skipped"
 msgstr ""
 
@@ -2495,6 +2504,21 @@ msgstr ""
 msgid "Define a custom pattern with cron syntax"
 msgstr ""
 
+msgid "DelayedJobs|Are you sure you want to run %{jobName} immediately? This job will run automatically after it's timer finishes."
+msgstr ""
+
+msgid "DelayedJobs|Are you sure you want to run %{job_name} immediately? This job will run automatically after it's timer finishes."
+msgstr ""
+
+msgid "DelayedJobs|Start now"
+msgstr ""
+
+msgid "DelayedJobs|Unschedule"
+msgstr ""
+
+msgid "DelayedJobs|scheduled"
+msgstr ""
+
 msgid "Delete"
 msgstr ""
 
@@ -7725,6 +7749,9 @@ msgstr ""
 msgid "This is a confidential issue."
 msgstr ""
 
+msgid "This is a scheduled to run in "
+msgstr ""
+
 msgid "This is the author's first Merge Request to this project."
 msgstr ""
 
@@ -7785,6 +7812,9 @@ msgstr ""
 msgid "This job requires a manual action"
 msgstr ""
 
+msgid "This job will automatically run after it's timer finishes. Often they are used for incremental roll-out deploys to production environments. When unscheduled it converts into a manual action."
+msgstr ""
+
 msgid "This means you can not push code until you create an empty repository or import existing one."
 msgstr ""
 
@@ -8200,6 +8230,9 @@ msgstr ""
 msgid "Unresolve discussion"
 msgstr ""
 
+msgid "Unschedule job"
+msgstr ""
+
 msgid "Unstage"
 msgstr ""
 

spec/controllers/projects/jobs_controller_spec.rb

@@ -631,6 +631,46 @@ describe Projects::JobsController, :clean_gitlab_redis_shared_state do
     end
   end
 
+  describe 'POST unschedule' do
+    before do
+      project.add_developer(user)
+
+      create(:protected_branch, :developers_can_merge,
+             name: 'master', project: project)
+
+      sign_in(user)
+
+      post_unschedule
+    end
+
+    context 'when job is scheduled' do
+      let(:job) { create(:ci_build, :scheduled, pipeline: pipeline) }
+
+      it 'redirects to the unscheduled job page' do
+        expect(response).to have_gitlab_http_status(:found)
+        expect(response).to redirect_to(namespace_project_job_path(id: job.id))
+      end
+
+      it 'transits to manual' do
+        expect(job.reload).to be_manual
+      end
+    end
+
+    context 'when job is not scheduled' do
+      let(:job) { create(:ci_build, pipeline: pipeline) }
+
+      it 'renders unprocessable_entity' do
+        expect(response).to have_gitlab_http_status(:unprocessable_entity)
+      end
+    end
+
+    def post_unschedule
+      post :unschedule, namespace_id: project.namespace,
+                        project_id: project,
+                        id: job.id
+    end
+  end
+
   describe 'POST cancel_all' do
     before do
       project.add_developer(user)

spec/factories/ci/builds.rb

@@ -70,6 +70,18 @@ FactoryBot.define do
       status 'created'
     end
 
+    trait :scheduled do
+      schedulable
+      status 'scheduled'
+      scheduled_at  { 1.minute.since }
+    end
+
+    trait :expired_scheduled do
+      schedulable
+      status 'scheduled'
+      scheduled_at { 1.minute.ago }
+    end
+
     trait :manual do
       status 'manual'
       self.when 'manual'
@@ -98,6 +110,15 @@ FactoryBot.define do
       success
     end
 
+    trait :schedulable do
+      self.when 'delayed'
+      options start_in: '1 minute'
+    end
+
+    trait :actionable do
+      self.when 'manual'
+    end
+
     trait :retried do
       retried true
     end

spec/factories/ci/pipelines.rb

@@ -54,6 +54,10 @@ FactoryBot.define do
         status :manual
       end
 
+      trait :scheduled do
+        status :scheduled
+      end
+
       trait :success do
         status :success
       end

spec/factories/commit_statuses.rb

@@ -41,6 +41,10 @@ FactoryBot.define do
       status 'manual'
     end
 
+    trait :scheduled do
+      status 'scheduled'
+    end
+
     after(:build) do |build, evaluator|
       build.project = build.pipeline.project
     end

spec/features/projects/jobs_spec.rb

@@ -559,6 +559,34 @@ describe 'Jobs', :clean_gitlab_redis_shared_state do
       end
     end
 
+    context 'Delayed job' do
+      let(:job) { create(:ci_build, :scheduled, pipeline: pipeline) }
+
+      before do
+        project.add_developer(user)
+        visit project_job_path(project, job)
+      end
+
+      it 'shows delayed job', :js do
+        time_diff = [0, job.scheduled_at - Time.now].max
+
+        expect(page).to have_content(job.detailed_status(user).illustration[:title])
+        expect(page).to have_content('This is a scheduled to run in')
+        expect(page).to have_content("This job will automatically run after it's timer finishes.")
+        expect(page).to have_content(Time.at(time_diff).utc.strftime("%H:%M:%S"))
+        expect(page).to have_link('Unschedule job')
+      end
+
+      it 'unschedules delayed job and shows manual action', :js do
+        click_link 'Unschedule job'
+
+        wait_for_requests
+        expect(page).to have_content('This job requires a manual action')
+        expect(page).to have_content('This job depends on a user to trigger its process. Often they are used to deploy code to production environments')
+        expect(page).to have_link('Trigger this manual action')
+      end
+    end
+
     context 'Non triggered job' do
       let(:job) { create(:ci_build, :created, pipeline: pipeline) }
 

spec/features/projects/pipelines/pipeline_spec.rb

@@ -31,6 +31,11 @@ describe 'Pipeline', :js do
              pipeline: pipeline, stage: 'deploy', name: 'manual-build')
     end
 
+    let!(:build_scheduled) do
+      create(:ci_build, :scheduled,
+             pipeline: pipeline, stage: 'deploy', name: 'delayed-job')
+    end
+
     let!(:build_external) do
       create(:generic_commit_status, status: 'success',
                                      pipeline: pipeline,
@@ -79,10 +84,12 @@ describe 'Pipeline', :js do
           end
         end
 
-        it 'should be possible to cancel the running build' do
+        it 'cancels the running build and shows retry button' do
           find('#ci-badge-deploy .ci-action-icon-container').click
 
-          expect(page).not_to have_content('Cancel running')
+          page.within('#ci-badge-deploy') do
+            expect(page).to have_css('.js-icon-retry')
+          end
         end
       end
 
@@ -105,6 +112,27 @@ describe 'Pipeline', :js do
         end
       end
 
+      context 'when pipeline has a delayed job' do
+        it 'shows the scheduled icon and an unschedule action for the delayed job' do
+          page.within('#ci-badge-delayed-job') do
+            expect(page).to have_selector('.js-ci-status-icon-scheduled')
+            expect(page).to have_content('delayed-job')
+          end
+
+          page.within('#ci-badge-delayed-job .ci-action-icon-container.js-icon-time-out') do
+            expect(page).to have_selector('svg')
+          end
+        end
+
+        it 'unschedules the delayed job and shows play button as a manual job' do
+          find('#ci-badge-delayed-job .ci-action-icon-container').click
+
+          page.within('#ci-badge-delayed-job') do
+            expect(page).to have_css('.js-icon-play')
+          end
+        end
+      end
+
       context 'when pipeline has failed builds' do
         it 'shows the failed icon and a retry action for the failed build' do
           page.within('#ci-badge-test') do
@@ -315,6 +343,18 @@ describe 'Pipeline', :js do
       it { expect(build_manual.reload).to be_pending }
     end
 
+    context 'when user unschedules a delayed job' do
+      before do
+        within '.pipeline-holder' do
+          click_link('Unschedule')
+        end
+      end
+
+      it 'unschedules the delayed job and shows play button as a manual job' do
+        expect(page).to have_content('Trigger this manual action')
+      end
+    end
+
     context 'failed jobs' do
       it 'displays a tooltip with the failure reason' do
         page.within('.ci-table') do

spec/features/projects/pipelines/pipelines_spec.rb

@@ -232,6 +232,60 @@ describe 'Pipelines', :js do
         end
       end
 
+      context 'when there is a delayed job' do
+        let!(:delayed_job) do
+          create(:ci_build, :scheduled,
+            pipeline: pipeline,
+            name: 'delayed job',
+            stage: 'test',
+            commands: 'test')
+        end
+
+        before do
+          visit_project_pipelines
+        end
+
+        it 'has a dropdown for actionable jobs' do
+          expect(page).to have_selector('.dropdown-new.btn.btn-default .icon-play')
+        end
+
+        it "has link to the delayed job's action" do
+          find('.js-pipeline-dropdown-manual-actions').click
+
+          time_diff = [0, delayed_job.scheduled_at - Time.now].max
+          expect(page).to have_button('delayed job')
+          expect(page).to have_content(Time.at(time_diff).utc.strftime("%H:%M:%S"))
+        end
+
+        context 'when delayed job is expired already' do
+          let!(:delayed_job) do
+            create(:ci_build, :expired_scheduled,
+              pipeline: pipeline,
+              name: 'delayed job',
+              stage: 'test',
+              commands: 'test')
+          end
+
+          it "shows 00:00:00 as the remaining time" do
+            find('.js-pipeline-dropdown-manual-actions').click
+
+            expect(page).to have_content("00:00:00")
+          end
+        end
+
+        context 'when user played a delayed job immediately' do
+          before do
+            find('.js-pipeline-dropdown-manual-actions').click
+            page.accept_confirm { click_button('delayed job') }
+            wait_for_requests
+          end
+
+          it 'enqueues the delayed job', :js do
+            expect(delayed_job.reload).to be_pending
+          end
+        end
+      end
+
       context 'for generic statuses' do
         context 'when running' do
           let!(:running) do

spec/helpers/time_helper_spec.rb

@@ -20,17 +20,35 @@ describe TimeHelper do
   end
 
   describe "#duration_in_numbers" do
-    it "returns minutes and seconds" do
-      durations_and_expectations = {
-        100 => "01:40",
-        121 => "02:01",
-        3721 => "01:02:01",
-        0 => "00:00",
-        42 => "00:42"
-      }
+    using RSpec::Parameterized::TableSyntax
+
+    context "without passing allow_overflow" do
+      where(:duration, :formatted_string) do
+        0                              | "00:00"
+        1.second                       | "00:01"
+        42.seconds                     | "00:42"
+        2.minutes + 1.second           | "02:01"
+        3.hours + 2.minutes + 1.second | "03:02:01"
+        30.hours                       | "06:00:00"
+      end
+
+      with_them do
+        it { expect(duration_in_numbers(duration)).to eq formatted_string }
+      end
+    end
+
+    context "with allow_overflow = true" do
+      where(:duration, :formatted_string) do
+        0                              | "00:00:00"
+        1.second                       | "00:00:01"
+        42.seconds                     | "00:00:42"
+        2.minutes + 1.second           | "00:02:01"
+        3.hours + 2.minutes + 1.second | "03:02:01"
+        30.hours                       | "30:00:00"
+      end
 
-      durations_and_expectations.each do |duration, expectation|
-        expect(duration_in_numbers(duration)).to eq(expectation)
+      with_them do
+        it { expect(duration_in_numbers(duration, true)).to eq formatted_string }
       end
     end
   end

spec/javascripts/datetime_utility_spec.js

@@ -6,9 +6,7 @@ describe('Date time utils', () => {
       const date = new Date();
       date.setFullYear(date.getFullYear() - 1);
 
-      expect(
-        datetimeUtility.timeFor(date),
-      ).toBe('Past due');
+      expect(datetimeUtility.timeFor(date)).toBe('Past due');
     });
 
     it('returns remaining time when in the future', () => {
@@ -19,9 +17,7 @@ describe('Date time utils', () => {
       // short of a full year, timeFor will return '11 months remaining'
       date.setDate(date.getDate() + 1);
 
-      expect(
-        datetimeUtility.timeFor(date),
-      ).toBe('1 year remaining');
+      expect(datetimeUtility.timeFor(date)).toBe('1 year remaining');
     });
   });
 
@@ -168,3 +164,20 @@ describe('getTimeframeWindowFrom', () => {
     });
   });
 });
+
+describe('formatTime', () => {
+  const expectedTimestamps = [
+    [0, '00:00:00'],
+    [1000, '00:00:01'],
+    [42000, '00:00:42'],
+    [121000, '00:02:01'],
+    [10921000, '03:02:01'],
+    [108000000, '30:00:00'],
+  ];
+
+  expectedTimestamps.forEach(([milliseconds, expectedTimestamp]) => {
+    it(`formats ${milliseconds}ms as ${expectedTimestamp}`, () => {
+      expect(datetimeUtility.formatTime(milliseconds)).toBe(expectedTimestamp);
+    });
+  });
+});

spec/javascripts/pipelines/pipelines_actions_spec.js

 import Vue from 'vue';
-import pipelinesActionsComp from '~/pipelines/components/pipelines_actions.vue';
+import eventHub from '~/pipelines/event_hub';
+import PipelinesActions from '~/pipelines/components/pipelines_actions.vue';
+import mountComponent from 'spec/helpers/vue_mount_component_helper';
+import { TEST_HOST } from 'spec/test_constants';
 
 describe('Pipelines Actions dropdown', () => {
-  let component;
-  let actions;
-  let ActionsComponent;
+  const Component = Vue.extend(PipelinesActions);
+  let vm;
 
-  beforeEach(() => {
-    ActionsComponent = Vue.extend(pipelinesActionsComp);
+  afterEach(() => {
+    vm.$destroy();
+  });
 
-    actions = [
+  describe('manual actions', () => {
+    const actions = [
       {
         name: 'stop_review',
-        path: '/root/review-app/builds/1893/play',
+        path: `${TEST_HOST}/root/review-app/builds/1893/play`,
       },
       {
         name: 'foo',
-        path: '#',
+        path: `${TEST_HOST}/disabled/pipeline/action`,
         playable: false,
       },
     ];
 
-    component = new ActionsComponent({
-      propsData: {
-        actions,
-      },
-    }).$mount();
-  });
+    beforeEach(() => {
+      vm = mountComponent(Component, { actions });
+    });
 
-  it('should render a dropdown with the provided actions', () => {
-    expect(
-      component.$el.querySelectorAll('.dropdown-menu li').length,
-    ).toEqual(actions.length);
+    it('renders a dropdown with the provided actions', () => {
+      const dropdownItems = vm.$el.querySelectorAll('.dropdown-menu li');
+      expect(dropdownItems.length).toEqual(actions.length);
+    });
+
+    it("renders a disabled action when it's not playable", () => {
+      const dropdownItem = vm.$el.querySelector('.dropdown-menu li:last-child button');
+      expect(dropdownItem).toBeDisabled();
+    });
   });
 
-  it('should render a disabled action when it\'s not playable', () => {
-    expect(
-      component.$el.querySelector('.dropdown-menu li:last-child button').getAttribute('disabled'),
-    ).toEqual('disabled');
+  describe('scheduled jobs', () => {
+    const scheduledJobAction = {
+      name: 'scheduled action',
+      path: `${TEST_HOST}/scheduled/job/action`,
+      playable: true,
+      scheduled_at: '2063-04-05T00:42:00Z',
+    };
+    const expiredJobAction = {
+      name: 'expired action',
+      path: `${TEST_HOST}/expired/job/action`,
+      playable: true,
+      scheduled_at: '2018-10-05T08:23:00Z',
+    };
+    const findDropdownItem = action => {
+      const buttons = vm.$el.querySelectorAll('.dropdown-menu li button');
+      return Array.prototype.find.call(buttons, element =>
+        element.innerText.trim().startsWith(action.name),
+      );
+    };
+
+    beforeEach(() => {
+      spyOn(Date, 'now').and.callFake(() => new Date('2063-04-04T00:42:00Z').getTime());
+      vm = mountComponent(Component, { actions: [scheduledJobAction, expiredJobAction] });
+    });
+
+    it('emits postAction event after confirming', () => {
+      const emitSpy = jasmine.createSpy('emit');
+      eventHub.$on('postAction', emitSpy);
+      spyOn(window, 'confirm').and.callFake(() => true);
+
+      findDropdownItem(scheduledJobAction).click();
+
+      expect(window.confirm).toHaveBeenCalled();
+      expect(emitSpy).toHaveBeenCalledWith(scheduledJobAction.path);
+    });
+
+    it('does not emit postAction event if confirmation is cancelled', () => {
+      const emitSpy = jasmine.createSpy('emit');
+      eventHub.$on('postAction', emitSpy);
+      spyOn(window, 'confirm').and.callFake(() => false);
+
+      findDropdownItem(scheduledJobAction).click();
+
+      expect(window.confirm).toHaveBeenCalled();
+      expect(emitSpy).not.toHaveBeenCalled();
+    });
+
+    it('displays the remaining time in the dropdown', () => {
+      expect(findDropdownItem(scheduledJobAction)).toContainText('24:00:00');
+    });
 
-    expect(
-      component.$el.querySelector('.dropdown-menu li:last-child button').classList.contains('disabled'),
-    ).toEqual(true);
+    it('displays 00:00:00 for expired jobs in the dropdown', () => {
+      expect(findDropdownItem(expiredJobAction)).toContainText('00:00:00');
+    });
   });
 });

spec/javascripts/pipelines/pipelines_table_row_spec.js

@@ -158,8 +158,13 @@ describe('Pipelines Table Row', () => {
   });
 
   describe('actions column', () => {
+    const scheduledJobAction = {
+      name: 'some scheduled job',
+    };
+
     beforeEach(() => {
       const withActions = Object.assign({}, pipeline);
+      withActions.details.scheduled_actions = [scheduledJobAction];
       withActions.flags.cancelable = true;
       withActions.flags.retryable = true;
       withActions.cancel_path = '/cancel';
@@ -171,6 +176,8 @@ describe('Pipelines Table Row', () => {
     it('should render the provided actions', () => {
       expect(component.$el.querySelector('.js-pipelines-retry-button')).not.toBeNull();
       expect(component.$el.querySelector('.js-pipelines-cancel-button')).not.toBeNull();
+      const dropdownMenu = component.$el.querySelectorAll('.dropdown-menu');
+      expect(dropdownMenu).toContainText(scheduledJobAction.name);
     });
 
     it('emits `retryPipeline` event when retry button is clicked and toggles loading', () => {

spec/lib/gitlab/ci/config/entry/job_spec.rb

@@ -39,6 +39,14 @@ describe Gitlab::Ci::Config::Entry::Job do
           expect(entry.errors).to include "job name can't be blank"
         end
       end
+
+      context 'when delayed job' do
+        context 'when start_in is specified' do
+          let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } }
+
+          it { expect(entry).to be_valid }
+        end
+      end
     end
 
     context 'when entry value is not correct' do
@@ -129,6 +137,52 @@ describe Gitlab::Ci::Config::Entry::Job do
           end
         end
       end
+
+      context 'when delayed job' do
+        context 'when start_in is specified' do
+          let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } }
+
+          it 'returns error about invalid type' do
+            expect(entry).to be_valid
+          end
+        end
+
+        context 'when start_in is empty' do
+          let(:config) { { when: 'delayed', start_in: nil } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job start in should be a duration'
+          end
+        end
+
+        context 'when start_in is not formatted as a duration' do
+          let(:config) { { when: 'delayed', start_in: 'test' } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job start in should be a duration'
+          end
+        end
+
+        context 'when start_in is longer than one day' do
+          let(:config) { { when: 'delayed', start_in: '2 days' } }
+
+          it 'returns error about exceeding the limit' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job start in should not exceed the limit'
+          end
+        end
+      end
+
+      context 'when start_in specified without delayed specification' do
+        let(:config) { { start_in: '1 day' } }
+
+        it 'returns error about invalid type' do
+          expect(entry).not_to be_valid
+          expect(entry.errors).to include 'job start in must be blank'
+        end
+      end
     end
   end
 
@@ -238,6 +292,24 @@ describe Gitlab::Ci::Config::Entry::Job do
     end
   end
 
+  describe '#delayed?' do
+    context 'when job is a delayed' do
+      let(:config) { { script: 'deploy', when: 'delayed' } }
+
+      it 'is a delayed' do
+        expect(entry).to be_delayed
+      end
+    end
+
+    context 'when job is not a delayed' do
+      let(:config) { { script: 'deploy' } }
+
+      it 'is not a delayed' do
+        expect(entry).not_to be_delayed
+      end
+    end
+  end
+
   describe '#ignored?' do
     context 'when job is a manual action' do
       context 'when it is not specified if job is allowed to fail' do

spec/lib/gitlab/ci/status/build/factory_spec.rb

@@ -319,4 +319,53 @@ describe Gitlab::Ci::Status::Build::Factory do
       end
     end
   end
+
+  context 'when build is a delayed action' do
+    let(:build) { create(:ci_build, :scheduled) }
+
+    it 'matches correct core status' do
+      expect(factory.core_status).to be_a Gitlab::Ci::Status::Scheduled
+    end
+
+    it 'matches correct extended statuses' do
+      expect(factory.extended_statuses)
+        .to eq [Gitlab::Ci::Status::Build::Scheduled,
+                Gitlab::Ci::Status::Build::Unschedule,
+                Gitlab::Ci::Status::Build::Action]
+    end
+
+    it 'fabricates action detailed status' do
+      expect(status).to be_a Gitlab::Ci::Status::Build::Action
+    end
+
+    it 'fabricates status with correct details' do
+      expect(status.text).to eq 'scheduled'
+      expect(status.group).to eq 'scheduled'
+      expect(status.icon).to eq 'status_scheduled'
+      expect(status.favicon).to eq 'favicon_status_scheduled'
+      expect(status.illustration).to include(:image, :size, :title, :content)
+      expect(status.label).to include 'unschedule action'
+      expect(status).to have_details
+      expect(status.action_path).to include 'unschedule'
+    end
+
+    context 'when user has ability to play action' do
+      it 'fabricates status that has action' do
+        expect(status).to have_action
+      end
+    end
+
+    context 'when user does not have ability to play action' do
+      before do
+        allow(build.project).to receive(:empty_repo?).and_return(false)
+
+        create(:protected_branch, :no_one_can_push,
+                name: build.ref, project: build.project)
+      end
+
+      it 'fabricates status that has no action' do
+        expect(status).not_to have_action
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/status/build/scheduled_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Build::Scheduled do
+  let(:user) { create(:user) }
+  let(:project) { create(:project, :stubbed_repository) }
+  let(:build) { create(:ci_build, :scheduled, project: project) }
+  let(:status) { Gitlab::Ci::Status::Core.new(build, user) }
+
+  subject { described_class.new(status) }
+
+  describe '#illustration' do
+    it { expect(subject.illustration).to include(:image, :size, :title) }
+  end
+
+  describe '#status_tooltip' do
+    context 'when scheduled_at is not expired' do
+      let(:build) { create(:ci_build, scheduled_at: 1.minute.since, project: project) }
+
+      it 'shows execute_in of the scheduled job' do
+        Timecop.freeze do
+          expect(subject.status_tooltip).to include('00:01:00')
+        end
+      end
+    end
+
+    context 'when scheduled_at is expired' do
+      let(:build) { create(:ci_build, :expired_scheduled, project: project) }
+
+      it 'shows 00:00:00' do
+        Timecop.freeze do
+          expect(subject.status_tooltip).to include('00:00:00')
+        end
+      end
+    end
+  end
+
+  describe '.matches?' do
+    subject { described_class.matches?(build, user) }
+
+    context 'when build is scheduled and scheduled_at is present' do
+      let(:build) { create(:ci_build, :expired_scheduled, project: project) }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when build is scheduled' do
+      let(:build) { create(:ci_build, status: :scheduled, project: project) }
+
+      it { is_expected.to be_falsy }
+    end
+
+    context 'when scheduled_at is present' do
+      let(:build) { create(:ci_build, scheduled_at: 1.minute.since, project: project) }
+
+      it { is_expected.to be_falsy }
+    end
+  end
+end

spec/lib/gitlab/ci/status/build/unschedule_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Build::Unschedule do
+  let(:status) { double('core status') }
+  let(:user) { double('user') }
+
+  subject do
+    described_class.new(status)
+  end
+
+  describe '#label' do
+    it { expect(subject.label).to eq 'unschedule action' }
+  end
+
+  describe 'action details' do
+    let(:user) { create(:user) }
+    let(:build) { create(:ci_build) }
+    let(:status) { Gitlab::Ci::Status::Core.new(build, user) }
+
+    describe '#has_action?' do
+      context 'when user is allowed to update build' do
+        before do
+          stub_not_protect_default_branch
+
+          build.project.add_developer(user)
+        end
+
+        it { is_expected.to have_action }
+      end
+
+      context 'when user is not allowed to update build' do
+        it { is_expected.not_to have_action }
+      end
+    end
+
+    describe '#action_path' do
+      it { expect(subject.action_path).to include "#{build.id}/unschedule" }
+    end
+
+    describe '#action_icon' do
+      it { expect(subject.action_icon).to eq 'time-out' }
+    end
+
+    describe '#action_title' do
+      it { expect(subject.action_title).to eq 'Unschedule' }
+    end
+
+    describe '#action_button_title' do
+      it { expect(subject.action_button_title).to eq 'Unschedule job' }
+    end
+  end
+
+  describe '.matches?' do
+    subject { described_class.matches?(build, user) }
+
+    context 'when build is scheduled' do
+      context 'when build unschedules an delayed job' do
+        let(:build) { create(:ci_build, :scheduled) }
+
+        it 'is a correct match' do
+          expect(subject).to be true
+        end
+      end
+
+      context 'when build unschedules an normal job' do
+        let(:build) { create(:ci_build) }
+
+        it 'does not match' do
+          expect(subject).to be false
+        end
+      end
+    end
+  end
+
+  describe '#status_tooltip' do
+    it 'does not override status status_tooltip' do
+      expect(status).to receive(:status_tooltip)
+
+      subject.status_tooltip
+    end
+  end
+
+  describe '#badge_tooltip' do
+    let(:user) { create(:user) }
+    let(:build) { create(:ci_build, :playable) }
+    let(:status) { Gitlab::Ci::Status::Core.new(build, user) }
+
+    it 'does not override status badge_tooltip' do
+      expect(status).to receive(:badge_tooltip)
+
+      subject.badge_tooltip
+    end
+  end
+end

spec/lib/gitlab/ci/status/pipeline/factory_spec.rb

@@ -11,8 +11,7 @@ describe Gitlab::Ci::Status::Pipeline::Factory do
   end
 
   context 'when pipeline has a core status' do
-    (HasStatus::AVAILABLE_STATUSES - [HasStatus::BLOCKED_STATUS])
-      .each do |simple_status|
+    HasStatus::AVAILABLE_STATUSES.each do |simple_status|
       context "when core status is #{simple_status}" do
         let(:pipeline) { create(:ci_pipeline, status: simple_status) }
 
@@ -24,12 +23,24 @@ describe Gitlab::Ci::Status::Pipeline::Factory do
           expect(factory.core_status).to be_a expected_status
         end
 
-        it 'does not match extended statuses' do
-          expect(factory.extended_statuses).to be_empty
-        end
-
-        it "fabricates a core status #{simple_status}" do
-          expect(status).to be_a expected_status
+        if simple_status == 'manual'
+          it 'matches a correct extended statuses' do
+            expect(factory.extended_statuses)
+              .to eq [Gitlab::Ci::Status::Pipeline::Blocked]
+          end
+        elsif simple_status == 'scheduled'
+          it 'matches a correct extended statuses' do
+            expect(factory.extended_statuses)
+              .to eq [Gitlab::Ci::Status::Pipeline::Scheduled]
+          end
+        else
+          it 'does not match extended statuses' do
+            expect(factory.extended_statuses).to be_empty
+          end
+
+          it "fabricates a core status #{simple_status}" do
+            expect(status).to be_a expected_status
+          end
         end
 
         it 'extends core status with common pipeline methods' do
@@ -40,27 +51,6 @@ describe Gitlab::Ci::Status::Pipeline::Factory do
         end
       end
     end
-
-    context "when core status is manual" do
-      let(:pipeline) { create(:ci_pipeline, status: :manual) }
-
-      it "matches manual core status" do
-        expect(factory.core_status)
-          .to be_a Gitlab::Ci::Status::Manual
-      end
-
-      it 'matches a correct extended statuses' do
-        expect(factory.extended_statuses)
-          .to eq [Gitlab::Ci::Status::Pipeline::Blocked]
-      end
-
-      it 'extends core status with common pipeline methods' do
-        expect(status).to have_details
-        expect(status).not_to have_action
-        expect(status.details_path)
-          .to include "pipelines/#{pipeline.id}"
-      end
-    end
   end
 
   context 'when pipeline has warnings' do

spec/lib/gitlab/ci/status/pipeline/scheduled_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Pipeline::Scheduled do
+  let(:pipeline) { double('pipeline') }
+
+  subject do
+    described_class.new(pipeline)
+  end
+
+  describe '#text' do
+    it 'overrides status text' do
+      expect(subject.text).to eq 'scheduled'
+    end
+  end
+
+  describe '#label' do
+    it 'overrides status label' do
+      expect(subject.label).to eq 'waiting for delayed job'
+    end
+  end
+
+  describe '.matches?' do
+    let(:user) { double('user') }
+    subject { described_class.matches?(pipeline, user) }
+
+    context 'when pipeline is scheduled' do
+      let(:pipeline) { create(:ci_pipeline, :scheduled) }
+
+      it 'is a correct match' do
+        expect(subject).to be true
+      end
+    end
+
+    context 'when pipeline is not scheduled' do
+      let(:pipeline) { create(:ci_pipeline, :success) }
+
+      it 'does not match' do
+        expect(subject).to be false
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/status/scheduled_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Scheduled do
+  subject do
+    described_class.new(double('subject'), double('user'))
+  end
+
+  describe '#text' do
+    it { expect(subject.text).to eq 'scheduled' }
+  end
+
+  describe '#label' do
+    it { expect(subject.label).to eq 'scheduled' }
+  end
+
+  describe '#icon' do
+    it { expect(subject.icon).to eq 'status_scheduled' }
+  end
+
+  describe '#favicon' do
+    it { expect(subject.favicon).to eq 'favicon_status_scheduled' }
+  end
+
+  describe '#group' do
+    it { expect(subject.group).to eq 'scheduled' }
+  end
+end

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -121,6 +121,21 @@ module Gitlab
             end
           end
         end
+
+        describe 'delayed job entry' do
+          context 'when delayed is defined' do
+            let(:config) do
+              YAML.dump(rspec: { script: 'rollout 10%',
+                                 when: 'delayed',
+                                 start_in: '1 day' })
+            end
+
+            it 'has the attributes' do
+              expect(subject[:when]).to eq 'delayed'
+              expect(subject[:options][:start_in]).to eq '1 day'
+            end
+          end
+        end
       end
 
       describe '#stages_attributes' do
@@ -1260,7 +1275,7 @@ module Gitlab
           config = YAML.dump({ rspec: { script: "test", when: 1 } })
           expect do
             Gitlab::Ci::YamlProcessor.new(config)
-          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec when should be on_success, on_failure, always or manual")
+          end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec when should be on_success, on_failure, always, manual or delayed")
         end
 
         it "returns errors if job artifacts:name is not an a string" do

spec/lib/gitlab/favicon_spec.rb

@@ -58,6 +58,7 @@ RSpec.describe Gitlab::Favicon, :request_store do
         favicon_status_not_found
         favicon_status_pending
         favicon_status_running
+        favicon_status_scheduled
         favicon_status_skipped
         favicon_status_success
         favicon_status_warning

spec/lib/gitlab/import_export/all_models.yml

@@ -117,6 +117,7 @@ pipelines:
 - retryable_builds
 - cancelable_statuses
 - manual_actions
+- scheduled_actions
 - artifacts
 - pipeline_schedule
 - merge_requests

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -303,6 +303,7 @@ CommitStatus:
 - retried
 - protected
 - failure_reason
+- scheduled_at
 Ci::Variable:
 - id
 - project_id

spec/models/ci/build_spec.rb

@@ -210,6 +210,155 @@ describe Ci::Build do
     end
   end
 
+  describe '#schedulable?' do
+    subject { build.schedulable? }
+
+    context 'when build is schedulable' do
+      let(:build) { create(:ci_build, :created, :schedulable, project: project) }
+
+      it { expect(subject).to be_truthy }
+
+      context 'when feature flag is diabled' do
+        before do
+          stub_feature_flags(ci_enable_scheduled_build: false)
+        end
+
+        it { expect(subject).to be_falsy }
+      end
+    end
+
+    context 'when build is not schedulable' do
+      let(:build) { create(:ci_build, :created, project: project) }
+
+      it { expect(subject).to be_falsy }
+    end
+  end
+
+  describe '#schedule' do
+    subject { build.schedule }
+
+    before do
+      project.add_developer(user)
+    end
+
+    let(:build) { create(:ci_build, :created, :schedulable, user: user, project: project) }
+
+    it 'transits to scheduled' do
+      allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+
+      subject
+
+      expect(build).to be_scheduled
+    end
+
+    it 'updates scheduled_at column' do
+      allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+
+      subject
+
+      expect(build.scheduled_at).not_to be_nil
+    end
+
+    it 'schedules BuildScheduleWorker at the right time' do
+      Timecop.freeze do
+        expect(Ci::BuildScheduleWorker)
+          .to receive(:perform_at).with(1.minute.since, build.id)
+
+        subject
+      end
+    end
+  end
+
+  describe '#unschedule' do
+    subject { build.unschedule }
+
+    context 'when build is scheduled' do
+      let(:build) { create(:ci_build, :scheduled, pipeline: pipeline) }
+
+      it 'cleans scheduled_at column' do
+        subject
+
+        expect(build.scheduled_at).to be_nil
+      end
+
+      it 'transits to manual' do
+        subject
+
+        expect(build).to be_manual
+      end
+    end
+
+    context 'when build is not scheduled' do
+      let(:build) { create(:ci_build, :created, pipeline: pipeline) }
+
+      it 'does not transit status' do
+        subject
+
+        expect(build).to be_created
+      end
+    end
+  end
+
+  describe '#options_scheduled_at' do
+    subject { build.options_scheduled_at }
+
+    let(:build) { build_stubbed(:ci_build, options: option) }
+
+    context 'when start_in is 1 day' do
+      let(:option) { { start_in: '1 day' } }
+
+      it 'returns date after 1 day' do
+        Timecop.freeze do
+          is_expected.to eq(1.day.since)
+        end
+      end
+    end
+
+    context 'when start_in is 1 week' do
+      let(:option) { { start_in: '1 week' } }
+
+      it 'returns date after 1 week' do
+        Timecop.freeze do
+          is_expected.to eq(1.week.since)
+        end
+      end
+    end
+  end
+
+  describe '#enqueue_scheduled' do
+    subject { build.enqueue_scheduled }
+
+    before do
+      stub_feature_flags(ci_enable_scheduled_build: true)
+    end
+
+    context 'when build is scheduled and the right time has not come yet' do
+      let(:build) { create(:ci_build, :scheduled, pipeline: pipeline) }
+
+      it 'does not transits the status' do
+        subject
+
+        expect(build).to be_scheduled
+      end
+    end
+
+    context 'when build is scheduled and the right time has already come' do
+      let(:build) { create(:ci_build, :expired_scheduled, pipeline: pipeline) }
+
+      it 'cleans scheduled_at column' do
+        subject
+
+        expect(build.scheduled_at).to be_nil
+      end
+
+      it 'transits to pending' do
+        subject
+
+        expect(build).to be_pending
+      end
+    end
+  end
+
   describe '#any_runners_online?' do
     subject { build.any_runners_online? }
 
@@ -1194,6 +1343,12 @@ describe Ci::Build do
         it { is_expected.to be_truthy }
       end
 
+      context 'when is set to delayed' do
+        let(:value) { 'delayed' }
+
+        it { is_expected.to be_truthy }
+      end
+
       context 'when set to something else' do
         let(:value) { 'something else' }
 
@@ -1477,6 +1632,12 @@ describe Ci::Build do
       end
     end
 
+    context 'when build is scheduled' do
+      subject { build_stubbed(:ci_build, :scheduled) }
+
+      it { is_expected.to be_playable }
+    end
+
     context 'when build is not a manual action' do
       subject { build_stubbed(:ci_build, :success) }
 

spec/models/ci/pipeline_spec.rb

@@ -79,6 +79,18 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '#delay' do
+    subject { pipeline.delay }
+
+    let(:pipeline) { build(:ci_pipeline, status: :created) }
+
+    it 'changes pipeline status to schedule' do
+      subject
+
+      expect(pipeline).to be_scheduled
+    end
+  end
+
   describe '#valid_commit_sha' do
     context 'commit.sha can not start with 00000000' do
       before do
@@ -1297,6 +1309,19 @@ describe Ci::Pipeline, :mailer do
       end
     end
 
+    context 'when updating status to scheduled' do
+      before do
+        allow(pipeline)
+          .to receive_message_chain(:statuses, :latest, :status)
+          .and_return(:scheduled)
+      end
+
+      it 'updates pipeline status to scheduled' do
+        expect { pipeline.update_status }
+          .to change { pipeline.reload.status }.to 'scheduled'
+      end
+    end
+
     context 'when statuses status was not recognized' do
       before do
         allow(pipeline)

spec/models/ci/stage_spec.rb

@@ -89,6 +89,18 @@ describe Ci::Stage, :models do
       end
     end
 
+    context 'when stage is scheduled because of scheduled builds' do
+      before do
+        create(:ci_build, :scheduled, stage_id: stage.id)
+      end
+
+      it 'updates status to scheduled' do
+        expect { stage.update_status }
+          .to change { stage.reload.status }
+          .to 'scheduled'
+      end
+    end
+
     context 'when stage is skipped because is empty' do
       it 'updates status to skipped' do
         expect { stage.update_status }
@@ -188,6 +200,18 @@ describe Ci::Stage, :models do
     end
   end
 
+  describe '#delay' do
+    subject { stage.delay }
+
+    let(:stage) { create(:ci_stage_entity, status: :created) }
+
+    it 'updates stage status' do
+      subject
+
+      expect(stage).to be_scheduled
+    end
+  end
+
   describe '#position' do
     context 'when stage has been imported and does not have position index set' do
       before do

spec/models/commit_status_spec.rb

@@ -129,6 +129,20 @@ describe CommitStatus do
     end
   end
 
+  describe '#cancel' do
+    subject { job.cancel }
+
+    context 'when status is scheduled' do
+      let(:job) { build(:commit_status, :scheduled) }
+
+      it 'updates the status' do
+        subject
+
+        expect(job).to be_canceled
+      end
+    end
+  end
+
   describe '#auto_canceled?' do
     subject { commit_status.auto_canceled? }
 
@@ -564,6 +578,12 @@ describe CommitStatus do
 
       it_behaves_like 'commit status enqueued'
     end
+
+    context 'when initial state is :scheduled' do
+      let(:commit_status) { create(:commit_status, :scheduled) }
+
+      it_behaves_like 'commit status enqueued'
+    end
   end
 
   describe '#present' do

spec/models/concerns/has_status_spec.rb

@@ -270,11 +270,11 @@ describe HasStatus do
     describe '.cancelable' do
       subject { CommitStatus.cancelable }
 
-      %i[running pending created].each do |status|
+      %i[running pending created scheduled].each do |status|
         it_behaves_like 'containing the job', status
       end
 
-      %i[failed success skipped canceled].each do |status|
+      %i[failed success skipped canceled manual].each do |status|
         it_behaves_like 'not containing the job', status
       end
     end
@@ -290,6 +290,18 @@ describe HasStatus do
         it_behaves_like 'not containing the job', status
       end
     end
+
+    describe '.scheduled' do
+      subject { CommitStatus.scheduled }
+
+      %i[scheduled].each do |status|
+        it_behaves_like 'containing the job', status
+      end
+
+      %i[failed success skipped canceled].each do |status|
+        it_behaves_like 'not containing the job', status
+      end
+    end
   end
 
   describe '::DEFAULT_STATUS' do
@@ -300,7 +312,41 @@ describe HasStatus do
 
   describe '::BLOCKED_STATUS' do
     it 'is a status manual' do
-      expect(described_class::BLOCKED_STATUS).to eq 'manual'
+      expect(described_class::BLOCKED_STATUS).to eq %w[manual scheduled]
+    end
+  end
+
+  describe 'blocked?' do
+    subject { object.blocked? }
+
+    %w[ci_pipeline ci_stage ci_build generic_commit_status].each do |type|
+      let(:object) { build(type, status: status) }
+
+      context 'when status is scheduled' do
+        let(:status) { :scheduled }
+
+        it { is_expected.to be_truthy }
+      end
+
+      context 'when status is manual' do
+        let(:status) { :manual }
+
+        it { is_expected.to be_truthy }
+      end
+
+      context 'when status is created' do
+        let(:status) { :created }
+
+        it { is_expected.to be_falsy }
+      end
+    end
+  end
+
+  describe '.status_sql' do
+    subject { Ci::Build.status_sql }
+
+    it 'returns SQL' do
+      puts subject
     end
   end
 end

spec/presenters/ci/build_presenter_spec.rb

@@ -218,6 +218,42 @@ describe Ci::BuildPresenter do
     end
   end
 
+  describe '#execute_in' do
+    subject { presenter.execute_in }
+
+    context 'when build is scheduled' do
+      context 'when schedule is not expired' do
+        let(:build) { create(:ci_build, :scheduled) }
+
+        it 'returns execution time' do
+          Timecop.freeze do
+            is_expected.to eq(60.0)
+          end
+        end
+      end
+
+      context 'when schedule is expired' do
+        let(:build) { create(:ci_build, :expired_scheduled) }
+
+        it 'returns execution time' do
+          Timecop.freeze do
+            is_expected.to eq(0)
+          end
+        end
+      end
+    end
+
+    context 'when build is not delayed' do
+      let(:build) { create(:ci_build) }
+
+      it 'does not return execution time' do
+        Timecop.freeze do
+          is_expected.to be_falsy
+        end
+      end
+    end
+  end
+
   describe '#callout_failure_message' do
     let(:build) { create(:ci_build, :failed, :api_failure) }
 

spec/serializers/build_action_entity_spec.rb

@@ -22,5 +22,17 @@ describe BuildActionEntity do
     it 'contains whether it is playable' do
       expect(subject[:playable]).to eq job.playable?
     end
+
+    context 'when job is scheduled' do
+      let(:job) { create(:ci_build, :scheduled) }
+
+      it 'returns scheduled_at' do
+        expect(subject[:scheduled_at]).to eq(job.scheduled_at)
+      end
+
+      it 'returns unschedule path' do
+        expect(subject[:unschedule_path]).to include "jobs/#{job.id}/unschedule"
+      end
+    end
   end
 end

spec/serializers/job_entity_spec.rb

@@ -109,6 +109,18 @@ describe JobEntity do
     end
   end
 
+  context 'when job is scheduled' do
+    let(:job) { create(:ci_build, :scheduled) }
+
+    it 'contains path to unschedule action' do
+      expect(subject).to include(:unschedule_path)
+    end
+
+    it 'contains scheduled_at' do
+      expect(subject[:scheduled_at]).to eq(job.scheduled_at)
+    end
+  end
+
   context 'when job is generic commit status' do
     let(:job) { create(:generic_commit_status, target_url: 'http://google.com') }
 

spec/serializers/pipeline_details_entity_spec.rb

@@ -29,7 +29,7 @@ describe PipelineDetailsEntity do
         expect(subject[:details])
           .to include :duration, :finished_at
         expect(subject[:details])
-          .to include :stages, :artifacts, :manual_actions
+          .to include :stages, :artifacts, :manual_actions, :scheduled_actions
         expect(subject[:details][:status]).to include :icon, :favicon, :text, :label
       end
 

spec/serializers/pipeline_serializer_spec.rb

@@ -125,7 +125,7 @@ describe PipelineSerializer do
         it 'verifies number of queries', :request_store do
           recorded = ActiveRecord::QueryRecorder.new { subject }
 
-          expect(recorded.count).to be_within(2).of(35)
+          expect(recorded.count).to be_within(2).of(38)
           expect(recorded.cached_count).to eq(0)
         end
       end

spec/services/ci/enqueue_build_service_spec.rb

-# frozen_string_literal: true
-require 'spec_helper'
-
-describe Ci::EnqueueBuildService, '#execute' do
-  let(:user) { create(:user) }
-  let(:project) { create(:project) }
-  let(:ci_build) { create(:ci_build, :created) }
-
-  subject { described_class.new(project, user).execute(ci_build) }
-
-  it 'enqueues the build' do
-    subject
-
-    expect(ci_build.pending?).to be_truthy
-  end
-end

spec/services/ci/process_build_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+describe Ci::ProcessBuildService, '#execute' do
+  let(:user) { create(:user) }
+  let(:project) { create(:project) }
+
+  subject { described_class.new(project, user).execute(build, current_status) }
+
+  before do
+    project.add_maintainer(user)
+  end
+
+  shared_examples_for 'Enqueuing properly' do |valid_statuses_for_when|
+    valid_statuses_for_when.each do |status_for_prior_stages|
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        let(:current_status) { status_for_prior_stages }
+
+        %w[created skipped manual scheduled].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) }
+
+            it 'enqueues the build' do
+              expect { subject }.to change { build.status }.to('pending')
+            end
+          end
+        end
+
+        %w[pending running success failed canceled].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) }
+
+            it 'does not change the build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+
+    (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages|
+      let(:current_status) { status_for_prior_stages }
+
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        %w[created pending].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) }
+
+            it 'skips the build' do
+              expect { subject }.to change { build.status }.to('skipped')
+            end
+          end
+        end
+
+        (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, when: when_option, user: user, project: project) }
+
+            it 'does not change build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+  end
+
+  shared_examples_for 'Actionizing properly' do |valid_statuses_for_when|
+    valid_statuses_for_when.each do |status_for_prior_stages|
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        let(:current_status) { status_for_prior_stages }
+
+        %w[created].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) }
+
+            it 'enqueues the build' do
+              expect { subject }.to change { build.status }.to('manual')
+            end
+          end
+        end
+
+        %w[manual skipped pending running success failed canceled scheduled].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) }
+
+            it 'does not change the build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+
+    (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages|
+      let(:current_status) { status_for_prior_stages }
+
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        %w[created pending].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) }
+
+            it 'skips the build' do
+              expect { subject }.to change { build.status }.to('skipped')
+            end
+          end
+        end
+
+        (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :actionable, user: user, project: project) }
+
+            it 'does not change build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+  end
+
+  shared_examples_for 'Scheduling properly' do |valid_statuses_for_when|
+    valid_statuses_for_when.each do |status_for_prior_stages|
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        let(:current_status) { status_for_prior_stages }
+
+        %w[created].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) }
+
+            it 'enqueues the build' do
+              expect { subject }.to change { build.status }.to('scheduled')
+            end
+          end
+        end
+
+        %w[manual skipped pending running success failed canceled scheduled].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) }
+
+            it 'does not change the build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+
+    (HasStatus::AVAILABLE_STATUSES - valid_statuses_for_when).each do |status_for_prior_stages|
+      let(:current_status) { status_for_prior_stages }
+
+      context "when status for prior stages is #{status_for_prior_stages}" do
+        %w[created pending].each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) }
+
+            it 'skips the build' do
+              expect { subject }.to change { build.status }.to('skipped')
+            end
+          end
+        end
+
+        (HasStatus::AVAILABLE_STATUSES - %w[created pending]).each do |status|
+          context "when build status is #{status}" do
+            let(:build) { create(:ci_build, status.to_sym, :schedulable, user: user, project: project) }
+
+            it 'does not change build status' do
+              expect { subject }.not_to change { build.status }
+            end
+          end
+        end
+      end
+    end
+  end
+
+  context 'when build has on_success option' do
+    let(:when_option) { :on_success }
+
+    it_behaves_like 'Enqueuing properly', %w[success skipped]
+  end
+
+  context 'when build has on_failure option' do
+    let(:when_option) { :on_failure }
+
+    it_behaves_like 'Enqueuing properly', %w[failed]
+  end
+
+  context 'when build has always option' do
+    let(:when_option) { :always }
+
+    it_behaves_like 'Enqueuing properly', %w[success failed skipped]
+  end
+
+  context 'when build has manual option' do
+    let(:when_option) { :manual }
+
+    it_behaves_like 'Actionizing properly', %w[success skipped]
+  end
+
+  context 'when build has delayed option' do
+    let(:when_option) { :delayed }
+
+    before do
+      allow(Ci::BuildScheduleWorker).to receive(:perform_at) { }
+    end
+
+    context 'when ci_enable_scheduled_build is enabled' do
+      before do
+        stub_feature_flags(ci_enable_scheduled_build: true)
+      end
+
+      it_behaves_like 'Scheduling properly', %w[success skipped]
+    end
+
+    context 'when ci_enable_scheduled_build is enabled' do
+      before do
+        stub_feature_flags(ci_enable_scheduled_build: false)
+      end
+
+      it_behaves_like 'Actionizing properly', %w[success skipped]
+    end
+  end
+end

spec/services/ci/process_pipeline_service_spec.rb

@@ -31,17 +31,14 @@ describe Ci::ProcessPipelineService, '#execute' do
       succeed_pending
 
       expect(builds.success.count).to eq(2)
-      expect(process_pipeline).to be_truthy
 
       succeed_pending
 
       expect(builds.success.count).to eq(4)
-      expect(process_pipeline).to be_truthy
 
       succeed_pending
 
       expect(builds.success.count).to eq(5)
-      expect(process_pipeline).to be_falsey
     end
 
     it 'does not process pipeline if existing stage is running' do
@@ -242,6 +239,187 @@ describe Ci::ProcessPipelineService, '#execute' do
     end
   end
 
+  context 'when delayed jobs are defined' do
+    context 'when the scene is timed incremental rollout' do
+      before do
+        create_build('build', stage_idx: 0)
+        create_build('rollout10%', **delayed_options, stage_idx: 1)
+        create_build('rollout100%', **delayed_options, stage_idx: 2)
+        create_build('cleanup', stage_idx: 3)
+
+        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+      end
+
+      context 'when builds are successful' do
+        it 'properly processes the pipeline' do
+          expect(process_pipeline).to be_truthy
+          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })
+
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })
+
+          enqueue_scheduled('rollout10%')
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' })
+
+          enqueue_scheduled('rollout100%')
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'pending' })
+
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'success', 'cleanup': 'success' })
+          expect(pipeline.reload.status).to eq 'success'
+        end
+      end
+
+      context 'when build job fails' do
+        it 'properly processes the pipeline' do
+          expect(process_pipeline).to be_truthy
+          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })
+
+          fail_running_or_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'failed' })
+          expect(pipeline.reload.status).to eq 'failed'
+        end
+      end
+
+      context 'when rollout 10% is unscheduled' do
+        it 'properly processes the pipeline' do
+          expect(process_pipeline).to be_truthy
+          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })
+
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })
+
+          unschedule
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'manual' })
+          expect(pipeline.reload.status).to eq 'manual'
+        end
+
+        context 'when user plays rollout 10%' do
+          it 'schedules rollout100%' do
+            process_pipeline
+            succeed_pending
+            unschedule
+            play_manual_action('rollout10%')
+            succeed_pending
+
+            expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'success', 'rollout100%': 'scheduled' })
+            expect(pipeline.reload.status).to eq 'scheduled'
+          end
+        end
+      end
+
+      context 'when rollout 10% fails' do
+        it 'properly processes the pipeline' do
+          expect(process_pipeline).to be_truthy
+          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })
+
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })
+
+          enqueue_scheduled('rollout10%')
+          fail_running_or_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'failed' })
+          expect(pipeline.reload.status).to eq 'failed'
+        end
+
+        context 'when user retries rollout 10%' do
+          it 'does not schedule rollout10% again' do
+            process_pipeline
+            succeed_pending
+            enqueue_scheduled('rollout10%')
+            fail_running_or_pending
+            retry_build('rollout10%')
+
+            expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' })
+            expect(pipeline.reload.status).to eq 'running'
+          end
+        end
+      end
+
+      context 'when rollout 10% is played immidiately' do
+        it 'properly processes the pipeline' do
+          expect(process_pipeline).to be_truthy
+          expect(builds_names_and_statuses).to eq({ 'build': 'pending' })
+
+          succeed_pending
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'scheduled' })
+
+          play_manual_action('rollout10%')
+
+          expect(builds_names_and_statuses).to eq({ 'build': 'success', 'rollout10%': 'pending' })
+          expect(pipeline.reload.status).to eq 'running'
+        end
+      end
+    end
+
+    context 'when only one scheduled job exists in a pipeline' do
+      before do
+        create_build('delayed', **delayed_options, stage_idx: 0)
+
+        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+      end
+
+      it 'properly processes the pipeline' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' })
+
+        expect(pipeline.reload.status).to eq 'scheduled'
+      end
+    end
+
+    context 'when there are two delayed jobs in a stage' do
+      before do
+        create_build('delayed1', **delayed_options, stage_idx: 0)
+        create_build('delayed2', **delayed_options, stage_idx: 0)
+        create_build('job', stage_idx: 1)
+
+        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+      end
+
+      it 'blocks the stage until all scheduled jobs finished' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names_and_statuses).to eq({ 'delayed1': 'scheduled', 'delayed2': 'scheduled' })
+
+        enqueue_scheduled('delayed1')
+
+        expect(builds_names_and_statuses).to eq({ 'delayed1': 'pending', 'delayed2': 'scheduled' })
+        expect(pipeline.reload.status).to eq 'running'
+      end
+    end
+
+    context 'when a delayed job is allowed to fail' do
+      before do
+        create_build('delayed', **delayed_options, allow_failure: true, stage_idx: 0)
+        create_build('job', stage_idx: 1)
+
+        allow(Ci::BuildScheduleWorker).to receive(:perform_at)
+      end
+
+      it 'blocks the stage and continues after it failed' do
+        expect(process_pipeline).to be_truthy
+        expect(builds_names_and_statuses).to eq({ 'delayed': 'scheduled' })
+
+        enqueue_scheduled('delayed')
+        fail_running_or_pending
+
+        expect(builds_names_and_statuses).to eq({ 'delayed': 'failed', 'job': 'pending' })
+        expect(pipeline.reload.status).to eq 'pending'
+      end
+    end
+  end
+
   context 'when there are manual action in earlier stages' do
     context 'when first stage has only optional manual actions' do
       before do
@@ -536,6 +714,13 @@ describe Ci::ProcessPipelineService, '#execute' do
     builds.pluck(:name)
   end
 
+  def builds_names_and_statuses
+    builds.each_with_object({}) do |b, h|
+      h[b.name.to_sym] = b.status
+      h
+    end
+  end
+
   def all_builds_names
     all_builds.pluck(:name)
   end
@@ -549,7 +734,7 @@ describe Ci::ProcessPipelineService, '#execute' do
   end
 
   def succeed_pending
-    builds.pending.update_all(status: 'success')
+    builds.pending.map(&:success)
   end
 
   def succeed_running_or_pending
@@ -568,6 +753,14 @@ describe Ci::ProcessPipelineService, '#execute' do
     builds.find_by(name: name).play(user)
   end
 
+  def enqueue_scheduled(name)
+    builds.scheduled.find_by(name: name).enqueue
+  end
+
+  def retry_build(name)
+    Ci::Build.retry(builds.find_by(name: name), user)
+  end
+
   def manual_actions
     pipeline.manual_actions(true)
   end
@@ -575,4 +768,12 @@ describe Ci::ProcessPipelineService, '#execute' do
   def create_build(name, **opts)
     create(:ci_build, :created, pipeline: pipeline, name: name, **opts)
   end
+
+  def delayed_options
+    { when: 'delayed', options: { start_in: '1 minute' } }
+  end
+
+  def unschedule
+    pipeline.builds.scheduled.map(&:unschedule)
+  end
 end

spec/services/ci/retry_build_service_spec.rb

@@ -27,7 +27,7 @@ describe Ci::RetryBuildService do
        job_artifacts_metadata job_artifacts_trace job_artifacts_junit
        job_artifacts_sast job_artifacts_dependency_scanning
        job_artifacts_container_scanning job_artifacts_dast
-       job_artifacts_codequality].freeze
+       job_artifacts_codequality scheduled_at].freeze
 
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags trace_sections
@@ -44,7 +44,8 @@ describe Ci::RetryBuildService do
       create(:ci_build, :failed, :expired, :erased, :queued, :coverage, :tags,
              :allowed_to_fail, :on_tag, :triggered, :teardown_environment,
              description: 'my-job', stage: 'test', stage_id: stage.id,
-             pipeline: pipeline, auto_canceled_by: another_pipeline)
+             pipeline: pipeline, auto_canceled_by: another_pipeline,
+             scheduled_at: 10.seconds.since)
     end
 
     before do

spec/services/ci/run_scheduled_build_service_spec.rb

+require 'spec_helper'
+
+describe Ci::RunScheduledBuildService do
+  let(:user) { create(:user) }
+  let(:project) { create(:project) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+
+  subject { described_class.new(project, user).execute(build) }
+
+  before do
+    stub_feature_flags(ci_enable_scheduled_build: true)
+  end
+
+  context 'when user can update build' do
+    before do
+      project.add_developer(user)
+
+      create(:protected_branch, :developers_can_merge,
+             name: pipeline.ref, project: project)
+    end
+
+    context 'when build is scheduled' do
+      context 'when scheduled_at is expired' do
+        let(:build) { create(:ci_build, :expired_scheduled, user: user, project: project, pipeline: pipeline) }
+
+        it 'can run the build' do
+          expect { subject }.not_to raise_error
+
+          expect(build).to be_pending
+        end
+      end
+
+      context 'when scheduled_at is not expired' do
+        let(:build) { create(:ci_build, :scheduled, user: user, project: project, pipeline: pipeline) }
+
+        it 'can not run the build' do
+          expect { subject }.to raise_error(StateMachines::InvalidTransition)
+
+          expect(build).to be_scheduled
+        end
+      end
+    end
+
+    context 'when build is not scheduled' do
+      let(:build) { create(:ci_build, :created, user: user, project: project, pipeline: pipeline) }
+
+      it 'can not run the build' do
+        expect { subject }.to raise_error(StateMachines::InvalidTransition)
+
+        expect(build).to be_created
+      end
+    end
+  end
+
+  context 'when user can not update build' do
+    context 'when build is scheduled' do
+      let(:build) { create(:ci_build, :scheduled, user: user, project: project, pipeline: pipeline) }
+
+      it 'can not run the build' do
+        expect { subject }.to raise_error(Gitlab::Access::AccessDeniedError)
+
+        expect(build).to be_scheduled
+      end
+    end
+  end
+end

spec/workers/ci/build_schedule_worker_spec.rb

+require 'spec_helper'
+
+describe Ci::BuildScheduleWorker do
+  subject { described_class.new.perform(build.id) }
+
+  context 'when build is found' do
+    context 'when build is scheduled' do
+      let(:build) { create(:ci_build, :scheduled) }
+
+      it 'executes RunScheduledBuildService' do
+        expect_any_instance_of(Ci::RunScheduledBuildService)
+          .to receive(:execute).once
+
+        subject
+      end
+    end
+
+    context 'when build is not scheduled' do
+      let(:build) { create(:ci_build, :created) }
+
+      it 'executes RunScheduledBuildService' do
+        expect_any_instance_of(Ci::RunScheduledBuildService)
+          .not_to receive(:execute)
+
+        subject
+      end
+    end
+  end
+
+  context 'when build is not found' do
+    let(:build) { build_stubbed(:ci_build, :scheduled) }
+
+    it 'does nothing' do
+      expect_any_instance_of(Ci::RunScheduledBuildService)
+        .not_to receive(:execute)
+
+      subject
+    end
+  end
+end

spec/workers/stuck_ci_jobs_worker_spec.rb

@@ -127,6 +127,47 @@ describe StuckCiJobsWorker do
     end
   end
 
+  describe 'drop stale scheduled builds' do
+    let(:status) { 'scheduled' }
+    let(:updated_at) { }
+
+    context 'when scheduled at 2 hours ago but it is not executed yet' do
+      let!(:job) { create(:ci_build, :scheduled, scheduled_at: 2.hours.ago) }
+
+      it 'drops the stale scheduled build' do
+        expect(Ci::Build.scheduled.count).to eq(1)
+        expect(job).to be_scheduled
+
+        worker.perform
+        job.reload
+
+        expect(Ci::Build.scheduled.count).to eq(0)
+        expect(job).to be_failed
+        expect(job).to be_stale_schedule
+      end
+    end
+
+    context 'when scheduled at 30 minutes ago but it is not executed yet' do
+      let!(:job) { create(:ci_build, :scheduled, scheduled_at: 30.minutes.ago) }
+
+      it 'does not drop the stale scheduled build yet' do
+        expect(Ci::Build.scheduled.count).to eq(1)
+        expect(job).to be_scheduled
+
+        worker.perform
+
+        expect(Ci::Build.scheduled.count).to eq(1)
+        expect(job).to be_scheduled
+      end
+    end
+
+    context 'when there are no stale scheduled builds' do
+      it 'does not drop the stale scheduled build yet' do
+        expect { worker.perform }.not_to raise_error
+      end
+    end
+  end
+
   describe 'exclusive lease' do
     let(:status) { 'running' }
     let(:updated_at) { 2.days.ago }