Drop builds that are out of CI minutes when pipeline is created

It drops only the builds that are out of CI minutes when the pipeline is created.
Previously we were dropping builds that didn't match any runners, but it turns
out that users register runners during the pipeline execution to handle jobs in
the later stages.

app/models/concerns/enums/ci/commit_status.rb

@@ -24,7 +24,7 @@ module Enums
           project_deleted: 15,
           ci_quota_exceeded: 16,
           pipeline_loop_detected: 17,
-          no_matching_runner: 18,
+          no_matching_runner: 18, # not used anymore, but cannot be deleted because of old data
           insufficient_bridge_permissions: 1_001,
           downstream_bridge_project_not_found: 1_002,
           invalid_bridge_trigger: 1_003,

app/services/ci/pipeline_creation/start_pipeline_service.rb

@@ -10,9 +10,10 @@ module Ci
       end
 
       def execute
-        DropNotRunnableBuildsService.new(pipeline).execute
         Ci::ProcessPipelineService.new(pipeline).execute
       end
     end
   end
 end
+
+::Ci::PipelineCreation::StartPipelineService.prepend_mod_with('Ci::PipelineCreation::StartPipelineService')

app/services/ci/pipeline_creation/drop_not_runnable_builds_service.rb → ee/app/services/ci/pipeline_creation/drop_not_runnable_builds_service.rb

@@ -3,7 +3,7 @@
 module Ci
   module PipelineCreation
     class DropNotRunnableBuildsService
-      include Gitlab::Utils::StrongMemoize
+      include ::Gitlab::Utils::StrongMemoize
 
       def initialize(pipeline)
         @pipeline = pipeline
@@ -16,40 +16,58 @@ module Ci
       def execute
         return unless ::Feature.enabled?(:ci_drop_new_builds_when_ci_quota_exceeded, project, default_enabled: :yaml)
         return unless pipeline.created?
+        return unless project.shared_runners_enabled?
+        return unless project.ci_minutes_quota.minutes_used_up?
 
-        load_runners
         validate_build_matchers
       end
 
       private
 
       attr_reader :pipeline
-      attr_reader :instance_runners, :private_runners
       delegate :project, to: :pipeline
 
-      def load_runners
-        @instance_runners, @private_runners = project
-          .all_runners
-          .active
-          .online
-          .runner_matchers
-          .partition(&:instance_type?)
+      def validate_build_matchers
+        build_ids = pipeline
+          .build_matchers
+          .filter_map { |matcher| matcher.build_ids if should_drop?(matcher) }
+          .flatten
+
+        drop_all_builds(build_ids, :ci_quota_exceeded)
       end
 
-      def validate_build_matchers
-        pipeline.build_matchers.each do |build_matcher|
-          failure_reason = validate_build_matcher(build_matcher)
-          next unless failure_reason
+      def should_drop?(build_matcher)
+        matches_instance_runners_and_quota_used_up?(build_matcher) &&
+          !matches_private_runners?(build_matcher)
+      end
 
-          drop_all_builds(build_matcher.build_ids, failure_reason)
+      def matches_instance_runners_and_quota_used_up?(build_matcher)
+        instance_runners.any? do |matcher|
+          matcher.matches?(build_matcher) &&
+            !matcher.matches_quota?(build_matcher)
         end
       end
 
-      def validate_build_matcher(build_matcher)
-        return if matching_private_runners?(build_matcher)
-        return if matching_instance_runners?(build_matcher)
+      def matches_private_runners?(build_matcher)
+        private_runners.any? { |matcher| matcher.matches?(build_matcher) }
+      end
+
+      def instance_runners
+        strong_memoize(:instance_runners) do
+          runner_matchers.select(&:instance_type?)
+        end
+      end
+
+      def private_runners
+        strong_memoize(:private_runners) do
+          runner_matchers.reject(&:instance_type?)
+        end
+      end
 
-        matching_failure_reason(build_matcher)
+      def runner_matchers
+        strong_memoize(:runner_matchers) do
+          project.all_runners.active.online.runner_matchers
+        end
       end
 
       ##
@@ -58,34 +76,12 @@ module Ci
       # transition to other states by `PipelineProcessWorker` running async.
       #
       def drop_all_builds(build_ids, failure_reason)
+        return if build_ids.empty?
+
         pipeline.builds.id_in(build_ids).each do |build|
-          build.drop(failure_reason, skip_pipeline_processing: true)
+          build.drop!(failure_reason, skip_pipeline_processing: true)
         end
       end
-
-      def matching_private_runners?(build_matcher)
-        private_runners
-          .find { |matcher| matcher.matches?(build_matcher) }
-          .present?
-      end
-
-      def matching_instance_runners?(build_matcher)
-        instance_runners
-          .find { |matcher| matching_criteria(matcher, build_matcher) }
-          .present?
-      end
-
-      # Overridden in EE
-      def matching_criteria(runner_matcher, build_matcher)
-        runner_matcher.matches?(build_matcher)
-      end
-
-      # Overridden in EE
-      def matching_failure_reason(build_matcher)
-        :no_matching_runner
-      end
     end
   end
 end
-
-Ci::PipelineCreation::DropNotRunnableBuildsService.prepend_mod_with('Ci::PipelineCreation::DropNotRunnableBuildsService')

ee/app/services/ee/ci/pipeline_creation/drop_not_runnable_builds_service.rb

-# frozen_string_literal: true
-
-module EE
-  module Ci
-    module PipelineCreation
-      module DropNotRunnableBuildsService
-        extend ::Gitlab::Utils::Override
-
-        private
-
-        override :matching_criteria
-        def matching_criteria(runner_matcher, build_matcher)
-          super && runner_matcher.matches_quota?(build_matcher)
-        end
-
-        override :matching_failure_reason
-        def matching_failure_reason(build_matcher)
-          if build_matcher.project.shared_runners_enabled_but_unavailable?
-            :ci_quota_exceeded
-          else
-            :no_matching_runner
-          end
-        end
-      end
-    end
-  end
-end

ee/app/services/ee/ci/pipeline_creation/start_pipeline_service.rb

+# frozen_string_literal: true
+
+module EE
+  module Ci
+    module PipelineCreation
+      module StartPipelineService
+        extend ::Gitlab::Utils::Override
+
+        override :execute
+        def execute
+          ::Ci::PipelineCreation::DropNotRunnableBuildsService.new(pipeline).execute
+          super
+        end
+      end
+    end
+  end
+end

ee/spec/services/ci/create_pipeline_service/runnable_builds_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService, :sidekiq_inline do
+  let_it_be(:namespace) { create(:namespace, :with_used_build_minutes_limit) }
+  let_it_be(:project) { create(:project, :repository, namespace: namespace) }
+  let_it_be(:user) { project.owner }
+  let_it_be(:instance_runner) { create(:ci_runner, :instance, :online) }
+
+  let(:service) do
+    described_class.new(project, user, { ref: 'refs/heads/master' })
+  end
+
+  let(:config) do
+    <<~EOY
+    job1:
+     stage: build
+     script:
+       - echo "deploy runner 123"
+
+    job2:
+      stage: test
+      script:
+        - echo "run on runner 123"
+      tags:
+        - "123"
+    EOY
+  end
+
+  before do
+    project.add_developer(user)
+    stub_ci_pipeline_yaml_file(config)
+  end
+
+  it 'drops builds that match shared runners' do
+    pipeline = create_pipeline!
+
+    job1 = pipeline.builds.find_by_name('job1')
+    job2 = pipeline.builds.find_by_name('job2')
+
+    expect(job1).to be_failed
+    expect(job1.failure_reason).to eq('ci_quota_exceeded')
+    expect(job2).not_to be_failed
+  end
+
+  context 'with private runners' do
+    let_it_be(:private_runner) do
+      create(:ci_runner, :project, :online, projects: [project])
+    end
+
+    it 'does not drop the builds' do
+      pipeline = create_pipeline!
+
+      job1 = pipeline.builds.find_by_name('job1')
+      job2 = pipeline.builds.find_by_name('job2')
+
+      expect(job1).not_to be_failed
+      expect(job2).not_to be_failed
+    end
+  end
+
+  def create_pipeline!
+    service.execute(:push)
+  end
+end

ee/spec/services/ci/pipeline_creation/drop_not_runnable_builds_service_spec.rb

@@ -3,14 +3,21 @@
 require 'spec_helper'
 
 RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
+
   let_it_be_with_reload(:pipeline) do
-    create(:ci_pipeline, status: :created)
+    create(:ci_pipeline, project: project, status: :created)
   end
 
   let_it_be_with_reload(:job) do
     create(:ci_build, project: pipeline.project, pipeline: pipeline)
   end
 
+  let_it_be_with_reload(:job_with_tags) do
+    create(:ci_build, :tags, project: pipeline.project, pipeline: pipeline)
+  end
+
   let_it_be(:instance_runner) do
     create(:ci_runner,
       :online,
@@ -22,19 +29,28 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
   describe '#execute' do
     subject(:execute) { described_class.new(pipeline).execute }
 
-    shared_examples 'available CI quota' do
+    shared_examples 'jobs allowed to run' do
       it 'does not drop the jobs' do
-        expect { execute }.not_to change { job.reload.status }
+        expect { execute }
+          .to not_change { job.reload.status }
+          .and not_change { job_with_tags.reload.status }
       end
     end
 
     shared_examples 'limit exceeded' do
+      before do
+        allow(pipeline.project).to receive(:ci_minutes_quota)
+          .and_return(double('quota', minutes_used_up?: true))
+      end
+
       it 'drops the job with ci_quota_exceeded reason' do
         execute
-        job.reload
+        [job, job_with_tags].each(&:reload)
 
         expect(job).to be_failed
         expect(job.failure_reason).to eq('ci_quota_exceeded')
+
+        expect(job_with_tags).to be_pending
       end
 
       context 'when shared runners are disabled' do
@@ -42,13 +58,39 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
           pipeline.project.update!(shared_runners_enabled: false)
         end
 
-        it 'drops the job with no_matching_runner reason' do
-          execute
-          job.reload
+        it_behaves_like 'jobs allowed to run'
+      end
+
+      context 'with project runners' do
+        let_it_be(:project_runner) do
+          create(:ci_runner, :online, runner_type: :project_type, projects: [project])
+        end
+
+        it_behaves_like 'jobs allowed to run'
+      end
+
+      context 'with group runners' do
+        let_it_be(:group_runner) do
+          create(:ci_runner, :online, runner_type: :group_type, groups: [group])
+        end
+
+        it_behaves_like 'jobs allowed to run'
+      end
+
+      context 'when the feature flag is disabled' do
+        before do
+          stub_feature_flags(ci_drop_new_builds_when_ci_quota_exceeded: false)
+        end
+
+        it_behaves_like 'jobs allowed to run'
+      end
 
-          expect(job).to be_failed
-          expect(job.failure_reason).to eq('no_matching_runner')
+      context 'when the pipeline status is running' do
+        before do
+          pipeline.update!(status: :running)
         end
+
+        it_behaves_like 'jobs allowed to run'
       end
     end
 
@@ -57,7 +99,7 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
         pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::PUBLIC)
       end
 
-      it_behaves_like 'available CI quota'
+      it_behaves_like 'jobs allowed to run'
 
       context 'when the CI quota is exceeded' do
         before do
@@ -65,9 +107,7 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
             .and_return(double('quota', minutes_used_up?: true))
         end
 
-        it 'does not drop the jobs' do
-          expect { execute }.not_to change { job.reload.status }
-        end
+        it_behaves_like 'jobs allowed to run'
       end
     end
 
@@ -76,16 +116,8 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
         pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::INTERNAL)
       end
 
-      it_behaves_like 'available CI quota'
-
-      context 'when the Ci quota is exceeded' do
-        before do
-          allow(pipeline.project).to receive(:ci_minutes_quota)
-            .and_return(double('quota', minutes_used_up?: true))
-        end
-
-        it_behaves_like 'limit exceeded'
-      end
+      it_behaves_like 'jobs allowed to run'
+      it_behaves_like 'limit exceeded'
     end
 
     context 'with private projects' do
@@ -93,16 +125,8 @@ RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
         pipeline.project.update!(visibility_level: ::Gitlab::VisibilityLevel::PRIVATE)
       end
 
-      it_behaves_like 'available CI quota'
-
-      context 'when the Ci quota is exceeded' do
-        before do
-          allow(pipeline.project).to receive(:ci_minutes_quota)
-            .and_return(double('quota', minutes_used_up?: true))
-        end
-
-        it_behaves_like 'limit exceeded'
-      end
+      it_behaves_like 'jobs allowed to run'
+      it_behaves_like 'limit exceeded'
     end
   end
 end

ee/spec/services/ci/pipeline_creation/start_pipeline_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::PipelineCreation::StartPipelineService do
+  let(:pipeline) { build(:ci_pipeline) }
+
+  subject(:service) { described_class.new(pipeline) }
+
+  describe '#execute' do
+    it 'calls the pipeline runners matching validation service' do
+      expect(Ci::PipelineCreation::DropNotRunnableBuildsService)
+        .to receive(:new)
+        .with(pipeline)
+        .and_return(double('service', execute: true))
+
+      service.execute
+    end
+  end
+end

ee/spec/services/ci/process_pipeline_service_spec.rb

@@ -29,48 +29,22 @@ RSpec.describe Ci::ProcessPipelineService, '#execute' do
       stub_ci_pipeline_to_return_yaml_file
     end
 
-    context 'when there is a runner available' do
-      let_it_be(:runner) do
-        create(:ci_runner, :online, tag_list: %w[ruby postgres mysql])
-      end
+    it 'creates a downstream cross-project pipeline' do
+      service.execute
+      Sidekiq::Worker.drain_all
 
-      it 'creates a downstream cross-project pipeline' do
-        service.execute
-        Sidekiq::Worker.drain_all
+      expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
 
-        expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
+      update_build_status(:test, :success)
+      Sidekiq::Worker.drain_all
 
-        update_build_status(:test, :success)
-        Sidekiq::Worker.drain_all
+      expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
 
-        expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
-
-        expect(downstream.ci_pipelines).to be_one
-        expect(downstream.ci_pipelines.first).to be_pending
-        expect(downstream.builds).not_to be_empty
-        expect(downstream.builds.first.variables)
-          .to include(key: 'BRIDGE', value: 'cross', public: false, masked: false)
-      end
-    end
-
-    context 'with no runners' do
-      it 'creates a failed downstream cross-project pipeline' do
-        service.execute
-        Sidekiq::Worker.drain_all
-
-        expect_statuses(%w[test pending], %w[cross created], %w[deploy created])
-
-        update_build_status(:test, :success)
-        Sidekiq::Worker.drain_all
-
-        expect_statuses(%w[test success], %w[cross success], %w[deploy pending])
-
-        expect(downstream.ci_pipelines).to be_one
-        expect(downstream.ci_pipelines.first).to be_failed
-        expect(downstream.builds).not_to be_empty
-        expect(downstream.builds).to all be_failed
-        expect(downstream.builds.map(&:failure_reason)).to all eq('no_matching_runner')
-      end
+      expect(downstream.ci_pipelines).to be_one
+      expect(downstream.ci_pipelines.first).to be_pending
+      expect(downstream.builds).not_to be_empty
+      expect(downstream.builds.first.variables)
+        .to include(key: 'BRIDGE', value: 'cross', public: false, masked: false)
     end
   end
 

ee/spec/workers/ci/initial_pipeline_process_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::InitialPipelineProcessWorker do
+  describe '#perform' do
+    let_it_be(:namespace) { create(:namespace, :with_used_build_minutes_limit) }
+    let_it_be(:project) { create(:project, namespace: namespace) }
+    let_it_be_with_reload(:pipeline) do
+      create(:ci_pipeline, :with_job, project: project, status: :created)
+    end
+
+    let_it_be(:instance_runner) { create(:ci_runner, :instance, :online) }
+
+    include_examples 'an idempotent worker' do
+      let(:job_args) { pipeline.id }
+
+      context 'when the project is out of CI minutes' do
+        it 'marks the pipeline as failed' do
+          expect(pipeline).to be_created
+
+          subject
+
+          expect(pipeline.reload).to be_failed
+        end
+      end
+    end
+  end
+end

qa/qa/specs/features/browser_ui/4_verify/pipeline/create_and_process_pipeline_spec.rb

 # frozen_string_literal: true
 
 module QA
-  # TODO: Remove `:requires_admin` meta when the feature flag is removed
-  RSpec.describe 'Verify', :runner, :requires_admin do
+  RSpec.describe 'Verify', :runner do
     describe 'Pipeline creation and processing' do
       let(:executor) { "qa-runner-#{Time.now.to_i}" }
       let(:max_wait) { 30 }
-      let(:feature_flag) { :ci_drop_new_builds_when_ci_quota_exceeded }
 
       let(:project) do
         Resource::Project.fabricate_via_api! do |project|
@@ -28,8 +26,6 @@ module QA
 
       it 'users creates a pipeline which gets processed', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1849' do
         # TODO: Convert back to :smoke once proved to be stable. Related issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300909
-        tags_mismatch_status = Runtime::Feature.enabled?(feature_flag, project: project) ? :failed : :pending
-
         Flow::Login.sign_in
 
         Resource::Repository::Commit.fabricate_via_api! do |commit|
@@ -75,7 +71,7 @@ module QA
         {
           'test-success': :passed,
           'test-failure': :failed,
-          'test-tags-mismatch': tags_mismatch_status,
+          'test-tags-mismatch': :pending,
           'test-artifacts': :passed
         }.each do |job, status|
           Page::Project::Pipeline::Show.perform do |pipeline|

spec/services/ci/create_pipeline_service/needs_spec.rb

@@ -202,37 +202,21 @@ RSpec.describe Ci::CreatePipelineService do
         YAML
       end
 
-      context 'when there are runners matching the builds' do
-        before do
-          create(:ci_runner, :online)
-        end
-
-        it 'creates a pipeline with build_a and test_b pending; deploy_b manual', :sidekiq_inline do
-          processables = pipeline.processables
-
-          build_a = processables.find { |processable| processable.name == 'build_a' }
-          test_a = processables.find { |processable| processable.name == 'test_a' }
-          test_b = processables.find { |processable| processable.name == 'test_b' }
-          deploy_a = processables.find { |processable| processable.name == 'deploy_a' }
-          deploy_b = processables.find { |processable| processable.name == 'deploy_b' }
-
-          expect(pipeline).to be_created_successfully
-          expect(build_a.status).to eq('pending')
-          expect(test_a.status).to eq('created')
-          expect(test_b.status).to eq('pending')
-          expect(deploy_a.status).to eq('created')
-          expect(deploy_b.status).to eq('manual')
-        end
-      end
-
-      context 'when there are no runners matching the builds' do
-        it 'creates a pipeline but all jobs failed', :sidekiq_inline do
-          processables = pipeline.processables
-
-          expect(pipeline).to be_created_successfully
-          expect(processables).to all be_failed
-          expect(processables.map(&:failure_reason)).to all eq('no_matching_runner')
-        end
+      it 'creates a pipeline with build_a and test_b pending; deploy_b manual', :sidekiq_inline do
+        processables = pipeline.processables
+
+        build_a = processables.find { |processable| processable.name == 'build_a' }
+        test_a = processables.find { |processable| processable.name == 'test_a' }
+        test_b = processables.find { |processable| processable.name == 'test_b' }
+        deploy_a = processables.find { |processable| processable.name == 'deploy_a' }
+        deploy_b = processables.find { |processable| processable.name == 'deploy_b' }
+
+        expect(pipeline).to be_created_successfully
+        expect(build_a.status).to eq('pending')
+        expect(test_a.status).to eq('created')
+        expect(test_b.status).to eq('pending')
+        expect(deploy_a.status).to eq('created')
+        expect(deploy_b.status).to eq('manual')
       end
     end
 

spec/services/ci/pipeline_creation/drop_not_runnable_builds_service_spec.rb

-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Ci::PipelineCreation::DropNotRunnableBuildsService do
-  let_it_be(:group) { create(:group) }
-  let_it_be(:project) { create(:project, group: group) }
-
-  let_it_be_with_reload(:pipeline) do
-    create(:ci_pipeline, project: project, status: :created)
-  end
-
-  let_it_be_with_reload(:job) do
-    create(:ci_build, project: project, pipeline: pipeline)
-  end
-
-  describe '#execute' do
-    subject(:execute) { described_class.new(pipeline).execute }
-
-    shared_examples 'jobs allowed to run' do
-      it 'does not drop the jobs' do
-        expect { execute }.not_to change { job.reload.status }
-      end
-    end
-
-    context 'when the feature flag is disabled' do
-      before do
-        stub_feature_flags(ci_drop_new_builds_when_ci_quota_exceeded: false)
-      end
-
-      it_behaves_like 'jobs allowed to run'
-    end
-
-    context 'when the pipeline status is running' do
-      before do
-        pipeline.update!(status: :running)
-      end
-
-      it_behaves_like 'jobs allowed to run'
-    end
-
-    context 'when there are no runners available' do
-      let_it_be(:offline_project_runner) do
-        create(:ci_runner, runner_type: :project_type, projects: [project])
-      end
-
-      it 'drops the job' do
-        execute
-        job.reload
-
-        expect(job).to be_failed
-        expect(job.failure_reason).to eq('no_matching_runner')
-      end
-    end
-
-    context 'with project runners' do
-      let_it_be(:project_runner) do
-        create(:ci_runner, :online, runner_type: :project_type, projects: [project])
-      end
-
-      it_behaves_like 'jobs allowed to run'
-    end
-
-    context 'with group runners' do
-      let_it_be(:group_runner) do
-        create(:ci_runner, :online, runner_type: :group_type, groups: [group])
-      end
-
-      it_behaves_like 'jobs allowed to run'
-    end
-
-    context 'with instance runners' do
-      let_it_be(:instance_runner) do
-        create(:ci_runner, :online, runner_type: :instance_type)
-      end
-
-      it_behaves_like 'jobs allowed to run'
-    end
-  end
-end

spec/services/ci/pipeline_creation/start_pipeline_service_spec.rb

@@ -8,15 +8,6 @@ RSpec.describe Ci::PipelineCreation::StartPipelineService do
   subject(:service) { described_class.new(pipeline) }
 
   describe '#execute' do
-    it 'calls the pipeline runners matching validation service' do
-      expect(Ci::PipelineCreation::DropNotRunnableBuildsService)
-        .to receive(:new)
-        .with(pipeline)
-        .and_return(double('service', execute: true))
-
-      service.execute
-    end
-
     it 'calls the pipeline process service' do
       expect(Ci::ProcessPipelineService)
         .to receive(:new)

spec/workers/ci/initial_pipeline_process_worker_spec.rb

@@ -11,26 +11,12 @@ RSpec.describe Ci::InitialPipelineProcessWorker do
     include_examples 'an idempotent worker' do
       let(:job_args) { pipeline.id }
 
-      context 'when there are runners available' do
-        before do
-          create(:ci_runner, :online)
-        end
-
-        it 'marks the pipeline as pending' do
-          expect(pipeline).to be_created
-
-          subject
-
-          expect(pipeline.reload).to be_pending
-        end
-      end
-
-      it 'marks the pipeline as failed' do
+      it 'marks the pipeline as pending' do
         expect(pipeline).to be_created
 
         subject
 
-        expect(pipeline.reload).to be_failed
+        expect(pipeline.reload).to be_pending
       end
     end
   end