Merge branch 'mc/feature/pipeline-tracking' into 'master'

Add bridge relation between Pipelines and Bridges

See merge request gitlab-org/gitlab-ee!11264

app/models/concerns/has_status.rb

@@ -66,6 +66,10 @@ module HasStatus
     def all_state_names
       state_machines.values.flat_map(&:states).flat_map { |s| s.map(&:name) }
     end
+
+    def completed_statuses
+      COMPLETED_STATUSES.map(&:to_sym)
+    end
   end
 
   included do

db/schema.rb

@@ -511,6 +511,7 @@ ActiveRecord::Schema.define(version: 20190506135400) do
     t.integer "failure_reason"
     t.datetime_with_timezone "scheduled_at"
     t.string "token_encrypted"
+    t.integer "upstream_pipeline_id"
     t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
     t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
     t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY (ARRAY[('sast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('sast:container'::character varying)::text, ('container_scanning'::character varying)::text, ('dast'::character varying)::text])))", using: :btree
@@ -530,6 +531,7 @@ ActiveRecord::Schema.define(version: 20190506135400) do
     t.index ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree
     t.index ["token_encrypted"], name: "index_ci_builds_on_token_encrypted", unique: true, where: "(token_encrypted IS NOT NULL)", using: :btree
     t.index ["updated_at"], name: "index_ci_builds_on_updated_at", using: :btree
+    t.index ["upstream_pipeline_id"], name: "index_ci_builds_on_upstream_pipeline_id", where: "(upstream_pipeline_id IS NOT NULL)", using: :btree
     t.index ["user_id"], name: "index_ci_builds_on_user_id", using: :btree
   end
 
@@ -3489,6 +3491,7 @@ ActiveRecord::Schema.define(version: 20190506135400) do
   add_foreign_key "ci_build_trace_sections", "projects", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
   add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade
+  add_foreign_key "ci_builds", "ci_pipelines", column: "upstream_pipeline_id", name: "fk_87f4cefcda", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
   add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
   add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade

ee/app/models/ee/ci/bridge.rb

@@ -13,6 +13,8 @@ module EE
         serialize :yaml_variables, ::Gitlab::Serializer::Ci::Variables
         # rubocop:enable Cop/ActiveRecordSerialize
 
+        belongs_to :upstream_pipeline, class_name: "::Ci::Pipeline",
+                                       foreign_key: :upstream_pipeline_id
         has_many :sourced_pipelines, class_name: "::Ci::Sources::Pipeline",
                                      foreign_key: :source_job_id
 

ee/app/models/ee/ci/pipeline.rb

@@ -28,6 +28,8 @@ module EE
         has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
         has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
+        has_many :downstream_bridges, class_name: '::Ci::Bridge', foreign_key: :upstream_pipeline_id
+
         # Legacy way to fetch security reports based on job name. This has been replaced by the reports feature.
         scope :with_legacy_security_reports, -> do
           joins(:artifacts).where(ci_builds: { name: %w[sast dependency_scanning sast:container container_scanning dast] })
@@ -86,7 +88,7 @@ module EE
         }.freeze
 
         state_machine :status do
-          after_transition any => ::Ci::Pipeline::COMPLETED_STATUSES.map(&:to_sym) do |pipeline|
+          after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
             next unless pipeline.has_reports?(::Ci::JobArtifact.security_reports) && pipeline.default_branch?
 
             pipeline.run_after_commit do
@@ -94,6 +96,14 @@ module EE
             end
           end
 
+          after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
+            next unless pipeline.downstream_bridges.any?
+
+            pipeline.run_after_commit do
+              ::Ci::PipelineBridgeStatusWorker.perform_async(pipeline.id)
+            end
+          end
+
           after_transition created: :pending do |pipeline|
             next unless pipeline.bridge_triggered?
 

ee/app/services/ci/pipeline_bridge_status_service.rb

+# frozen_string_literal: true
+
+module Ci
+  class PipelineBridgeStatusService < ::BaseService
+    def execute(pipeline)
+      pipeline.downstream_bridges.each do |bridged_job|
+        process_bridged_job(pipeline.status, bridged_job)
+      end
+    end
+
+    def process_bridged_job(status, job)
+      case status
+      when 'success'
+        job.success!
+      when 'failed'
+        job.drop!
+      when 'canceled'
+        job.cancel!
+      when 'skipped'
+        job.skip!
+      end
+    end
+  end
+end

ee/app/workers/all_queues.yml

@@ -44,6 +44,7 @@
 
 - pipeline_default:store_security_reports
 - pipeline_default:ci_create_cross_project_pipeline
+- pipeline_default:ci_pipeline_bridge_status
 
 - incident_management:incident_management_process_alert
 

ee/app/workers/ci/pipeline_bridge_status_worker.rb

+# frozen_string_literal: true
+
+module Ci
+  class PipelineBridgeStatusWorker
+    include ::ApplicationWorker
+    include ::PipelineQueue
+
+    def perform(pipeline_id)
+      ::Ci::Pipeline.find_by_id(pipeline_id).try do |pipeline|
+        ::Ci::PipelineBridgeStatusService
+          .new(pipeline.project, pipeline.user)
+          .execute(pipeline)
+      end
+    end
+  end
+end

ee/db/migrate/20190419121952_add_bridged_pipeline_id_to_bridges.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddBridgedPipelineIdToBridges < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :ci_builds, :upstream_pipeline_id, :integer
+  end
+end

ee/db/migrate/20190419123057_add_bridged_pipeline_id_foreign_key.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddBridgedPipelineIdForeignKey < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_builds, :upstream_pipeline_id, where: 'upstream_pipeline_id IS NOT NULL'
+    add_concurrent_foreign_key :ci_builds, :ci_pipelines, column: :upstream_pipeline_id
+  end
+
+  def down
+    remove_foreign_key :ci_builds, column: :upstream_pipeline_id
+    remove_concurrent_index :ci_builds, :upstream_pipeline_id
+  end
+end

ee/spec/lib/gitlab/import_export/all_models.yml

@@ -22,6 +22,7 @@ ci_pipelines:
 - sourced_pipelines
 - triggered_by_pipeline
 - triggered_pipelines
+- downstream_bridges
 - job_artifacts
 - vulnerabilities_occurrence_pipelines
 - vulnerabilities

ee/spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -6,3 +6,5 @@ ProjectTracingSetting:
 - external_url
 Note:
 - review_id
+CommitStatus:
+- upstream_pipeline_id

ee/spec/models/ci/bridge_spec.rb

@@ -14,6 +14,10 @@ describe Ci::Bridge do
     { trigger: { project: 'my/project', branch: 'master' } }
   end
 
+  it 'belongs to an upstream pipeline' do
+    expect(bridge).to belong_to(:upstream_pipeline)
+  end
+
   it 'has many sourced pipelines' do
     expect(bridge).to have_many(:sourced_pipelines)
   end

ee/spec/models/ci/pipeline_spec.rb

@@ -14,6 +14,7 @@ describe Ci::Pipeline do
   it { is_expected.to have_many(:sourced_pipelines) }
   it { is_expected.to have_one(:triggered_by_pipeline) }
   it { is_expected.to have_many(:triggered_pipelines) }
+  it { is_expected.to have_many(:downstream_bridges) }
   it { is_expected.to have_many(:job_artifacts).through(:builds) }
   it { is_expected.to have_many(:vulnerabilities).through(:vulnerabilities_occurrence_pipelines).class_name('Vulnerabilities::Occurrence') }
   it { is_expected.to have_many(:vulnerabilities_occurrence_pipelines).class_name('Vulnerabilities::OccurrencePipeline') }
@@ -439,6 +440,20 @@ describe Ci::Pipeline do
         end
       end
     end
+
+    context 'when pipeline has bridged jobs' do
+      before do
+        pipeline.downstream_bridges << create(:ci_bridge)
+      end
+
+      context "when transitioning to success" do
+        it 'schedules the pipeline bridge worker' do
+          expect(::Ci::PipelineBridgeStatusWorker).to receive(:perform_async).with(pipeline.id)
+
+          pipeline.succeed!
+        end
+      end
+    end
   end
 
   describe '#ci_yaml_file_path' do

ee/spec/services/ci/pipeline_bridge_status_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::PipelineBridgeStatusService do
+  let(:user) { create(:user) }
+  let(:project) { create(:project) }
+  let(:pipeline) { create(:ci_pipeline, :success, project: project) }
+
+  describe '#execute' do
+    subject { described_class.new(project, user).execute(pipeline) }
+
+    context 'when pipeline has bridged jobs' do
+      let(:bridge) { create(:ci_bridge, status: :pending) }
+
+      before do
+        pipeline.downstream_bridges << bridge
+      end
+
+      it 'updates the bridge status with the pipeline status' do
+        expect { subject }.to change { bridge.status }.from('pending').to('success')
+      end
+    end
+  end
+end

ee/spec/workers/ci/pipeline_bridge_status_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::PipelineBridgeStatusWorker do
+  describe '#perform' do
+    subject { described_class.new.perform(pipeline_id) }
+
+    context 'when pipeline exists' do
+      let(:pipeline) { create(:ci_pipeline) }
+      let(:pipeline_id) { pipeline.id }
+
+      it 'calls the service' do
+        service = double('bridge status service')
+
+        expect(Ci::PipelineBridgeStatusService)
+          .to receive(:new)
+          .with(pipeline.project, pipeline.user)
+          .and_return(service)
+
+        expect(service).to receive(:execute).with(pipeline)
+
+        subject
+      end
+    end
+
+    context 'when pipeline does not exist' do
+      let(:pipeline_id) { 1234 }
+
+      it 'does not call the service' do
+        expect(Ci::PipelineBridgeStatusService)
+          .not_to receive(:new)
+
+        subject
+      end
+    end
+  end
+end

spec/services/ci/retry_build_service_spec.rb

@@ -38,7 +38,7 @@ describe Ci::RetryBuildService do
        runner_id tag_taggings taggings tags trigger_request_id
        user_id auto_canceled_by_id retried failure_reason
        sourced_pipelines artifacts_file_store artifacts_metadata_store
-       metadata runner_session trace_chunks].freeze
+       metadata runner_session trace_chunks upstream_pipeline_id].freeze
 
   shared_examples 'build duplication' do
     let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }