Merge branch 'data-endpoint-for-dag-visualization' into 'master'

Add endpoint for serialized DAG pipeline

See merge request gitlab-org/gitlab!31324

app/models/ci/group.rb

@@ -46,7 +46,7 @@ module Ci
     end
 
     def self.fabricate(project, stage)
-      stage.statuses.ordered.latest
+      stage.latest_statuses
         .sort_by(&:sortable_name).group_by(&:group_name)
         .map do |group_name, grouped_statuses|
           self.new(project, stage, name: group_name, jobs: grouped_statuses)

app/models/ci/legacy_stage.rb

@@ -41,6 +41,10 @@ module Ci
         .fabricate!
     end
 
+    def latest_statuses
+      statuses.ordered.latest
+    end
+
     def statuses
       @statuses ||= pipeline.statuses.where(stage: name)
     end

app/models/ci/processable.rb

@@ -83,7 +83,7 @@ module Ci
 
     # Overriding scheduling_type enum's method for nil `scheduling_type`s
     def scheduling_type_dag?
-      super || find_legacy_scheduling_type == :dag
+      scheduling_type.nil? ? find_legacy_scheduling_type == :dag : super
     end
 
     # scheduling_type column of previous builds/bridges have not been populated,

app/models/ci/stage.rb

@@ -13,6 +13,7 @@ module Ci
     belongs_to :pipeline
 
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
+    has_many :latest_statuses, -> { ordered.latest }, class_name: 'CommitStatus', foreign_key: :stage_id
     has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id
     has_many :builds, foreign_key: :stage_id
     has_many :bridges, foreign_key: :stage_id

app/serializers/ci/dag_job_entity.rb

+# frozen_string_literal: true
+
+module Ci
+  class DagJobEntity < Grape::Entity
+    expose :name
+
+    expose :needs, if: -> (job, _) { job.scheduling_type_dag? } do |job|
+      job.needs.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord
+    end
+  end
+end

app/serializers/ci/dag_job_group_entity.rb

+# frozen_string_literal: true
+
+module Ci
+  class DagJobGroupEntity < Grape::Entity
+    expose :name
+    expose :size
+    expose :jobs, with: Ci::DagJobEntity
+  end
+end

app/serializers/ci/dag_pipeline_entity.rb

+# frozen_string_literal: true
+
+module Ci
+  class DagPipelineEntity < Grape::Entity
+    expose :ordered_stages_with_preloads, as: :stages, using: Ci::DagStageEntity
+
+    private
+
+    def ordered_stages_with_preloads
+      object.ordered_stages.preload(preloaded_relations) # rubocop: disable CodeReuse/ActiveRecord
+    end
+
+    def preloaded_relations
+      [
+        :project,
+        { latest_statuses: :needs }
+      ]
+    end
+  end
+end

app/serializers/ci/dag_pipeline_serializer.rb

+# frozen_string_literal: true
+
+module Ci
+  class DagPipelineSerializer < BaseSerializer
+    entity Ci::DagPipelineEntity
+  end
+end

app/serializers/ci/dag_stage_entity.rb

+# frozen_string_literal: true
+
+module Ci
+  class DagStageEntity < Grape::Entity
+    expose :name
+
+    expose :groups, with: Ci::DagJobGroupEntity
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -229,6 +229,7 @@ stages:
 - processables
 - builds
 - bridges
+- latest_statuses
 statuses:
 - project
 - pipeline

spec/serializers/ci/dag_job_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DagJobEntity do
+  let_it_be(:request) { double(:request) }
+
+  let(:job) { create(:ci_build, name: 'dag_job') }
+  let(:entity) { described_class.new(job, request: request) }
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    it 'contains the name' do
+      expect(subject[:name]).to eq 'dag_job'
+    end
+
+    context 'when job is stage scheduled' do
+      it 'does not expose needs' do
+        expect(subject).not_to include(:needs)
+      end
+    end
+
+    context 'when job is dag scheduled' do
+      context 'when job has needs' do
+        let(:job) { create(:ci_build, scheduling_type: 'dag') }
+        let!(:need) { create(:ci_build_need, build: job, name: 'compile') }
+
+        it 'exposes the array of needs' do
+          expect(subject[:needs]).to eq ['compile']
+        end
+      end
+
+      context 'when job has empty needs' do
+        let(:job) { create(:ci_build, scheduling_type: 'dag') }
+
+        it 'exposes an empty array of needs' do
+          expect(subject[:needs]).to eq []
+        end
+      end
+    end
+  end
+end

spec/serializers/ci/dag_job_group_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DagJobGroupEntity do
+  let_it_be(:request) { double(:request) }
+  let_it_be(:pipeline) { create(:ci_pipeline) }
+  let_it_be(:stage) { create(:ci_stage, pipeline: pipeline) }
+
+  let(:group) { Ci::Group.new(pipeline.project, stage, name: 'test', jobs: jobs) }
+  let(:entity) { described_class.new(group, request: request) }
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    context 'when group contains 1 job' do
+      let(:job) { create(:ci_build, stage: stage, pipeline: pipeline, name: 'test') }
+      let(:jobs) { [job] }
+
+      it 'exposes a name' do
+        expect(subject.fetch(:name)).to eq 'test'
+      end
+
+      it 'exposes the size' do
+        expect(subject.fetch(:size)).to eq 1
+      end
+
+      it 'exposes the jobs' do
+        exposed_jobs = subject.fetch(:jobs)
+
+        expect(exposed_jobs.size).to eq 1
+        expect(exposed_jobs.first.fetch(:name)).to eq 'test'
+      end
+    end
+
+    context 'when group contains multiple parallel jobs' do
+      let(:job_1) { create(:ci_build, stage: stage, pipeline: pipeline, name: 'test 1/2') }
+      let(:job_2) { create(:ci_build, stage: stage, pipeline: pipeline, name: 'test 2/2') }
+      let(:jobs) { [job_1, job_2] }
+
+      it 'exposes a name' do
+        expect(subject.fetch(:name)).to eq 'test'
+      end
+
+      it 'exposes the size' do
+        expect(subject.fetch(:size)).to eq 2
+      end
+
+      it 'exposes the jobs' do
+        exposed_jobs = subject.fetch(:jobs)
+
+        expect(exposed_jobs.size).to eq 2
+        expect(exposed_jobs.first.fetch(:name)).to eq 'test 1/2'
+        expect(exposed_jobs.last.fetch(:name)).to eq 'test 2/2'
+      end
+    end
+  end
+end

spec/serializers/ci/dag_pipeline_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DagPipelineEntity do
+  let_it_be(:request) { double(:request) }
+
+  let(:pipeline) { create(:ci_pipeline) }
+  let(:entity) { described_class.new(pipeline, request: request) }
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    context 'when pipeline is empty' do
+      it 'contains stages' do
+        expect(subject).to include(:stages)
+
+        expect(subject[:stages]).to be_empty
+      end
+    end
+
+    context 'when pipeline has jobs' do
+      let!(:build_job)  { create(:ci_build, stage: 'build',  pipeline: pipeline) }
+      let!(:test_job)   { create(:ci_build, stage: 'test',   pipeline: pipeline) }
+      let!(:deploy_job) { create(:ci_build, stage: 'deploy', pipeline: pipeline) }
+
+      it 'contains 3 stages' do
+        stages = subject[:stages]
+
+        expect(stages.size).to eq 3
+        expect(stages.map { |s| s[:name] }).to contain_exactly('build', 'test', 'deploy')
+      end
+    end
+
+    context 'when pipeline has parallel jobs and DAG needs' do
+      let!(:stage_build)  { create(:ci_stage_entity, name: 'build',  position: 1, pipeline: pipeline) }
+      let!(:stage_test)   { create(:ci_stage_entity, name: 'test',   position: 2, pipeline: pipeline) }
+      let!(:stage_deploy) { create(:ci_stage_entity, name: 'deploy', position: 3, pipeline: pipeline) }
+
+      let!(:job_build_1) { create(:ci_build, name: 'build 1', stage: 'build', pipeline: pipeline) }
+      let!(:job_build_2) { create(:ci_build, name: 'build 2', stage: 'build', pipeline: pipeline) }
+
+      let!(:job_rspec_1) { create(:ci_build, name: 'rspec 1/2', stage: 'test', pipeline: pipeline) }
+      let!(:job_rspec_2) { create(:ci_build, name: 'rspec 2/2', stage: 'test', pipeline: pipeline) }
+
+      let!(:job_jest) do
+        create(:ci_build, name: 'jest', stage: 'test', scheduling_type: 'dag', pipeline: pipeline).tap do |job|
+          create(:ci_build_need, name: 'build 1', build: job)
+        end
+      end
+
+      let!(:job_deploy_ruby) do
+        create(:ci_build, name: 'deploy_ruby', stage: 'deploy', scheduling_type: 'dag', pipeline: pipeline).tap do |job|
+          create(:ci_build_need, name: 'rspec 1/2', build: job)
+          create(:ci_build_need, name: 'rspec 2/2', build: job)
+        end
+      end
+
+      let!(:job_deploy_js) do
+        create(:ci_build, name: 'deploy_js', stage: 'deploy', scheduling_type: 'dag', pipeline: pipeline).tap do |job|
+          create(:ci_build_need, name: 'jest', build: job)
+        end
+      end
+
+      it 'performs the smallest number of queries' do
+        log = ActiveRecord::QueryRecorder.new { subject }
+
+        # stages, project, builds, build_needs
+        expect(log.count).to eq 4
+      end
+
+      it 'contains all the data' do
+        expected_result = {
+          stages: [
+            {
+              name: 'build',
+              groups: [
+                { name: 'build 1', size: 1, jobs: [{ name: 'build 1' }] },
+                { name: 'build 2', size: 1, jobs: [{ name: 'build 2' }] }
+              ]
+            },
+            {
+              name: 'test',
+              groups: [
+                { name: 'jest', size: 1, jobs: [{ name: 'jest', needs: ['build 1'] }] },
+                { name: 'rspec', size: 2, jobs: [{ name: 'rspec 1/2' }, { name: 'rspec 2/2' }] }
+              ]
+            },
+            {
+              name: 'deploy',
+              groups: [
+                { name: 'deploy_js', size: 1, jobs: [{ name: 'deploy_js', needs: ['jest'] }] },
+                { name: 'deploy_ruby', size: 1, jobs: [{ name: 'deploy_ruby', needs: ['rspec 1/2', 'rspec 2/2'] }] }
+              ]
+            }
+          ]
+        }
+
+        expect(subject.fetch(:stages)).not_to be_empty
+
+        expect(subject.fetch(:stages)[0].fetch(:name)).to eq 'build'
+        expect(subject.fetch(:stages)[0]).to eq expected_result.fetch(:stages)[0]
+
+        expect(subject.fetch(:stages)[1].fetch(:name)).to eq 'test'
+        expect(subject.fetch(:stages)[1]).to eq expected_result.fetch(:stages)[1]
+
+        expect(subject.fetch(:stages)[2].fetch(:name)).to eq 'deploy'
+        expect(subject.fetch(:stages)[2]).to eq expected_result.fetch(:stages)[2]
+      end
+    end
+  end
+end

spec/serializers/ci/dag_pipeline_serializer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DagPipelineSerializer do
+  describe '#represent' do
+    subject { described_class.new.represent(pipeline) }
+
+    let(:pipeline) { create(:ci_pipeline) }
+    let!(:job) { create(:ci_build, pipeline: pipeline) }
+
+    it 'includes stages' do
+      expect(subject[:stages]).to be_present
+      expect(subject[:stages].size).to eq 1
+    end
+  end
+end

spec/serializers/ci/dag_stage_entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DagStageEntity do
+  let_it_be(:pipeline) { create(:ci_pipeline) }
+  let_it_be(:request) { double(:request) }
+
+  let(:stage) { build(:ci_stage, pipeline: pipeline, name: 'test') }
+  let(:entity) { described_class.new(stage, request: request) }
+
+  let!(:job) { create(:ci_build, :success, pipeline: pipeline) }
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    it 'contains valid name' do
+      expect(subject[:name]).to eq 'test'
+    end
+
+    it 'contains the job groups' do
+      expect(subject).to include :groups
+      expect(subject[:groups]).not_to be_empty
+
+      job_group = subject[:groups].first
+      expect(job_group[:name]).to eq 'test'
+      expect(job_group[:size]).to eq 1
+      expect(job_group[:jobs]).not_to be_empty
+    end
+  end
+end