Merge branch '11759-dependency-list-report' into 'master'

Add DependencyList report

See merge request gitlab-org/gitlab-ee!13900

ee/app/models/ee/ci/build.rb

@@ -80,6 +80,18 @@ module EE
         license_management_report
       end
 
+      def collect_dependency_list_reports!(dependency_list_report)
+        if project.feature_available?(:dependency_list)
+          dependency_list = ::Gitlab::Ci::Parsers::Security::DependencyList.new(project, sha)
+
+          each_report(::Ci::JobArtifact::DEPENDENCY_LIST_REPORT_FILE_TYPES) do |file_type, blob|
+            dependency_list.parse!(blob, dependency_list_report)
+          end
+        end
+
+        dependency_list_report
+      end
+
       def collect_metrics_reports!(metrics_report)
         each_report(::Ci::JobArtifact::METRICS_REPORT_FILE_TYPES) do |file_type, blob|
           next unless project.feature_available?(:metrics_reports)

ee/app/models/ee/ci/job_artifact.rb

@@ -13,6 +13,7 @@ module EE
 
       SECURITY_REPORT_FILE_TYPES = %w[sast dependency_scanning container_scanning dast].freeze
       LICENSE_MANAGEMENT_REPORT_FILE_TYPES = %w[license_management].freeze
+      DEPENDENCY_LIST_REPORT_FILE_TYPES = %w[dependency_scanning].freeze
       METRICS_REPORT_FILE_TYPES = %w[metrics].freeze
 
       scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
@@ -28,6 +29,10 @@ module EE
         with_file_types(LICENSE_MANAGEMENT_REPORT_FILE_TYPES)
       end
 
+      scope :dependency_list_reports, -> do
+        with_file_types(DEPENDENCY_LIST_REPORT_FILE_TYPES)
+      end
+
       scope :metrics_reports, -> do
         with_file_types(METRICS_REPORT_FILE_TYPES)
       end

ee/app/models/ee/ci/pipeline.rb

@@ -171,6 +171,14 @@ module EE
         end
       end
 
+      def dependency_list_report
+        ::Gitlab::Ci::Reports::DependencyList::Report.new.tap do |dependency_list_report|
+          builds.latest.with_reports(::Ci::JobArtifact.dependency_list_reports).each do |build|
+            build.collect_dependency_list_reports!(dependency_list_report)
+          end
+        end
+      end
+
       def metrics_report
         ::Gitlab::Ci::Reports::Metrics::Report.new.tap do |metrics_report|
           builds.latest.with_reports(::Ci::JobArtifact.metrics_reports).each do |build|

ee/app/models/license.rb

@@ -90,6 +90,7 @@ class License < ApplicationRecord
   EEU_FEATURES = EEP_FEATURES + %i[
     security_dashboard
     dependency_scanning
+    dependency_list
     license_management
     sast
     sast_container

ee/changelogs/unreleased/11759-dependency-list-report.yml

+---
+title: Add `dependency_list` report
+merge_request: 13900
+author:
+type: added

ee/lib/gitlab/ci/parsers/security/dependency_list.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class DependencyList
+          def initialize(project, sha)
+            @formatter = Formatters::DependencyList.new(project, sha)
+          end
+
+          def parse!(json_data, report)
+            report_data = JSON.parse!(json_data)
+
+            report_data.fetch('dependency_files', []).each do |file|
+              file['dependencies'].each do |dependency|
+                report.add_dependency(formatter.format(dependency, file['package_manager'], file['path']))
+              end
+            end
+          end
+
+          private
+
+          attr_reader :formatter
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/parsers/security/formatters/dependency_list.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        module Formatters
+          class DependencyList
+            def initialize(project, sha)
+              @commit_path = ::Gitlab::Routing.url_helpers.project_blob_path(project, sha)
+            end
+
+            def format(dependency, package_manager, file_path)
+              {
+                name:     dependency['package']['name'],
+                packager: packager(package_manager),
+                location: {
+                  blob_path: blob_path(file_path),
+                  path:      file_path
+                },
+                version:  dependency['version']
+              }
+            end
+
+            private
+
+            attr_reader :commit_path
+
+            def blob_path(file_path)
+              "#{commit_path}/#{file_path}"
+            end
+
+            def packager(package_manager)
+              case package_manager
+              when 'bundler'
+                'Ruby (Bundler)'
+              when 'yarn'
+                'JavaScript (Yarn)'
+              when 'npm'
+                'JavaScript (npm)'
+              when 'pip'
+                'Python (pip)'
+              when 'maven'
+                'Java (Maven)'
+              when 'composer'
+                'PHP (Composer)'
+              else
+                package_manager
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/reports/dependency_list/report.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module DependencyList
+        class Report
+          attr_accessor :dependencies
+
+          def initialize
+            @dependencies = []
+          end
+
+          def add_dependency(dependency)
+            dependencies << dependency
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/factories/ci/builds.rb

@@ -28,6 +28,15 @@ FactoryBot.define do
       end
     end
 
+    trait :dependency_list do
+      success
+      name :dependency_scanning
+
+      after(:build) do |build|
+        build.job_artifacts << build(:ee_ci_job_artifact, :dependency_list, job: build)
+      end
+    end
+
     trait :metrics do
       after(:build) do |build|
         build.job_artifacts << build(:ee_ci_job_artifact, :metrics, job: build)

ee/spec/factories/ci/job_artifacts.rb

@@ -151,5 +151,15 @@ FactoryBot.define do
           Rails.root.join('ee/spec/fixtures/alternate_metrics.txt.gz'), 'application/x-gzip')
       end
     end
+
+    trait :dependency_list do
+      file_format :raw
+      file_type :dependency_scanning
+
+      after(:build) do |artifact, _|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('spec/fixtures/security-reports/dependency_list/gl-dependency-scanning-report.json'), 'text/plain')
+      end
+    end
   end
 end

ee/spec/factories/ci/pipelines.rb

@@ -7,11 +7,13 @@ FactoryBot.define do
       config_source :webide_source
     end
 
-    trait :with_license_management_report do
-      status :success
+    %i[license_management dependency_list].each do |report_type|
+      trait "with_#{report_type}_report".to_sym do
+        status :success
 
-      after(:build) do |pipeline, evaluator|
-        pipeline.builds << build(:ee_ci_build, :license_management, pipeline: pipeline, project: pipeline.project)
+        after(:build) do |pipeline, evaluator|
+          pipeline.builds << build(:ee_ci_build, report_type, pipeline: pipeline, project: pipeline.project)
+        end
       end
     end
 

ee/spec/lib/gitlab/ci/parsers/security/dependency_list_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Parsers::Security::DependencyList do
+  let(:parser) { described_class.new(project, sha) }
+  let(:project) { create(:project) }
+  let(:sha) { '4242424242424242' }
+
+  describe '#parse!' do
+    let(:report) { Gitlab::Ci::Reports::DependencyList::Report.new }
+
+    before do
+      artifact.each_blob do |blob|
+        parser.parse!(blob, report)
+      end
+    end
+
+    context 'with dependency_list artifact' do
+      let(:artifact) { create(:ee_ci_job_artifact, :dependency_list) }
+
+      it 'parses all files' do
+        blob_path = "/#{project.namespace.name}/#{project.name}/blob/#{sha}/yarn/yarn.lock"
+
+        expect(report.dependencies.size).to eq(21)
+        expect(report.dependencies[0][:name]).to eq('mini_portile2')
+        expect(report.dependencies[0][:version]).to eq('2.2.0')
+        expect(report.dependencies[0][:packager]).to eq('Ruby (Bundler)')
+        expect(report.dependencies[12][:packager]).to eq('JavaScript (Yarn)')
+        expect(report.dependencies[0][:location][:path]).to eq('rails/Gemfile.lock')
+        expect(report.dependencies[12][:location][:blob_path]).to eq(blob_path)
+      end
+    end
+
+    context 'with old dependency scanning artifact' do
+      let(:artifact) { create(:ee_ci_job_artifact, :dependency_scanning) }
+
+      it 'returns empty list of dependencies' do
+        expect(report.dependencies.size).to eq(0)
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/parsers/security/formatters/dependency_list_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Parsers::Security::Formatters::DependencyList do
+  let(:formatter) { described_class.new(project, sha) }
+  let(:project) { create(:project) }
+  let(:sha) { '4242424242424242' }
+
+  let(:parsed_report) do
+    JSON.parse!(
+      File.read(
+        Rails.root.join('spec/fixtures/security-reports/dependency_list/gl-dependency-scanning-report.json')
+      )
+    )
+  end
+
+  describe '#format' do
+    let(:dependency) { parsed_report['dependency_files'][0]['dependencies'][0] }
+    let(:package_manager) { 'bundler' }
+    let(:file_path) { 'rails/Gemfile.lock' }
+
+    it 'format report into a right format' do
+      data = formatter.format(dependency, package_manager, file_path)
+      blob_path = "/#{project.namespace.name}/#{project.name}/blob/#{sha}/rails/Gemfile.lock"
+
+      expect(data[:name]).to eq('mini_portile2')
+      expect(data[:packager]).to eq('Ruby (Bundler)')
+      expect(data[:location][:blob_path]).to eq(blob_path)
+      expect(data[:location][:path]).to eq('rails/Gemfile.lock')
+      expect(data[:version]).to eq('2.2.0')
+    end
+  end
+
+  describe 'packager' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:packager, :expected) do
+      'bundler'  | 'Ruby (Bundler)'
+      'yarn'     | 'JavaScript (Yarn)'
+      'npm'      | 'JavaScript (npm)'
+      'pip'      | 'Python (pip)'
+      'maven'    | 'Java (Maven)'
+      'composer' | 'PHP (Composer)'
+      ''         | ''
+    end
+
+    with_them do
+      it 'substitutes with right values' do
+        expect(formatter.send(:packager, packager)).to eq(expected)
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/reports/dependency_list/report_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Reports::DependencyList::Report do
+  let(:report) { described_class.new }
+
+  describe '#add_dependency' do
+    let(:dependency) { { name: 'gitlab', version: '12.0' } }
+
+    subject { report.add_dependency(dependency) }
+
+    it 'stores given dependency params in the map' do
+      subject
+
+      expect(report.dependencies).to eq([dependency])
+    end
+  end
+end

ee/spec/models/ci/build_spec.rb

@@ -292,6 +292,36 @@ describe Ci::Build do
     end
   end
 
+  describe '#collect_dependency_list_reports!' do
+    let!(:artifact) { create(:ee_ci_job_artifact, :dependency_list, job: job, project: job.project) }
+    let(:dependency_list_report) { Gitlab::Ci::Reports::DependencyList::Report.new }
+
+    subject { job.collect_dependency_list_reports!(dependency_list_report) }
+
+    context 'with available licensed feature' do
+      before do
+        stub_licensed_features(dependency_list: true)
+      end
+
+      it 'parses blobs and add the results to the report' do
+        subject
+        blob_path = "/#{group.name}/#{project.name}/blob/#{job.sha}/yarn/yarn.lock"
+
+        expect(dependency_list_report.dependencies.count).to eq(21)
+        expect(dependency_list_report.dependencies[0][:name]).to eq('mini_portile2')
+        expect(dependency_list_report.dependencies[20][:location][:blob_path]).to eq(blob_path)
+      end
+    end
+
+    context 'with disabled licensed feature' do
+      it 'does NOT parse dependency list report' do
+        subject
+
+        expect(dependency_list_report.dependencies.count).to eq(0)
+      end
+    end
+  end
+
   describe '#collect_metrics_reports!' do
     subject { job.collect_metrics_reports!(metrics_report) }
 

ee/spec/models/ci/pipeline_spec.rb

@@ -330,6 +330,39 @@ describe Ci::Pipeline do
     end
   end
 
+  describe '#dependency_list_reports' do
+    subject { pipeline.dependency_list_report }
+
+    before do
+      stub_licensed_features(dependency_list: true)
+    end
+
+    context 'when pipeline has a build with dependency list reports' do
+      let!(:build) { create(:ci_build, :success, name: 'dependency_list', pipeline: pipeline, project: project) }
+      let!(:artifact) { create(:ee_ci_job_artifact, :dependency_list, job: build, project: project) }
+
+      it 'returns a dependency list report with collected data' do
+        expect(subject.dependencies.count).to eq(21)
+        expect(subject.dependencies[0][:name]).to eq('mini_portile2')
+      end
+
+      context 'when builds are retried' do
+        let!(:build) { create(:ci_build, :retried, :success, name: 'dependency_list', pipeline: pipeline, project: project) }
+        let!(:artifact) { create(:ee_ci_job_artifact, :dependency_list, job: build, project: project) }
+
+        it 'does not take retried builds into account' do
+          expect(subject.dependencies).to be_empty
+        end
+      end
+    end
+
+    context 'when pipeline does not have any builds with dependency_list reports' do
+      it 'returns an empty dependency_list report' do
+        expect(subject.dependencies).to be_empty
+      end
+    end
+  end
+
   describe '#metrics_report' do
     subject { pipeline.metrics_report }