Merge branch 'track_secure_scans' into 'master'

Track Secure scans

See merge request gitlab-org/gitlab!64578

ee/app/models/ee/ci/build.rb

@@ -87,6 +87,10 @@ module EE
           artifacts_metadata?
       end
 
+      def has_security_reports?
+        job_artifacts.security_reports.any?
+      end
+
       def collect_security_reports!(security_reports)
         each_report(::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES) do |file_type, blob, report_artifact|
           security_reports.get_report(file_type, report_artifact).tap do |security_report|
@@ -99,6 +103,17 @@ module EE
         end
       end
 
+      def unmerged_security_reports
+        security_reports = ::Gitlab::Ci::Reports::Security::Reports.new(pipeline)
+
+        each_report(::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES) do |file_type, blob, report_artifact|
+          report = security_reports.get_report(file_type, report_artifact)
+          parse_raw_security_artifact_blob(report, blob)
+        end
+
+        security_reports
+      end
+
       def collect_license_scanning_reports!(license_scanning_report)
         return license_scanning_report unless project.feature_available?(:license_scanning)
 
@@ -213,11 +228,15 @@ module EE
 
       def parse_security_artifact_blob(security_report, blob)
         report_clone = security_report.clone_as_blank
-        signatures_enabled = ::Feature.enabled?(:vulnerability_finding_tracking_signatures, project) && project.licensed_feature_available?(:vulnerability_finding_signatures)
-        ::Gitlab::Ci::Parsers.fabricate!(security_report.type, blob, report_clone, signatures_enabled).parse!
+        parse_raw_security_artifact_blob(report_clone, blob)
         security_report.merge!(report_clone)
       end
 
+      def parse_raw_security_artifact_blob(security_report, blob)
+        signatures_enabled = ::Feature.enabled?(:vulnerability_finding_tracking_signatures, project) && project.licensed_feature_available?(:vulnerability_finding_signatures)
+        ::Gitlab::Ci::Parsers.fabricate!(security_report.type, blob, security_report, signatures_enabled).parse!
+      end
+
       def ee_runner_required_feature_names
         strong_memoize(:ee_runner_required_feature_names) do
           EE_RUNNER_FEATURES.select do |feature, method|

ee/app/services/security/track_scan_service.rb

+# frozen_string_literal: true
+
+# This service track executions of a Secure analyzer scan using Snowplow.
+#
+# @param build [Ci::Build] the build that ran the scan.
+module Security
+  class TrackScanService
+    SECURE_SCAN_SCHEMA_URL = 'iglu:com.gitlab/secure_scan/jsonschema/1-0-1'
+
+    def initialize(build)
+      @build = build
+    end
+
+    def execute
+      build.unmerged_security_reports.reports.each { |report_type, report| track_scan_event(report_type, report) }
+    end
+
+    private
+
+    attr_reader :build
+
+    def track_scan_event(report_type, report)
+      context = SnowplowTracker::SelfDescribingJson.new(SECURE_SCAN_SCHEMA_URL, data_to_track(report_type, report))
+
+      idempotency_key = [build.project_id, build.id, scan_type(report, report_type), report&.scan&.start_time || ""].join("::")
+
+      ::Gitlab::Tracking.event('secure::scan',
+                               'scan',
+                               context: [context],
+                               idempotency_key: Digest::SHA256.hexdigest(idempotency_key),
+                               user: build.user_id,
+                               project: build.project_id)
+    end
+
+    def data_to_track(report_type, report)
+      analyzer = report&.analyzer
+      scan = report&.scan
+      primary_scanner = report&.primary_scanner
+
+      {
+        analyzer: analyzer&.id,
+        analyzer_vendor: analyzer&.vendor,
+        analyzer_version: analyzer&.version,
+        end_time: scan&.end_time,
+        report_schema_version: report&.version,
+        scan_type: scan_type(report, report_type),
+        scanner: primary_scanner&.external_id,
+        scanner_vendor: primary_scanner&.vendor,
+        scanner_version: primary_scanner&.version,
+        start_time: scan&.start_time,
+        status: scan&.status || 'success'
+      }
+    end
+
+    def scan_type(report, report_type)
+      report&.scan&.type || report_type
+    end
+  end
+end

ee/app/workers/all_queues.yml

@@ -828,6 +828,15 @@
   :idempotent:
   :tags:
   - :exclude_from_kubernetes
+- :name: security_scans:security_track_secure_scans
+  :worker_name: Security::TrackSecureScansWorker
+  :feature_category: :vulnerability_management
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :cpu
+  :weight: 2
+  :idempotent:
+  :tags: []
 - :name: security_scans:store_security_reports
   :worker_name: StoreSecurityReportsWorker
   :feature_category: :vulnerability_management

ee/app/workers/ee/ci/build_finished_worker.rb

@@ -12,6 +12,10 @@ module EE
           RequirementsManagement::ProcessRequirementsReportsWorker.perform_async(build.id)
         end
 
+        if ::Gitlab.com? && build.has_security_reports?
+          ::Security::TrackSecureScansWorker.perform_async(build.id)
+        end
+
         super
       end
     end

ee/app/workers/security/track_secure_scans_worker.rb

+# frozen_string_literal: true
+
+# Worker for tracking each run of a security scan.
+module Security
+  class TrackSecureScansWorker # rubocop:disable Scalability/IdempotentWorker
+    include ApplicationWorker
+    include SecurityScansQueue
+
+    sidekiq_options retry: 1
+
+    worker_resource_boundary :cpu
+
+    def perform(build_id)
+      build = ::Ci::Build.find_by_id(build_id)
+      return unless build
+
+      ::Security::TrackScanService.new(build).execute
+    end
+  end
+end

ee/spec/factories/ci/job_artifacts.rb

@@ -42,6 +42,16 @@ FactoryBot.define do
       end
     end
 
+    trait :dast_14_0_2 do
+      file_format { :raw }
+      file_type { :dast }
+
+      after(:build) do |artifact, _|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('ee/spec/fixtures/security_reports/master/gl-dast-report-14.0.2.json'), 'application/json')
+      end
+    end
+
     trait :dast_feature_branch do
       file_format { :raw }
       file_type { :dast }

ee/spec/fixtures/security_reports/master/gl-dast-report-14.0.2.json

+{
+  "scan": {
+    "analyzer": {
+      "id": "gitlab-dast",
+      "name": "GitLab DAST",
+      "url": "https://docs.gitlab.com/ee/user/application_security/dast/",
+      "version": "2.0.1",
+      "vendor": {
+        "name": "GitLab"
+      }
+    },
+    "end_time": "2021-06-11T07:27:50",
+    "messages": [],
+    "scanned_resources": [
+      {
+        "method": "GET",
+        "type": "url",
+        "url": "http://pancakes/"
+      },
+      {
+        "method": "GET",
+        "type": "url",
+        "url": "http://pancakes/components.jsx"
+      },
+      {
+        "method": "GET",
+        "type": "url",
+        "url": "http://pancakes/styles.css"
+      }
+    ],
+    "scanner": {
+      "id": "zaproxy-browserker",
+      "name": "OWASP Zed Attack Proxy (ZAP) and Browserker",
+      "url": "https://www.zaproxy.org",
+      "version": "D-2020-08-26",
+      "vendor": {
+        "name": "GitLab"
+      }
+    },
+    "start_time": "2021-06-11T07:26:17",
+    "status": "success",
+    "type": "dast"
+  },
+  "version": "14.0.2",
+  "vulnerabilities": [
+    {
+      "category": "dast",
+      "confidence": "High",
+      "cve": "10038-aggregated",
+      "details": {
+        "urls": {
+          "name": "URLs",
+          "type": "list",
+          "items": [
+            {
+              "type": "url",
+              "href": "http://pancakes/"
+            }
+          ]
+        }
+      },
+      "description": "Content Security Policy (CSP) is an added layer of security that helps to detect and mitigate certain types of attacks, including Cross Site Scripting (XSS) and data injection attacks. These attacks are used for everything from data theft to site defacement or distribution of malware. CSP provides a set of standard HTTP headers that allow website owners to declare approved sources of content that browsers should be allowed to load on that page \u2014 covered types are JavaScript, CSS, HTML frames, fonts, images and embeddable objects such as Java applets, ActiveX, audio and video files.",
+      "discovered_at": "2021-06-11T07:26:24.632",
+      "evidence": {
+        "request": {
+          "headers": [
+            {
+              "name": "Accept",
+              "value": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+            },
+            {
+              "name": "Accept-Language",
+              "value": "en-US"
+            },
+            {
+              "name": "Cookie",
+              "value": "dast_scan=browserker"
+            },
+            {
+              "name": "Host",
+              "value": "pancakes"
+            },
+            {
+              "name": "Proxy-Connection",
+              "value": "keep-alive"
+            },
+            {
+              "name": "Upgrade-Insecure-Requests",
+              "value": "1"
+            },
+            {
+              "name": "User-Agent",
+              "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.0 Safari/537.36"
+            },
+            {
+              "name": "Via",
+              "value": "GitLab DAST/ZAP v2.0.0"
+            },
+            {
+              "name": "Via-Scanner",
+              "value": "Browserker"
+            },
+            {
+              "name": "x-scanner",
+              "value": "browserker"
+            },
+            {
+              "name": "x-scanner-app",
+              "value": "pancake"
+            }
+          ],
+          "method": "GET",
+          "url": "http://pancakes/"
+        },
+        "response": {
+          "headers": [
+            {
+              "name": "Accept",
+              "value": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+            },
+            {
+              "name": "Accept-Language",
+              "value": "en-US"
+            },
+            {
+              "name": "Cookie",
+              "value": "dast_scan=browserker"
+            },
+            {
+              "name": "Host",
+              "value": "pancakes"
+            },
+            {
+              "name": "Proxy-Connection",
+              "value": "keep-alive"
+            },
+            {
+              "name": "Upgrade-Insecure-Requests",
+              "value": "1"
+            },
+            {
+              "name": "User-Agent",
+              "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.0 Safari/537.36"
+            },
+            {
+              "name": "Via",
+              "value": "GitLab DAST/ZAP v2.0.0"
+            },
+            {
+              "name": "Via-Scanner",
+              "value": "Browserker"
+            },
+            {
+              "name": "x-scanner",
+              "value": "browserker"
+            },
+            {
+              "name": "x-scanner-app",
+              "value": "pancake"
+            }
+          ],
+          "reason_phrase": "OK",
+          "status_code": 200
+        },
+        "summary": ""
+      },
+      "id": "47308329-7297-4f8f-b98e-1cc33f26fad4",
+      "identifiers": [
+        {
+          "name": "Content Security Policy (CSP) Header Not Set",
+          "type": "ZAProxy_PluginId",
+          "url": "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md",
+          "value": "10038"
+        },
+        {
+          "name": "CWE-16",
+          "type": "CWE",
+          "url": "https://cwe.mitre.org/data/definitions/16.html",
+          "value": "16"
+        }
+      ],
+      "links": [
+        {
+          "url": "https://developer.mozilla.org/en-US/docs/Web/Security/CSP/Introducing_Content_Security_Policy"
+        }
+      ],
+      "location": {
+        "hostname": "http://pancakes",
+        "method": "",
+        "param": "",
+        "path": ""
+      },
+      "message": "Content Security Policy (CSP) Header Not Set",
+      "scanner": {
+        "id": "zaproxy-browserker",
+        "name": "ZAProxy and Browserker"
+      },
+      "severity": "Medium",
+      "solution": "Ensure that your web server, application server, load balancer, etc. is configured to set the Content-Security-Policy header, to achieve optimal browser support: \"Content-Security-Policy\" for Chrome 25+, Firefox 23+ and Safari 7+, \"X-Content-Security-Policy\" for Firefox 4.0+ and Internet Explorer 10+, and \"X-WebKit-CSP\" for Chrome 14+ and Safari 6+."
+    }
+  ]
+}

ee/spec/models/ci/build_spec.rb

@@ -343,6 +343,54 @@ RSpec.describe Ci::Build do
     end
   end
 
+  describe '#has_security_reports?' do
+    subject { job.has_security_reports? }
+
+    context 'when build has a security report' do
+      let!(:artifact) { create(:ee_ci_job_artifact, :sast, job: job, project: job.project) }
+
+      it { is_expected.to be true }
+    end
+
+    context 'when build does not have a security report' do
+      it { is_expected.to be false }
+    end
+  end
+
+  describe '#unmerged_security_reports' do
+    subject(:security_reports) { job.unmerged_security_reports }
+
+    context 'when build has a security report' do
+      context 'when there is a sast report' do
+        let!(:artifact) { create(:ee_ci_job_artifact, :sast, job: job, project: job.project) }
+
+        it 'parses blobs and add the results to the report' do
+          expect(security_reports.get_report('sast', artifact).findings.size).to eq(5)
+        end
+      end
+
+      context 'when there are multiple reports' do
+        let!(:sast_artifact) { create(:ee_ci_job_artifact, :sast, job: job, project: job.project) }
+        let!(:ds_artifact) { create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project) }
+        let!(:cs_artifact) { create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project) }
+        let!(:dast_artifact) { create(:ee_ci_job_artifact, :dast, job: job, project: job.project) }
+
+        it 'parses blobs and adds unmerged results to the reports' do
+          expect(security_reports.get_report('sast', sast_artifact).findings.size).to eq(5)
+          expect(security_reports.get_report('dependency_scanning', ds_artifact).findings.size).to eq(4)
+          expect(security_reports.get_report('container_scanning', cs_artifact).findings.size).to eq(8)
+          expect(security_reports.get_report('dast', dast_artifact).findings.size).to eq(24)
+        end
+      end
+    end
+
+    context 'when build has no security reports' do
+      it 'has no parsed reports' do
+        expect(security_reports.reports).to be_empty
+      end
+    end
+  end
+
   describe '#collect_security_reports!' do
     let(:security_reports) { ::Gitlab::Ci::Reports::Security::Reports.new(pipeline) }
 

ee/spec/services/security/track_scan_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Security::TrackScanService do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :public) }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project, user: user) }
+
+  let_it_be_with_reload(:build) { create(:ee_ci_build, pipeline: pipeline, user: user) }
+
+  describe '#execute' do
+    subject { described_class.new(build).execute }
+
+    context 'report has all metadata' do
+      let_it_be(:dast_artifact) { create(:ee_ci_job_artifact, :dast_14_0_2, job: build) }
+
+      before do
+        allow(Digest::SHA256).to receive(:hexdigest).and_return('82fc6391e4be61e03e51fa8c5c6bfc32b3d3f0065ad2fe0a01211606952b8d82')
+      end
+
+      it 'tracks the scan event', :snowplow do
+        subject
+
+        expect_snowplow_event(
+          category: 'secure::scan',
+          action: 'scan',
+          context: [{
+                      schema: described_class::SECURE_SCAN_SCHEMA_URL,
+                      data: {
+                        analyzer: 'gitlab-dast',
+                        analyzer_vendor: 'GitLab',
+                        analyzer_version: '2.0.1',
+                        end_time: '2021-06-11T07:27:50',
+                        scan_type: 'dast',
+                        scanner: 'zaproxy-browserker',
+                        scanner_vendor: 'GitLab',
+                        scanner_version: 'D-2020-08-26',
+                        start_time: '2021-06-11T07:26:17',
+                        status: 'success',
+                        report_schema_version: '14.0.2'
+                      }
+                    }],
+          idempotency_key: '82fc6391e4be61e03e51fa8c5c6bfc32b3d3f0065ad2fe0a01211606952b8d82',
+          user: user.id,
+          project: project.id)
+      end
+    end
+
+    context 'report is missing metadata' do
+      let_it_be(:dast_artifact) { create(:ee_ci_job_artifact, :dast_missing_scan_field, job: build) }
+
+      before do
+        allow(Digest::SHA256).to receive(:hexdigest).and_return('62bc6c62686b327dbf420f8891e1418406b60f49e574b6ff22f4d6a272dbc595')
+      end
+
+      it 'tracks the scan event', :snowplow do
+        subject
+
+        expect_snowplow_event(
+          category: 'secure::scan',
+          action: 'scan',
+          context: [{
+                      schema: described_class::SECURE_SCAN_SCHEMA_URL,
+                      data: {
+                        analyzer: nil,
+                        analyzer_vendor: nil,
+                        analyzer_version: nil,
+                        end_time: nil,
+                        scan_type: 'dast',
+                        scanner: "zaproxy",
+                        scanner_vendor: nil,
+                        scanner_version: nil,
+                        start_time: nil,
+                        status: 'success',
+                        report_schema_version: '2.5'
+                      }
+                    }],
+          idempotency_key: '62bc6c62686b327dbf420f8891e1418406b60f49e574b6ff22f4d6a272dbc595',
+          user: user.id,
+          project: project.id)
+      end
+    end
+  end
+end

ee/spec/workers/ee/ci/build_finished_worker_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 RSpec.describe Ci::BuildFinishedWorker do
   let(:ci_runner) { create(:ci_runner) }
-  let(:build) { create(:ee_ci_build, :success, runner: ci_runner) }
+  let(:build) { create(:ee_ci_build, :sast, :success, runner: ci_runner) }
   let(:project) { build.project }
   let(:namespace) { project.shared_runners_limit_namespace }
 
@@ -49,6 +49,22 @@ RSpec.describe Ci::BuildFinishedWorker do
           subject
         end
       end
+
+      it 'tracks secure scans' do
+        expect(::Security::TrackSecureScansWorker).to receive(:perform_async)
+
+        subject
+      end
+
+      context 'when build does not have a security report' do
+        let(:build) { create(:ee_ci_build, :success, runner: ci_runner) }
+
+        it 'does not track secure scans' do
+          expect(::Security::TrackSecureScansWorker).not_to receive(:perform_async)
+
+          subject
+        end
+      end
     end
 
     context 'when not on .com' do
@@ -61,6 +77,12 @@ RSpec.describe Ci::BuildFinishedWorker do
 
         subject
       end
+
+      it 'does not track secure scans' do
+        expect(::Security::TrackSecureScansWorker).not_to receive(:perform_async)
+
+        subject
+      end
     end
 
     it 'does not schedule processing of requirement reports by default' do

ee/spec/workers/security/track_secure_scans_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Security::TrackSecureScansWorker do
+  let!(:ci_build) { create(:ee_ci_build) }
+
+  describe '#perform' do
+    subject { described_class.new.perform(ci_build.id) }
+
+    context 'build is found' do
+      it 'executes track service' do
+        expect(Security::TrackScanService).to receive(:new).with(ci_build).and_call_original
+
+        subject
+      end
+    end
+
+    context 'build is not found' do
+      let(:ci_build) { build(:ee_ci_build) }
+
+      it 'skips track service' do
+        expect(Security::TrackScanService).not_to receive(:new)
+
+        subject
+      end
+    end
+  end
+end

spec/workers/every_sidekiq_worker_spec.rb

@@ -418,6 +418,7 @@ RSpec.describe 'Every Sidekiq worker' do
         'ScanSecurityReportSecretsWorker' => 17,
         'Security::AutoFixWorker' => 3,
         'Security::StoreScansWorker' => 3,
+        'Security::TrackSecureScansWorker' => 1,
         'SelfMonitoringProjectCreateWorker' => 3,
         'SelfMonitoringProjectDeleteWorker' => 3,
         'ServiceDeskEmailReceiverWorker' => 3,