Merge branch 'rf-migrate-vulnerability-dismissals' into 'master'

Migrate dismissals to vulnerabilities

See merge request gitlab-org/gitlab!29711

changelogs/unreleased/migrate-vulnerability-dismissals.yml

+---
+title: Migration of dismissals to vulnerabilities
+merge_request: 29711
+author:
+type: other

db/post_migrate/20200416111111_migrate_vulnerability_dismissals.rb

+# frozen_string_literal: true
+
+class MigrateVulnerabilityDismissals < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  MIGRATION = 'UpdateVulnerabilitiesToDismissed'.freeze
+  BATCH_SIZE = 500
+  DELAY_INTERVAL = 2.minutes.to_i
+
+  class Vulnerability < ActiveRecord::Base
+    self.table_name = 'vulnerabilities'
+    self.inheritance_column = :_type_disabled
+
+    include ::EachBatch
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    Vulnerability.select('project_id').group(:project_id).each_batch(of: BATCH_SIZE, column: "project_id") do |project_batch, index|
+      batch_delay = (index - 1) * BATCH_SIZE * DELAY_INTERVAL
+
+      project_batch.each_with_index do |project, project_batch_index|
+        project_delay = project_batch_index * DELAY_INTERVAL
+        migrate_in(batch_delay + project_delay, MIGRATION, project[:project_id])
+      end
+    end
+  end
+
+  def down
+    # nothing to do
+  end
+end

db/structure.sql

@@ -13256,6 +13256,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200415161021
 20200415161206
 20200415192656
+20200416111111
 20200416120128
 20200416120354
 \.

ee/lib/ee/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      # This migration updates the states of vulnerabilities records to dismissed if the corresponding
+      # vulnerability_occurrences record was dismissed.
+      module UpdateVulnerabilitiesToDismissed
+        extend ::Gitlab::Utils::Override
+
+        VULNERABILITY_DETECTED = 1
+        VULNERABILITY_DISMISSED = 2
+        VULNERABILITY_FEEDBACK_DISMISSAL = 0
+
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+          self.inheritance_column = :_type_disabled
+        end
+
+        override :perform
+        def perform(project_id)
+          project = Project.find_by(id: project_id)
+
+          return unless project
+          return if project.archived? || project.pending_delete?
+
+          update_vulnerability_to_dismissed(project.id)
+        end
+
+        private
+
+        def update_vulnerability_to_dismissed(project_id)
+          update_vulnerability_to_dismissed_sql = <<-SQL
+            UPDATE vulnerabilities
+            SET state = #{VULNERABILITY_DISMISSED}
+            FROM vulnerability_occurrences
+            WHERE vulnerability_occurrences.vulnerability_id = vulnerabilities.id
+              AND vulnerabilities.state = #{VULNERABILITY_DETECTED}
+              AND (
+                EXISTS (
+                  SELECT 1
+                  FROM vulnerability_feedback
+                  WHERE vulnerability_occurrences.report_type = vulnerability_feedback.category
+                    AND vulnerability_occurrences.project_id = vulnerability_feedback.project_id
+                    AND ENCODE(vulnerability_occurrences.project_fingerprint, 'HEX') = vulnerability_feedback.project_fingerprint
+                    AND vulnerability_feedback.feedback_type = #{VULNERABILITY_FEEDBACK_DISMISSAL}
+                )
+              )
+              AND vulnerability_occurrences.project_id = #{project_id};
+          SQL
+          connection.execute(update_vulnerability_to_dismissed_sql)
+        rescue => e
+          logger.warn(
+            message: 'update_vulnerability_to_dismissed errored out',
+            project_id: project_id,
+            error: e.message
+          )
+        end
+
+        def connection
+          @connection ||= ActiveRecord::Base.connection
+        end
+
+        def logger
+          @logger ||= ::Gitlab::BackgroundMigration::Logger.build
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/update_vulnerabilities_to_dismissed_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed, :migration, schema: 20200416111111 do
+  let(:users) { table(:users) }
+  let(:projects) { table(:projects) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:feedback) { table(:vulnerability_feedback) }
+
+  let(:severity) { Vulnerabilities::Occurrence::SEVERITY_LEVELS[:unknown] }
+  let(:confidence) { Vulnerabilities::Occurrence::CONFIDENCE_LEVELS[:medium] }
+  let(:report_type) { Vulnerabilities::Occurrence::REPORT_TYPES[:sast] }
+
+  let!(:user) { users.create!(id: 13, email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let!(:project) { projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab') }
+
+  let(:scanner) do
+    scanners.create!(id: 6, project_id: project.id, external_id: 'clair', name: 'Security Scanner')
+  end
+
+  let(:identifier) do
+    identifiers.create!(id: 7,
+      project_id: 123,
+      fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c7',
+      external_type: 'SECURITY_ID',
+      external_id: 'SECURITY_0',
+      name: 'SECURITY_IDENTIFIER 0')
+  end
+
+  context 'vulnerability_occurrence has an associated vulnerability' do
+    let!(:vulnerability) { vulnerabilities.create!(vuln_params) }
+    let!(:pipeline) { pipelines.create!(id: 234, project_id: project.id, ref: 'master', sha: 'adf43c3a', status: :success, user_id: user.id) }
+
+    let!(:vulnerability_occurrence) do
+      vulnerability_occurrences.create!(
+        id: 1, report_type: vulnerability.report_type, name: 'finding_1',
+        primary_identifier_id: identifier.id, uuid: 'abc', project_fingerprint: 'abc123',
+        location_fingerprint: 'abc456', project_id: project.id, scanner_id: scanner.id, severity: severity,
+        confidence: confidence, metadata_version: 'sast:1.0', raw_metadata: '{}', vulnerability_id: vulnerability.id)
+    end
+
+    context 'has been dismissed' do
+      let!(:dismiss_feedback) do
+        feedback.create!(category: vulnerability_occurrence.report_type, feedback_type: 0,
+               project_id: project.id, project_fingerprint: vulnerability_occurrence.project_fingerprint.unpack1('H*'),
+               author_id: user.id)
+      end
+
+      it 'vulnerability should now have state of dismissed' do
+        expect(vulnerability.state)
+          .to eq(described_class::VULNERABILITY_DETECTED)
+
+        expect { described_class.new.perform(project.id) }
+          .to change { vulnerability.reload.state }
+          .from(described_class::VULNERABILITY_DETECTED)
+          .to(described_class::VULNERABILITY_DISMISSED)
+      end
+
+      context 'project is archived' do
+        let!(:project) { projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab', archived: true) }
+
+        it 'vulnerability should remain in detected state' do
+          expect(vulnerability.state).to eq(described_class::VULNERABILITY_DETECTED)
+
+          expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.state }.from(described_class::VULNERABILITY_DETECTED)
+        end
+      end
+
+      context 'project is set to be deleted' do
+        let!(:project) { projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab', pending_delete: true) }
+
+        it 'vulnerability should remain in detected state' do
+          expect(vulnerability.state).to eq(described_class::VULNERABILITY_DETECTED)
+
+          expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.state }.from(described_class::VULNERABILITY_DETECTED)
+        end
+      end
+    end
+
+    context 'has not been dismissed' do
+      it 'vulnerability should remain in detected state' do
+        expect(vulnerability.state)
+          .to eq(described_class::VULNERABILITY_DETECTED)
+
+        expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.state }
+          .from(described_class::VULNERABILITY_DETECTED)
+      end
+    end
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: described_class::VULNERABILITY_DETECTED,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_id: project.id,
+      author_id: user.id
+    }
+  end
+end

ee/spec/migrations/migrate_vulnerability_dismissals_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200416111111_migrate_vulnerability_dismissals.rb')
+
+describe MigrateVulnerabilityDismissals, :migration, :sidekiq do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let!(:user) { users.create!(id: 13, email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let(:project_1) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
+  let(:project_2) { projects.create!(name: 'gitlab2', path: 'gitlab-ce', namespace_id: namespace.id) }
+
+  let(:vulnerabilities) { table(:vulnerabilities) }
+
+  let(:detected_state) {  Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed::VULNERABILITY_DETECTED }
+  let(:severity) { Vulnerabilities::Occurrence::SEVERITY_LEVELS[:unknown] }
+  let(:confidence) { Vulnerabilities::Occurrence::CONFIDENCE_LEVELS[:medium] }
+  let(:report_type) { Vulnerabilities::Occurrence::REPORT_TYPES[:sast] }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_1.id }) )
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_2.id }) )
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_2.id }) )
+  end
+
+  context 'EE' do
+    before do
+      allow(Gitlab).to receive(:ee?).and_return(true)
+    end
+
+    it 'creates background job for each project' do
+      migrate!
+
+      expect(BackgroundMigrationWorker.jobs.size).to eq 2
+    end
+
+    it 'calls the UpdateVulnerabilitiesToDismissed migration' do
+      expect(BackgroundMigrationWorker).to receive(:perform_in).with(0, 'UpdateVulnerabilitiesToDismissed', project_1.id )
+      expect(BackgroundMigrationWorker).to receive(:perform_in).with(120, 'UpdateVulnerabilitiesToDismissed', project_2.id )
+
+      migrate!
+    end
+  end
+
+  context 'FOSS' do
+    before do
+      allow(Gitlab).to receive(:ee?).and_return(false)
+    end
+
+    it 'skips migration for FOSS' do
+      Sidekiq::Testing.fake! do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq 0
+      end
+    end
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: detected_state,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      author_id: user.id
+    }
+  end
+end

lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class UpdateVulnerabilitiesToDismissed
+      def perform(project_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed')